diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 68d8f2df2..ebd650c1f 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -44,8 +44,7 @@ emqx_test(){ echo "running ${packagename} start" "${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' - do + while ! curl http://localhost:8081/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -116,8 +115,7 @@ running_test(){ emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' - do + while ! curl http://localhost:8081/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -134,8 +132,7 @@ running_test(){ || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) IDLE_TIME=0 - while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted' - do + while ! curl http://localhost:8081/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx service error" diff --git a/.ci/docker-compose-file/conf.env b/.ci/docker-compose-file/conf.env index 93dfecd2b..0b1b7c512 100644 --- a/.ci/docker-compose-file/conf.env +++ b/.ci/docker-compose-file/conf.env @@ -11,3 +11,4 @@ EMQX_AUTH__PGSQL__DATABASE=mqtt EMQX_AUTH__REDIS__SERVER=redis_server:6379 EMQX_AUTH__REDIS__PASSWORD=public CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ +HOCON_ENV_OVERRIDE_PREFIX=EMQX_ diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 18e1bb6cc..6bc8e67e2 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -9,7 +9,7 @@ services: - emqx2 volumes: - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - - ../../etc/certs:/usr/local/etc/haproxy/certs + - ../../apps/emqx/etc/certs:/usr/local/etc/haproxy/certs ports: - "18083:18083" # - "1883:1883" diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index c936a426b..9126d97ac 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -48,7 +48,7 @@ jobs: export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap" + docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_ldap" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" - uses: actions/upload-artifact@v1 if: failure() @@ -86,7 +86,7 @@ jobs: if: matrix.connect_type == 'tls' run: | cat <<-EOF >> "$GITHUB_ENV" - EMQX_AUTH__MONGO__SSL=on + EMQX_AUTH__MONGO__SSL__ENABLE=on EMQX_AUTH__MONGO__SSL__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem EMQX_AUTH__MONGO__SSL__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem EMQX_AUTH__MONGO__SSL__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem @@ -98,7 +98,7 @@ jobs: MONGO_TAG: ${{ matrix.mongo_tag }} if: matrix.connect_type == 'tcp' run: | - echo EMQX_AUTH__MONGO__SSL=off >> "$GITHUB_ENV" + echo EMQX_AUTH__MONGO__SSL__ENABLE=off >> "$GITHUB_ENV" - name: setup if: matrix.network_type == 'ipv4' run: | @@ -117,7 +117,7 @@ jobs: export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo" + docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mongo" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo" - uses: actions/upload-artifact@v1 if: failure() @@ -160,10 +160,10 @@ jobs: if: matrix.connect_type == 'tls' run: | cat <<-EOF >> "$GITHUB_ENV" + EMQX_AUTH__MYSQL__SSL__ENABLE=on EMQX_AUTH__MYSQL__USERNAME=ssluser EMQX_AUTH__MYSQL__PASSWORD=public EMQX_AUTH__MYSQL__DATABASE=mqtt - EMQX_AUTH__MYSQL__SSL=on EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem @@ -179,7 +179,7 @@ jobs: EMQX_AUTH__MYSQL__USERNAME=root EMQX_AUTH__MYSQL__PASSWORD=public EMQX_AUTH__MYSQL__DATABASE=mqtt - EMQX_AUTH__MYSQL__SSL=off + EMQX_AUTH__MYSQL__SSL__ENABLE=off EOF - name: setup if: matrix.network_type == 'ipv4' @@ -199,7 +199,7 @@ jobs: export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql" + docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mysql" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql" - uses: actions/upload-artifact@v1 if: failure() @@ -239,7 +239,7 @@ jobs: if: matrix.connect_type == 'tls' run: | cat <<-EOF >> "$GITHUB_ENV" - EMQX_AUTH__PGSQL__SSL=on + EMQX_AUTH__PGSQL__SSL__ENABLE=on EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem EMQX_AUTH__PGSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem EMQX_AUTH__PGSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem @@ -251,7 +251,7 @@ jobs: PGSQL_TAG: ${{ matrix.pgsql_tag }} if: matrix.connect_type == 'tcp' run: | - echo EMQX_AUTH__PGSQL__SSL=off >> "$GITHUB_ENV" + echo EMQX_AUTH__PGSQL__SSL__ENABLE=off >> "$GITHUB_ENV" - name: setup if: matrix.network_type == 'ipv4' run: | @@ -273,7 +273,7 @@ jobs: CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql" + docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_pgsql" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql" - uses: actions/upload-artifact@v1 if: failure() @@ -318,7 +318,7 @@ jobs: if: matrix.connect_type == 'tls' run: | cat <<-EOF >> "$GITHUB_ENV" - EMQX_AUTH__REDIS__SSL=on + EMQX_AUTH__REDIS__SSL__ENABLE=on EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key @@ -330,7 +330,7 @@ jobs: REDIS_TAG: ${{ matrix.redis_tag }} if: matrix.connect_type == 'tcp' run: | - echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV" + echo EMQX_AUTH__REDIS__SSL__ENABLE=off >> "$GITHUB_ENV" - name: get server address run: | ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis) @@ -394,7 +394,7 @@ jobs: export EMQX_AUTH__REIDS__PASSWORD=public printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis" + docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_redis" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis" - uses: actions/upload-artifact@v1 if: failure() diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml new file mode 100644 index 000000000..3fa30b484 --- /dev/null +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -0,0 +1,26 @@ +name: Check emqx app standalone + +on: + push: + tags: + - v* + - e* + pull_request: + +jobs: + check_all: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + + steps: + - uses: actions/checkout@v2 + - name: run + run: | + make ensure-rebar3 + cp rebar3 apps/emqx/ + cd apps/emqx + ./rebar3 xref + ./rebar3 dialyzer + ./rebar3 eunit -v + ./rebar3 ct -v + ./rebar3 proper -d test/props diff --git a/etc/acl.conf b/apps/emqx/etc/acl.conf similarity index 100% rename from etc/acl.conf rename to apps/emqx/etc/acl.conf diff --git a/etc/acl.conf.paho b/apps/emqx/etc/acl.conf.paho similarity index 100% rename from etc/acl.conf.paho rename to apps/emqx/etc/acl.conf.paho diff --git a/etc/certs/README b/apps/emqx/etc/certs/README similarity index 100% rename from etc/certs/README rename to apps/emqx/etc/certs/README diff --git a/etc/certs/cacert.pem b/apps/emqx/etc/certs/cacert.pem similarity index 100% rename from etc/certs/cacert.pem rename to apps/emqx/etc/certs/cacert.pem diff --git a/etc/certs/cert.pem b/apps/emqx/etc/certs/cert.pem similarity index 100% rename from etc/certs/cert.pem rename to apps/emqx/etc/certs/cert.pem diff --git a/etc/certs/client-cert.pem b/apps/emqx/etc/certs/client-cert.pem similarity index 100% rename from etc/certs/client-cert.pem rename to apps/emqx/etc/certs/client-cert.pem diff --git a/etc/certs/client-key.pem b/apps/emqx/etc/certs/client-key.pem similarity index 100% rename from etc/certs/client-key.pem rename to apps/emqx/etc/certs/client-key.pem diff --git a/etc/certs/key.pem b/apps/emqx/etc/certs/key.pem similarity index 100% rename from etc/certs/key.pem rename to apps/emqx/etc/certs/key.pem diff --git a/etc/emqx.conf b/apps/emqx/etc/emqx.conf similarity index 90% rename from etc/emqx.conf rename to apps/emqx/etc/emqx.conf index 8d3a9c74e..f6627bf1c 100644 --- a/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -56,7 +56,7 @@ cluster.autoclean = 5m ## Node list of the cluster. ## ## Value: String -## cluster.static.seeds = emqx1@127.0.0.1,emqx2@127.0.0.1 +## cluster.static.seeds = "emqx1@127.0.0.1,emqx2@127.0.0.1" ##-------------------------------------------------------------------- ## Cluster using IP Multicast. @@ -64,19 +64,19 @@ cluster.autoclean = 5m ## IP Multicast Address. ## ## Value: IP Address -## cluster.mcast.addr = 239.192.0.1 +## cluster.mcast.addr = "239.192.0.1" ## Multicast Ports. ## ## Value: Port List -## cluster.mcast.ports = 4369,4370 +## cluster.mcast.ports = "4369,4370" ## Multicast Iface. ## ## Value: Iface Address ## -## Default: 0.0.0.0 -## cluster.mcast.iface = 0.0.0.0 +## Default: "0.0.0.0" +## cluster.mcast.iface = "0.0.0.0" ## Multicast Ttl. ## @@ -107,7 +107,14 @@ cluster.autoclean = 5m ## Etcd server list, seperated by ','. ## ## Value: String -## cluster.etcd.server = http://127.0.0.1:2379 +## cluster.etcd.server = "http://127.0.0.1:2379" + +## Etcd api version +## +## Value: Enum +## - v2 +## - v3 +## cluster.etcd.version = v3 ## The prefix helps build nodes path in etcd. Each node in the cluster ## will create a path in etcd: v2/keys/// @@ -125,18 +132,18 @@ cluster.autoclean = 5m ## Path to a file containing the client's private PEM-encoded key. ## ## Value: File -## cluster.etcd.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +## cluster.etcd.ssl.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ## The path to a file containing the client's certificate. ## ## Value: File -## cluster.etcd.ssl.certfile = {{ platform_etc_dir }}/certs/client.pem +## cluster.etcd.ssl.certfile = "{{ platform_etc_dir }}/certs/client.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## cluster.etcd.ssl.cacertfile = {{ platform_etc_dir }}/certs/ca.pem +## cluster.etcd.ssl.cacertfile = "{{ platform_etc_dir }}/certs/ca.pem" ##-------------------------------------------------------------------- ## Cluster using Kubernetes @@ -144,7 +151,7 @@ cluster.autoclean = 5m ## Kubernetes API server list, seperated by ','. ## ## Value: String -## cluster.k8s.apiserver = http://10.110.111.204:8080 +## cluster.k8s.apiserver = "http://10.110.111.204:8080" ## The service name helps lookup EMQ nodes in the cluster. ## @@ -184,17 +191,17 @@ cluster.autoclean = 5m ## Value: @ ## ## Default: emqx@127.0.0.1 -node.name = emqx@127.0.0.1 +node.name = "emqx@127.0.0.1" ## Cookie for distributed node communication. ## ## Value: String -node.cookie = emqxsecretcookie +node.cookie = "emqxsecretcookie" ## Data dir for the node ## ## Value: Folder -node.data_dir = {{ platform_data_dir }} +node.data_dir = "{{ platform_data_dir }}" ## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable ## heartbeat, or set the value as 'on' @@ -271,14 +278,14 @@ node.global_gc_interval = 15m ## Crash dump log file. ## ## Value: Log file -node.crash_dump = {{ platform_log_dir }}/crash.dump +node.crash_dump = "{{ platform_log_dir }}/crash.dump" ## Specify SSL Options in the file if using SSL for Erlang Distribution. ## ## Value: File ## ## vm.args: -ssl_dist_optfile -## node.ssl_dist_optfile = {{ platform_etc_dir }}/ssl_dist.conf +## node.ssl_dist_optfile = "{{ platform_etc_dir }}/ssl_dist.conf" ## Sets the net_kernel tick time. TickTime is specified in seconds. ## Notice that all communicating nodes are to have the same TickTime @@ -419,10 +426,17 @@ log.to = file ## Default: warning log.level = warning +## Timezone offset to display in logs +## Value: +## - "system" use system zone +## - "utc" for Universal Coordinated Time (UTC) +## - "+hh:mm" or "-hh:mm" for a specified offset +log.time_offset = system + ## The dir for log files. ## ## Value: Folder -log.dir = {{ platform_log_dir }} +log.dir = "{{ platform_log_dir }}" ## The log filename for logs of level specified in "log.level". ## @@ -460,7 +474,7 @@ log.file = emqx.log ## ## Value: on | off ## Default: on -log.rotation = on +log.rotation.enable = on ## Maximum size of each log file. ## @@ -579,7 +593,7 @@ log.rotation.count = 5 ## Value: MaxBurstCount,TimeWindow ## Default: disabled ## -#log.burst_limit = 20000, 1s +#log.burst_limit = "20000, 1s" ## CONFIG_SECTION_END=logger =================================================== @@ -591,42 +605,42 @@ log.rotation.count = 5 ## Notice: Disable the option in production deployment! ## ## Value: true | false -allow_anonymous = true +acl.allow_anonymous = true ## Allow or deny if no ACL rules matched. ## ## Value: allow | deny -acl_nomatch = allow +acl.acl_nomatch = allow ## Default ACL File. ## ## Value: File Name -acl_file = {{ platform_etc_dir }}/acl.conf +acl.acl_file = "{{ platform_etc_dir }}/acl.conf" ## Whether to enable ACL cache. ## ## If enabled, ACLs roles for each client will be cached in the memory ## ## Value: on | off -enable_acl_cache = on +acl.enable_acl_cache = on ## The maximum count of ACL entries can be cached for a client. ## ## Value: Integer greater than 0 ## Default: 32 -acl_cache_max_size = 32 +acl.acl_cache_max_size = 32 ## The time after which an ACL cache entry will be deleted ## ## Value: Duration ## Default: 1 minute -acl_cache_ttl = 1m +acl.acl_cache_ttl = 1m ## The action when acl check reject current operation ## ## Value: ignore | disconnect ## Default: ignore -acl_deny_action = ignore +acl.acl_deny_action = ignore ## Specify the global flapping detect policy. ## The value is a string composed of flapping threshold, duration and banned interval. @@ -635,7 +649,7 @@ acl_deny_action = ignore ## 3. banned interval: the banned interval if a flapping is detected. ## ## Value: Integer,Duration,Duration -flapping_detect_policy = 30, 1m, 5m +acl.flapping_detect_policy = "30, 1m, 5m" ##-------------------------------------------------------------------- ## MQTT Protocol @@ -732,7 +746,7 @@ zone.external.acl_deny_action = ignore ## messages | bytes passed through. ## ## Numbers delimited by `|'. Zero or negative is to disable. -zone.external.force_gc_policy = 16000|16MB +zone.external.force_gc_policy = "16000|16MB" ## Max message queue length and total heap size to force shutdown ## connection/session process. @@ -742,9 +756,9 @@ zone.external.force_gc_policy = 16000|16MB ## Numbers delimited by `|'. Zero or negative is to disable. ## ## Default: -## - 10000|64MB on ARCH_64 system -## - 1000|32MB on ARCH_32 sytem -#zone.external.force_shutdown_policy = 10000|64MB +## - "10000|64MB" on ARCH_64 system +## - "1000|32MB" on ARCH_32 sytem +#zone.external.force_shutdown_policy = "10000|64MB" ## Maximum MQTT packet size allowed. ## @@ -850,7 +864,7 @@ zone.external.max_mqueue_len = 1000 ## are treated equal ## ## Priority number [1-255] -## Example: topic/1=10,topic/2=8 +## Example: "topic/1=10,topic/2=8" ## NOTE: comma and equal signs are not allowed for priority topic names ## NOTE: messages for topics not in the priority table are treated as ## either highest or lowest priority depending on the configured @@ -877,13 +891,13 @@ zone.external.enable_flapping_detect = off ## ## Value: Number,Duration ## Example: 100 messages per 10 seconds. -#zone.external.rate_limit.conn_messages_in = 100,10s +#zone.external.rate_limit.conn_messages_in = "100,10s" ## Bytes limit for a external MQTT connections. ## ## Value: Number,Duration ## Example: 100KB incoming per 10 seconds. -#zone.external.rate_limit.conn_bytes_in = 100KB,10s +#zone.external.rate_limit.conn_bytes_in = "100KB,10s" ## Whether to alarm the congested connections. ## @@ -914,16 +928,16 @@ zone.external.enable_flapping_detect = off ## ## Value: Number, Duration ## -## Example: 100 messaegs per 1s -#zone.external.quota.conn_messages_routing = 100,1s +## Example: 100 messages per 1s +#zone.external.quota.conn_messages_routing = "100,1s" ## Messages quota for the all of external MQTT connections. ## This value consumed by the number of recipient on a message. ## ## Value: Number, Duration ## -## Example: 200000 messaegs per 1s -#zone.external.quota.overall_messages_routing = 200000,1s +## Example: 200000 messages per 1s +#zone.external.quota.overall_messages_routing = "200000,1s" ## All the topics will be prefixed with the mountpoint path if this option is enabled. ## @@ -932,7 +946,7 @@ zone.external.enable_flapping_detect = off ## - %u: username ## ## Value: String -## zone.external.mountpoint = devicebound/ +## zone.external.mountpoint = "devicebound/" ## Whether use username replace client id ## @@ -977,7 +991,7 @@ zone.internal.enable_acl = off zone.internal.acl_deny_action = ignore ## See zone.$name.force_gc_policy -## zone.internal.force_gc_policy = 128000|128MB +## zone.internal.force_gc_policy = "128000|128MB" ## See zone.$name.wildcard_subscription. ## @@ -1022,8 +1036,8 @@ zone.internal.enable_flapping_detect = off ## See zone.$name.force_shutdown_policy ## ## Default: -## - 10000|64MB on ARCH_64 system -## - 1000|32MB on ARCH_32 sytem +## - "10000|64MB" on ARCH_64 system +## - "1000|32MB" on ARCH_32 sytem #zone.internal.force_shutdown_policy = 10000|64MB ## All the topics will be prefixed with the mountpoint path if this option is enabled. @@ -1033,7 +1047,7 @@ zone.internal.enable_flapping_detect = off ## - %u: username ## ## Value: String -## zone.internal.mountpoint = cloudbound/ +## zone.internal.mountpoint = "cloudbound/" ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) ## @@ -1067,8 +1081,8 @@ zone.internal.bypass_auth_plugins = true ## ## Value: IP:Port | Port ## -## Examples: 1883, 127.0.0.1:1883, ::1:1883 -listener.tcp.external = 0.0.0.0:1883 +## Examples: 1883, "127.0.0.1:1883", "::1:1883" +listener.tcp.external.endpoint = "0.0.0.0:1883" ## The acceptor pool for external MQTT/TCP listener. ## @@ -1103,8 +1117,8 @@ listener.tcp.external.zone = external ## ## Value: ACL Rule ## -## Example: allow 192.168.0.0/24 -listener.tcp.external.access.1 = allow all +## Example: "allow 192.168.0.0/24" +listener.tcp.external.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed ## behind HAProxy or Nginx. @@ -1207,8 +1221,8 @@ listener.tcp.external.reuseaddr = true ## ## Value: IP:Port, Port ## -## Examples: 11883, 127.0.0.1:11883, ::1:11883 -listener.tcp.internal = 127.0.0.1:11883 +## Examples: 11883, "127.0.0.1:11883", "::1:11883" +listener.tcp.internal.endpoint = "127.0.0.1:11883" ## The acceptor pool for internal MQTT/TCP listener. ## @@ -1304,8 +1318,8 @@ listener.tcp.internal.reuseaddr = true ## ## Value: IP:Port | Port ## -## Examples: 8883, 127.0.0.1:8883, ::1:8883 -listener.ssl.external = 8883 +## Examples: 8883, "127.0.0.1:8883", "::1:8883" +listener.ssl.external.endpoint = 8883 ## The acceptor pool for external MQTT/SSL listener. ## @@ -1337,7 +1351,7 @@ listener.ssl.external.zone = external ## See: listener.tcp.$name.access ## ## Value: ACL Rule -listener.ssl.external.access.1 = allow all +listener.ssl.external.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. @@ -1360,7 +1374,7 @@ listener.ssl.external.access.1 = allow all ## ## Value: String, seperated by ',' ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## listener.ssl.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +## listener.ssl.external.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## TLS Handshake timeout. ## @@ -1384,20 +1398,20 @@ listener.ssl.external.handshake_timeout = 15s ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -listener.ssl.external.keyfile = {{ platform_etc_dir }}/certs/key.pem +listener.ssl.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem +listener.ssl.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## listener.ssl.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit @@ -1414,7 +1428,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## openssl dhparam -out dh-params.pem 2048 ## ## Value: File -## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -1449,14 +1463,13 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA - +listener.ssl.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#listener.ssl.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#listener.ssl.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## SSL parameter renegotiation is a feature that allows a client and a server ## to renegotiate the parameters of the SSL connection on the fly. @@ -1566,13 +1579,13 @@ listener.ssl.external.reuseaddr = true ## ## Value: IP:Port | Port ## -## Examples: 8083, 127.0.0.1:8083, ::1:8083 -listener.ws.external = 8083 +## Examples: 8083, "127.0.0.1:8083", "::1:8083" +listener.ws.external.endpoint = 8083 ## The path of WebSocket MQTT endpoint ## ## Value: URL Path -listener.ws.external.mqtt_path = /mqtt +listener.ws.external.mqtt_path = "/mqtt" ## The acceptor pool for external MQTT/WebSocket listener. ## @@ -1604,7 +1617,7 @@ listener.ws.external.zone = external ## See: listener.ws.$name.access ## ## Value: ACL Rule -listener.ws.external.access.1 = allow all +listener.ws.external.access.1 = "allow all" ## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. ## Set to false for WeChat MiniApp. @@ -1615,7 +1628,7 @@ listener.ws.external.access.1 = allow all ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.ws.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.ws.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" ## Specify which HTTP header for real source IP if the EMQ X cluster is ## deployed behind NGINX or HAProxy. @@ -1823,7 +1836,7 @@ listener.ws.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for websocket connection ## ## Value: http://url eg. local http dashboard url - http://localhost:18083, http://127.0.0.1:18083 -listener.ws.external.check_origins = http://localhost:18083, http://127.0.0.1:18083 +listener.ws.external.check_origins = "http://localhost:18083, http://127.0.0.1:18083" ##-------------------------------------------------------------------- ## External WebSocket/SSL listener for MQTT Protocol @@ -1833,13 +1846,13 @@ listener.ws.external.check_origins = http://localhost:18083, http://127.0.0.1:18 ## ## Value: IP:Port | Port ## -## Examples: 8084, 127.0.0.1:8084, ::1:8084 -listener.wss.external = 8084 +## Examples: 8084, "127.0.0.1:8084", "::1:8084" +listener.wss.external.endpoint = 8084 ## The path of WebSocket MQTT endpoint ## ## Value: URL Path -listener.wss.external.mqtt_path = /mqtt +listener.wss.external.mqtt_path = "/mqtt" ## The acceptor pool for external MQTT/WebSocket/SSL listener. ## @@ -1873,7 +1886,7 @@ listener.wss.external.zone = external ## See: listener.tcp.$name.access. ## ## Value: ACL Rule -listener.wss.external.access.1 = allow all +listener.wss.external.access.1 = "allow all" ## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. ## Set to false for WeChat MiniApp. @@ -1884,7 +1897,7 @@ listener.wss.external.access.1 = allow all ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.wss.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.wss.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" ## Specify which HTTP header for real source IP if the EMQ X cluster is ## deployed behind NGINX or HAProxy. @@ -1918,28 +1931,28 @@ listener.wss.external.access.1 = allow all ## ## Value: String, seperated by ',' ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## listener.wss.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +## listener.wss.external.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## Path to the file containing the user's private PEM-encoded key. ## ## See: listener.ssl.$name.keyfile ## ## Value: File -listener.wss.external.keyfile = {{ platform_etc_dir }}/certs/key.pem +listener.wss.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: listener.ssl.$name.certfile ## ## Value: File -listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem +listener.wss.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## See: listener.ssl.$name.cacert ## ## Value: File -## listener.wss.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## listener.wss.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Maximum number of non-self-issued intermediate certificates that ## can follow the peer certificate in a valid certification path. @@ -1960,7 +1973,7 @@ listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## See: listener.ssl.$name.dhfile ## ## Value: File -## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## See: listener.ssl.$name.verify ## @@ -1975,13 +1988,13 @@ listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## See: listener.ssl.$name.ciphers ## ## Value: Ciphers -listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +listener.wss.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.wss.external.ciphers' and 'listener.wss.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## listener.wss.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +## listener.wss.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## See: listener.ssl.$name.secure_renegotiate ## @@ -2140,7 +2153,7 @@ listener.wss.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for secure websocket connection ## ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 +listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" ## CONFIG_SECTION_END=listeners ================================================ @@ -2149,7 +2162,7 @@ listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1: ## The file to store loaded module names. ## ## Value: File -modules.loaded_file = {{ platform_data_dir }}/loaded_modules +module.loaded_file = "{{ platform_data_dir }}/loaded_modules" ##-------------------------------------------------------------------- ## Presence Module @@ -2165,7 +2178,7 @@ module.presence.qos = 1 ## Subscribe the Topics automatically when client connected. ## ## Value: String -## module.subscription.1.topic = connected/%c/%u +## module.subscription.1.topic = "connected/%c/%u" ## Qos of the proxy subscription. ## @@ -2198,8 +2211,8 @@ module.presence.qos = 1 ## Rewrite Module ## {rewrite, Topic, Re, Dest} -## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 -## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +## module.rewrite.pub_rule.1 = "x/# ^x/y/(.+)$ z/y/$1" +## module.rewrite.sub_rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" ## CONFIG_SECTION_END=modules ================================================== @@ -2210,17 +2223,17 @@ module.presence.qos = 1 ## The etc dir for plugins' config. ## ## Value: Folder -plugins.etc_dir = {{ platform_etc_dir }}/plugins/ +plugins.etc_dir = "{{ platform_etc_dir }}/plugins/" ## The file to store loaded plugin names. ## ## Value: File -plugins.loaded_file = {{ platform_data_dir }}/loaded_plugins +plugins.loaded_file = "{{ platform_data_dir }}/loaded_plugins" ## The directory of extension plugins. ## ## Value: File -plugins.expand_plugins_dir = {{ platform_plugins_dir }}/ +plugins.expand_plugins_dir = "{{ platform_plugins_dir }}/" ##-------------------------------------------------------------------- ## Broker @@ -2334,7 +2347,6 @@ sysmon.long_gc = 0 ## Examples: ## - 2h: 2 hours ## - 30m: 30 minutes -## - 0.1s: 0.1 seconds ## - 100ms: 100 milliseconds ## ## Default: 0ms @@ -2426,8 +2438,8 @@ vm_mon.process_low_watermark = 60% ## - log ## - publish ## -## Default: log,publish -alarm.actions = log,publish +## Default: "log,publish" +alarm.actions = "log,publish" ## The maximum number of deactivated alarms ## diff --git a/etc/emqx_cloud/vm.args b/apps/emqx/etc/emqx_cloud/vm.args similarity index 100% rename from etc/emqx_cloud/vm.args rename to apps/emqx/etc/emqx_cloud/vm.args diff --git a/etc/emqx_edge/vm.args b/apps/emqx/etc/emqx_edge/vm.args similarity index 100% rename from etc/emqx_edge/vm.args rename to apps/emqx/etc/emqx_edge/vm.args diff --git a/etc/ssl_dist.conf b/apps/emqx/etc/ssl_dist.conf similarity index 100% rename from etc/ssl_dist.conf rename to apps/emqx/etc/ssl_dist.conf diff --git a/include/emqx.hrl b/apps/emqx/include/emqx.hrl similarity index 100% rename from include/emqx.hrl rename to apps/emqx/include/emqx.hrl diff --git a/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl similarity index 100% rename from include/emqx_mqtt.hrl rename to apps/emqx/include/emqx_mqtt.hrl diff --git a/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl similarity index 96% rename from include/emqx_release.hrl rename to apps/emqx/include/emqx_release.hrl index ebd8e9558..e69b07558 100644 --- a/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.3"}). +-define(EMQX_RELEASE, {opensource, "5.0-pre"}). -else. diff --git a/include/logger.hrl b/apps/emqx/include/logger.hrl similarity index 100% rename from include/logger.hrl rename to apps/emqx/include/logger.hrl diff --git a/include/types.hrl b/apps/emqx/include/types.hrl similarity index 100% rename from include/types.hrl rename to apps/emqx/include/types.hrl diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config new file mode 100644 index 000000000..2c566d7f8 --- /dev/null +++ b/apps/emqx/rebar.config @@ -0,0 +1,47 @@ +{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, + warn_obsolete_guard,compressed]}. + +{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, + deprecated_function_calls,warnings_as_errors,deprecated_functions]}. + +%% Deps here may duplicate with emqx.git root level rebar.config +%% but there not be any descrpancy. +%% This rebar.config is necessary because the app may be used as a +%% `git_subdir` dependency in other projects. +{deps, + [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.9.0"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.0"}}} + , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} + , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} + ]}. + +{plugins, [rebar3_proper]}. +{extra_src_dirs, [{"etc", [recursive]}]}. +{profiles, [ + {test, + [{deps, + [ meck + , {bbmustache,"1.10.0"} + , {emqx_ct_helpers, {git,"https://github.com/zmstone/emqx-ct-helpers", {branch,"hocon"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} + ]}, + {extra_src_dirs, [{"test",[recursive]}]} + ]} +]}. + +{dialyzer, [ + {warnings, [unmatched_returns, error_handling, race_conditions]}, + {plt_location, "."}, + {plt_prefix, "emqx_dialyzer"}, + {plt_apps, all_apps}, + {plt_extra_apps, [hocon]}, + {statistics, true} + ] +}. diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script new file mode 100644 index 000000000..eae18f106 --- /dev/null +++ b/apps/emqx/rebar.config.script @@ -0,0 +1,11 @@ +Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}, +AddBcrypt = fun(C) -> + {deps, Deps0} = lists:keyfind(deps, 1, C), + Deps = [Bcrypt | Deps0], + lists:keystore(deps, 1, C, {deps, Deps}) +end, + +case os:type() of + {win32, _} -> CONFIG; + _ -> AddBcrypt(CONFIG) +end. diff --git a/src/emqx.app.src b/apps/emqx/src/emqx.app.src similarity index 89% rename from src/emqx.app.src rename to apps/emqx/src/emqx.app.src index 884c511df..e909702ae 100644 --- a/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src similarity index 100% rename from src/emqx.appup.src rename to apps/emqx/src/emqx.appup.src diff --git a/src/emqx.erl b/apps/emqx/src/emqx.erl similarity index 100% rename from src/emqx.erl rename to apps/emqx/src/emqx.erl diff --git a/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl similarity index 100% rename from src/emqx_access_control.erl rename to apps/emqx/src/emqx_access_control.erl diff --git a/src/emqx_access_rule.erl b/apps/emqx/src/emqx_access_rule.erl similarity index 100% rename from src/emqx_access_rule.erl rename to apps/emqx/src/emqx_access_rule.erl diff --git a/src/emqx_acl_cache.erl b/apps/emqx/src/emqx_acl_cache.erl similarity index 100% rename from src/emqx_acl_cache.erl rename to apps/emqx/src/emqx_acl_cache.erl diff --git a/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl similarity index 100% rename from src/emqx_alarm.erl rename to apps/emqx/src/emqx_alarm.erl diff --git a/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl similarity index 100% rename from src/emqx_alarm_handler.erl rename to apps/emqx/src/emqx_alarm_handler.erl diff --git a/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl similarity index 100% rename from src/emqx_app.erl rename to apps/emqx/src/emqx_app.erl diff --git a/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl similarity index 100% rename from src/emqx_banned.erl rename to apps/emqx/src/emqx_banned.erl diff --git a/src/emqx_base62.erl b/apps/emqx/src/emqx_base62.erl similarity index 100% rename from src/emqx_base62.erl rename to apps/emqx/src/emqx_base62.erl diff --git a/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl similarity index 100% rename from src/emqx_batch.erl rename to apps/emqx/src/emqx_batch.erl diff --git a/src/emqx_boot.erl b/apps/emqx/src/emqx_boot.erl similarity index 100% rename from src/emqx_boot.erl rename to apps/emqx/src/emqx_boot.erl diff --git a/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl similarity index 100% rename from src/emqx_broker.erl rename to apps/emqx/src/emqx_broker.erl diff --git a/src/emqx_broker_bench.erl b/apps/emqx/src/emqx_broker_bench.erl similarity index 100% rename from src/emqx_broker_bench.erl rename to apps/emqx/src/emqx_broker_bench.erl diff --git a/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl similarity index 100% rename from src/emqx_broker_helper.erl rename to apps/emqx/src/emqx_broker_helper.erl diff --git a/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl similarity index 100% rename from src/emqx_broker_sup.erl rename to apps/emqx/src/emqx_broker_sup.erl diff --git a/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl similarity index 100% rename from src/emqx_channel.erl rename to apps/emqx/src/emqx_channel.erl diff --git a/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl similarity index 100% rename from src/emqx_cm.erl rename to apps/emqx/src/emqx_cm.erl diff --git a/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl similarity index 100% rename from src/emqx_cm_locker.erl rename to apps/emqx/src/emqx_cm_locker.erl diff --git a/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl similarity index 100% rename from src/emqx_cm_registry.erl rename to apps/emqx/src/emqx_cm_registry.erl diff --git a/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl similarity index 100% rename from src/emqx_cm_sup.erl rename to apps/emqx/src/emqx_cm_sup.erl diff --git a/src/emqx_congestion.erl b/apps/emqx/src/emqx_congestion.erl similarity index 100% rename from src/emqx_congestion.erl rename to apps/emqx/src/emqx_congestion.erl diff --git a/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl similarity index 100% rename from src/emqx_connection.erl rename to apps/emqx/src/emqx_connection.erl diff --git a/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl similarity index 100% rename from src/emqx_ctl.erl rename to apps/emqx/src/emqx_ctl.erl diff --git a/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl similarity index 100% rename from src/emqx_flapping.erl rename to apps/emqx/src/emqx_flapping.erl diff --git a/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl similarity index 100% rename from src/emqx_frame.erl rename to apps/emqx/src/emqx_frame.erl diff --git a/src/emqx_gc.erl b/apps/emqx/src/emqx_gc.erl similarity index 100% rename from src/emqx_gc.erl rename to apps/emqx/src/emqx_gc.erl diff --git a/src/emqx_gen_mod.erl b/apps/emqx/src/emqx_gen_mod.erl similarity index 100% rename from src/emqx_gen_mod.erl rename to apps/emqx/src/emqx_gen_mod.erl diff --git a/src/emqx_global_gc.erl b/apps/emqx/src/emqx_global_gc.erl similarity index 100% rename from src/emqx_global_gc.erl rename to apps/emqx/src/emqx_global_gc.erl diff --git a/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl similarity index 100% rename from src/emqx_guid.erl rename to apps/emqx/src/emqx_guid.erl diff --git a/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl similarity index 100% rename from src/emqx_hooks.erl rename to apps/emqx/src/emqx_hooks.erl diff --git a/src/emqx_inflight.erl b/apps/emqx/src/emqx_inflight.erl similarity index 100% rename from src/emqx_inflight.erl rename to apps/emqx/src/emqx_inflight.erl diff --git a/src/emqx_json.erl b/apps/emqx/src/emqx_json.erl similarity index 100% rename from src/emqx_json.erl rename to apps/emqx/src/emqx_json.erl diff --git a/src/emqx_keepalive.erl b/apps/emqx/src/emqx_keepalive.erl similarity index 100% rename from src/emqx_keepalive.erl rename to apps/emqx/src/emqx_keepalive.erl diff --git a/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl similarity index 100% rename from src/emqx_kernel_sup.erl rename to apps/emqx/src/emqx_kernel_sup.erl diff --git a/src/emqx_limiter.erl b/apps/emqx/src/emqx_limiter.erl similarity index 100% rename from src/emqx_limiter.erl rename to apps/emqx/src/emqx_limiter.erl diff --git a/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl similarity index 100% rename from src/emqx_listeners.erl rename to apps/emqx/src/emqx_listeners.erl diff --git a/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl similarity index 100% rename from src/emqx_logger.erl rename to apps/emqx/src/emqx_logger.erl diff --git a/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl similarity index 99% rename from src/emqx_logger_jsonfmt.erl rename to apps/emqx/src/emqx_logger_jsonfmt.erl index 31e4c2fda..fdc1a8bb3 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -32,6 +32,9 @@ -export([format/2]). -ifdef(TEST). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -export([report_cb_1/1, report_cb_2/2, report_cb_crash/2]). -endif. @@ -220,8 +223,6 @@ json_key(Term) -> end. -ifdef(TEST). --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). no_crash_test_() -> Opts = [{numtests, 1000}, {to_file, user}], diff --git a/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl similarity index 100% rename from src/emqx_logger_textfmt.erl rename to apps/emqx/src/emqx_logger_textfmt.erl diff --git a/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl similarity index 100% rename from src/emqx_message.erl rename to apps/emqx/src/emqx_message.erl diff --git a/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl similarity index 100% rename from src/emqx_metrics.erl rename to apps/emqx/src/emqx_metrics.erl diff --git a/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl similarity index 100% rename from src/emqx_misc.erl rename to apps/emqx/src/emqx_misc.erl diff --git a/src/emqx_mountpoint.erl b/apps/emqx/src/emqx_mountpoint.erl similarity index 100% rename from src/emqx_mountpoint.erl rename to apps/emqx/src/emqx_mountpoint.erl diff --git a/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl similarity index 100% rename from src/emqx_mqtt_caps.erl rename to apps/emqx/src/emqx_mqtt_caps.erl diff --git a/src/emqx_mqtt_props.erl b/apps/emqx/src/emqx_mqtt_props.erl similarity index 100% rename from src/emqx_mqtt_props.erl rename to apps/emqx/src/emqx_mqtt_props.erl diff --git a/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl similarity index 100% rename from src/emqx_mqueue.erl rename to apps/emqx/src/emqx_mqueue.erl diff --git a/src/emqx_node_dump.erl b/apps/emqx/src/emqx_node_dump.erl similarity index 100% rename from src/emqx_node_dump.erl rename to apps/emqx/src/emqx_node_dump.erl diff --git a/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl similarity index 100% rename from src/emqx_os_mon.erl rename to apps/emqx/src/emqx_os_mon.erl diff --git a/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl similarity index 100% rename from src/emqx_packet.erl rename to apps/emqx/src/emqx_packet.erl diff --git a/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl similarity index 100% rename from src/emqx_passwd.erl rename to apps/emqx/src/emqx_passwd.erl diff --git a/src/emqx_pd.erl b/apps/emqx/src/emqx_pd.erl similarity index 100% rename from src/emqx_pd.erl rename to apps/emqx/src/emqx_pd.erl diff --git a/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl similarity index 100% rename from src/emqx_plugins.erl rename to apps/emqx/src/emqx_plugins.erl diff --git a/src/emqx_pmon.erl b/apps/emqx/src/emqx_pmon.erl similarity index 100% rename from src/emqx_pmon.erl rename to apps/emqx/src/emqx_pmon.erl diff --git a/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl similarity index 100% rename from src/emqx_pool.erl rename to apps/emqx/src/emqx_pool.erl diff --git a/src/emqx_pool_sup.erl b/apps/emqx/src/emqx_pool_sup.erl similarity index 100% rename from src/emqx_pool_sup.erl rename to apps/emqx/src/emqx_pool_sup.erl diff --git a/src/emqx_pqueue.erl b/apps/emqx/src/emqx_pqueue.erl similarity index 100% rename from src/emqx_pqueue.erl rename to apps/emqx/src/emqx_pqueue.erl diff --git a/src/emqx_psk.erl b/apps/emqx/src/emqx_psk.erl similarity index 100% rename from src/emqx_psk.erl rename to apps/emqx/src/emqx_psk.erl diff --git a/src/emqx_reason_codes.erl b/apps/emqx/src/emqx_reason_codes.erl similarity index 100% rename from src/emqx_reason_codes.erl rename to apps/emqx/src/emqx_reason_codes.erl diff --git a/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl similarity index 100% rename from src/emqx_router.erl rename to apps/emqx/src/emqx_router.erl diff --git a/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl similarity index 100% rename from src/emqx_router_helper.erl rename to apps/emqx/src/emqx_router_helper.erl diff --git a/src/emqx_router_sup.erl b/apps/emqx/src/emqx_router_sup.erl similarity index 100% rename from src/emqx_router_sup.erl rename to apps/emqx/src/emqx_router_sup.erl diff --git a/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl similarity index 100% rename from src/emqx_rpc.erl rename to apps/emqx/src/emqx_rpc.erl diff --git a/src/emqx_rule_actions_trans.erl b/apps/emqx/src/emqx_rule_actions_trans.erl similarity index 100% rename from src/emqx_rule_actions_trans.erl rename to apps/emqx/src/emqx_rule_actions_trans.erl diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl new file mode 100644 index 000000000..bac2e5cfe --- /dev/null +++ b/apps/emqx/src/emqx_schema.erl @@ -0,0 +1,1230 @@ +-module(emqx_schema). + +% tmp +-dialyzer(no_return). +-dialyzer(no_match). +-dialyzer(no_contracts). +-dialyzer(no_unused). +-dialyzer(no_fail_call). + +-include_lib("typerefl/include/types.hrl"). + +-type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. +-type flag() :: true | false. +-type duration() :: integer(). +-type duration_s() :: integer(). +-type bytesize() :: integer(). +-type percent() :: float(). +-type file() :: string(). +-type comma_separated_list() :: list(). +-type bar_separated_list() :: list(). +-type ip_port() :: tuple(). + +-typerefl_from_string({flag/0, emqx_schema, to_flag}). +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). +-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({percent/0, emqx_schema, to_percent}). +-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). +-typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +% workaround: prevent being recognized as unused functions +-export([to_duration/1, to_duration_s/1, to_bytesize/1, + to_flag/1, to_percent/1, to_comma_separated_list/1, + to_bar_separated_list/1, to_ip_port/1]). + +-behaviour(hocon_schema). + +-reflect_type([ log_level/0, flag/0, duration/0, duration_s/0, + bytesize/0, percent/0, file/0, + comma_separated_list/0, bar_separated_list/0, ip_port/0]). + +-export([structs/0, fields/1, translations/0, translation/1]). +-export([t/1, t/3, t/4, ref/1]). +-export([conf_get/2, conf_get/3, keys/2, filter/1]). +-export([ssl/2, tr_ssl/2, tr_password_hash/2]). + +structs() -> ["cluster", "node", "rpc", "log", "lager", + "acl", "mqtt", "zone", "listener", "module", "broker", + "plugins", "sysmon", "os_mon", "vm_mon", "alarm", "telemetry"]. + +fields("cluster") -> + [ {"name", t(atom(), "ekka.cluster_name", emqxcl)} + , {"discovery", t(atom(), undefined, manual)} + , {"autoclean", t(duration(), "ekka.cluster_autoclean", undefined)} + , {"autoheal", t(flag(), "ekka.cluster_autoheal", false)} + , {"static", ref("static")} + , {"mcast", ref("mcast")} + , {"proto_dist", t(union([inet_tcp, inet6_tcp, inet_tls]), "ekka.proto_dist", inet_tcp)} + , {"dns", ref("dns")} + , {"etcd", ref("etcd")} + , {"k8s", ref("k8s")} + ]; + +fields("static") -> + [ {"seeds", t(comma_separated_list())}]; + +fields("mcast") -> + [ {"addr", t(string(), undefined, "239.192.0.1")} + , {"ports", t(comma_separated_list(), undefined, "4369")} + , {"iface", t(string(), undefined, "0.0.0.0")} + , {"ttl", t(integer(), undefined, 255)} + , {"loop", t(flag(), undefined, true)} + , {"sndbuf", t(bytesize(), undefined, "16KB")} + , {"recbuf", t(bytesize(), undefined, "16KB")} + , {"buffer", t(bytesize(), undefined, "32KB")} + ]; + +fields("dns") -> + [ {"app", t(string())}]; + +fields("etcd") -> + [ {"server", t(comma_separated_list())} + , {"prefix", t(string())} + , {"node_ttl", t(duration(), undefined, "1m")} + , {"ssl", ref("etcd_ssl")} + ]; + +fields("etcd_ssl") -> + ssl(undefined, #{}); + +fields("k8s") -> + [ {"apiserver", t(string())} + , {"service_name", t(string())} + , {"address_type", t(union([ip, dns, hostname]))} + , {"app_name", t(string())} + , {"namespace", t(string())} + , {"suffix", t(string(), undefined, "")} + ]; + +fields("node") -> + [ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "NODE_NAME")} + , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)} + , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "NODE_COOKIE")} + , {"data_dir", t(string(), "emqx.data_dir", undefined)} + , {"heartbeat", t(flag(), undefined, false)} + , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)} + , {"process_limit", t(integer(), "vm_args.+P", undefined)} + , {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "MAX_PORTS")} + , {"dist_buffer_size", fun node__dist_buffer_size/1} + , {"global_gc_interval", t(duration_s(), "emqx.global_gc_interval", undefined)} + , {"fullsweep_after", t(non_neg_integer(), + "vm_args.-env ERL_FULLSWEEP_AFTER", 1000)} + , {"max_ets_tables", t(integer(), "vm_args.+e", 256000)} + , {"crash_dump", t(file(), "vm_args.-env ERL_CRASH_DUMP", undefined)} + , {"dist_net_ticktime", t(integer(), "vm_args.-kernel net_ticktime", undefined)} + , {"dist_listen_min", t(integer(), "kernel.inet_dist_listen_min", undefined)} + , {"dist_listen_max", t(integer(), "kernel.inet_dist_listen_max", undefined)} + , {"backtrace_depth", t(integer(), "emqx.backtrace_depth", 16)} + ]; + +fields("rpc") -> + [ {"mode", t(union(sync, async), "emqx.rpc_mode", async)} + , {"async_batch_size", t(integer(), "gen_rpc.max_batch_size", 256)} + , {"port_discovery",t(union(manual, stateless), "gen_rpc.port_discovery", stateless)} + , {"tcp_server_port", t(integer(), "gen_rpc.tcp_server_port", 5369)} + , {"tcp_client_num", t(range(0, 255), undefined, 0)} + , {"connect_timeout", t(duration(), "gen_rpc.connect_timeout", "5s")} + , {"send_timeout", t(duration(), "gen_rpc.send_timeout", "5s")} + , {"authentication_timeout", t(duration(), "gen_rpc.authentication_timeout", "5s")} + , {"call_receive_timeout", t(duration(), "gen_rpc.call_receive_timeout", "15s")} + , {"socket_keepalive_idle", t(duration_s(), "gen_rpc.socket_keepalive_idle", "7200s")} + , {"socket_keepalive_interval", t(duration_s(), "gen_rpc.socket_keepalive_interval", "75s")} + , {"socket_keepalive_count", t(integer(), "gen_rpc.socket_keepalive_count", 9)} + , {"socket_sndbuf", t(bytesize(), "gen_rpc.socket_sndbuf", "1MB")} + , {"socket_recbuf", t(bytesize(), "gen_rpc.socket_recbuf", "1MB")} + , {"socket_buffer", t(bytesize(), "gen_rpc.socket_buffer", "1MB")} + ]; + +fields("log") -> + [ {"to", t(union([file, console, both]), undefined, file)} + , {"level", t(log_level(), undefined, warning)} + , {"time_offset", t(string(), undefined, "system")} + , {"primary_log_level", t(log_level(), undefined, warning)} + , {"dir", t(string(), undefined, "log")} + , {"file", t(file(), undefined, "emqx.log")} + , {"chars_limit", t(integer(), undefined, -1)} + , {"supervisor_reports", t(union([error, progress]), undefined, error)} + , {"max_depth", t(union([infinity, integer()]), + "kernel.error_logger_format_depth", 20)} + , {"formatter", t(union([text, json]), undefined, text)} + , {"single_line", t(boolean(), undefined, true)} + , {"rotation", ref("rotation")} + , {"size", t(union(bytesize(), infinity), undefined, infinity)} + , {"sync_mode_qlen", t(integer(), undefined, 100)} + , {"drop_mode_qlen", t(integer(), undefined, 3000)} + , {"flush_qlen", t(integer(), undefined, 8000)} + , {"overload_kill", t(flag(), undefined, true)} + , {"overload_kill_mem_size", t(bytesize(), undefined, "30MB")} + , {"overload_kill_qlen", t(integer(), undefined, 20000)} + , {"overload_kill_restart_after", t(union(duration(), infinity), undefined, "5s")} + , {"burst_limit", t(comma_separated_list(), undefined, "disabled")} + , {"error_logger", t(atom(), "kernel.error_logger", silent)} + , {"debug", ref("additional_log_file")} + , {"info", ref("additional_log_file")} + , {"notice", ref("additional_log_file")} + , {"warning", ref("additional_log_file")} + , {"error", ref("additional_log_file")} + , {"critical", ref("additional_log_file")} + , {"alert", ref("additional_log_file")} + , {"emergency", ref("additional_log_file")} + ]; + +fields("additional_log_file") -> + [ {"file", t(string())}]; + +fields("rotation") -> + [ {"enable", t(flag(), undefined, true)} + , {"size", t(bytesize(), undefined, "10MB")} + , {"count", t(integer(), undefined, 5)} + ]; + +fields("lager") -> + [ {"handlers", t(string(), "lager.handlers", "")} + , {"crash_log", t(flag(), "lager.crash_log", false)} + ]; + +fields("acl") -> + [ {"allow_anonymous", t(boolean(), "emqx.allow_anonymous", false)} + , {"acl_nomatch", t(union(allow, deny), "emqx.acl_nomatch", deny)} + , {"acl_file", t(string(), "emqx.acl_file", undefined)} + , {"enable_acl_cache", t(flag(), "emqx.enable_acl_cache", true)} + , {"acl_cache_ttl", t(duration(), "emqx.acl_cache_ttl", "1m")} + , {"acl_cache_max_size", t(range(1, inf), "emqx.acl_cache_max_size", 32)} + , {"acl_deny_action", t(union(ignore, disconnect), "emqx.acl_deny_action", ignore)} + , {"flapping_detect_policy", t(comma_separated_list(), undefined, "30,1m,5m")} + ]; + +fields("mqtt") -> + [ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "MAX_PACKET_SIZE")} + , {"max_clientid_len", t(integer(), "emqx.max_clientid_len", 65535)} + , {"max_topic_levels", t(integer(), "emqx.max_topic_levels", 0)} + , {"max_qos_allowed", t(range(0, 2), "emqx.max_qos_allowed", 2)} + , {"max_topic_alias", t(integer(), "emqx.max_topic_alias", 65535)} + , {"retain_available", t(boolean(), "emqx.retain_available", true)} + , {"wildcard_subscription", t(boolean(), "emqx.wildcard_subscription", true)} + , {"shared_subscription", t(boolean(), "emqx.shared_subscription", true)} + , {"ignore_loop_deliver", t(boolean(), "emqx.ignore_loop_deliver", true)} + , {"strict_mode", t(boolean(), "emqx.strict_mode", false)} + , {"response_information", t(string(), "emqx.response_information", undefined)} + ]; + +fields("zone") -> + [ {"$name", ref("zone_settings")}]; + +fields("zone_settings") -> + [ {"idle_timeout", t(duration(), undefined, "15s")} + , {"allow_anonymous", t(boolean())} + , {"acl_nomatch", t(union(allow, deny))} + , {"enable_acl", t(flag(), undefined, false)} + , {"acl_deny_action", t(union(ignore, disconnect), undefined, ignore)} + , {"enable_ban", t(flag(), undefined, false)} + , {"enable_stats", t(flag(), undefined, false)} + , {"max_packet_size", t(bytesize())} + , {"max_clientid_len", t(integer())} + , {"max_topic_levels", t(integer())} + , {"max_qos_allowed", t(range(0, 2))} + , {"max_topic_alias", t(integer())} + , {"retain_available", t(boolean())} + , {"wildcard_subscription", t(boolean())} + , {"shared_subscription", t(boolean())} + , {"server_keepalive", t(integer())} + , {"keepalive_backoff", t(float(), undefined, 0.75)} + , {"max_subscriptions", t(integer(), undefined, 0)} + , {"upgrade_qos", t(flag(), undefined, false)} + , {"max_inflight", t(range(0, 65535))} + , {"retry_interval", t(duration_s(), undefined, "30s")} + , {"max_awaiting_rel", t(duration(), undefined, 0)} + , {"await_rel_timeout", t(duration_s(), undefined, "300s")} + , {"ignore_loop_deliver", t(boolean())} + , {"session_expiry_interval", t(duration_s(), undefined, "2h")} + , {"max_mqueue_len", t(integer(), undefined, 1000)} + , {"mqueue_priorities", t(comma_separated_list(), undefined, "none")} + , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} + , {"mqueue_store_qos0", t(boolean(), undefined, true)} + , {"enable_flapping_detect", t(flag(), undefined, false)} + , {"rate_limit", ref("rate_limit")} + , {"conn_congestion", ref("conn_congestion")} + , {"quota", ref("quota")} + , {"force_gc_policy", t(bar_separated_list())} + , {"force_shutdown_policy", t(bar_separated_list(), undefined, "default")} + , {"mountpoint", t(string())} + , {"use_username_as_clientid", t(boolean(), undefined, false)} + , {"strict_mode", t(boolean(), undefined, false)} + , {"response_information", t(string())} + , {"bypass_auth_plugins", t(boolean(), undefined, false)} + ]; + +fields("rate_limit") -> + [ {"conn_messages_in", t(comma_separated_list())} + , {"conn_bytes_in", t(comma_separated_list())} + ]; + +fields("conn_congestion") -> + [ {"alarm", t(flag(), undefined, false)} + , {"min_alarm_sustain_duration", t(duration(), undefined, "1m")} + ]; + +fields("quota") -> + [ {"conn_messages_routing", t(comma_separated_list())} + , {"overall_messages_routing", t(comma_separated_list())} + ]; + +fields("listener") -> + [ {"tcp", ref("tcp_listener")} + , {"ssl", ref("ssl_listener")} + , {"ws", ref("ws_listener")} + , {"wss", ref("wss_listener")} + ]; + +fields("tcp_listener") -> + [ {"$name", ref("tcp_listener_settings")}]; + +fields("ssl_listener") -> + [ {"$name", ref("ssl_listener_settings")}]; + +fields("ws_listener") -> + [ {"$name", ref("ws_listener_settings")}]; + +fields("wss_listener") -> + [ {"$name", ref("wss_listener_settings")}]; + +fields("listener_settings") -> + [ {"endpoint", t(union(ip_port(), integer()))} + , {"acceptors", t(integer(), undefined, 8)} + , {"max_connections", t(integer(), undefined, 1024)} + , {"max_conn_rate", t(integer())} + , {"active_n", t(integer(), undefined, 100)} + , {"zone", t(string())} + , {"rate_limit", t(comma_separated_list())} + , {"access", ref("access")} + , {"proxy_protocol", t(flag())} + , {"proxy_protocol_timeout", t(duration())} + , {"backlog", t(integer(), undefined, 1024)} + , {"send_timeout", t(duration(), undefined, "15s")} + , {"send_timeout_close", t(flag(), undefined, true)} + , {"recbuf", t(bytesize())} + , {"sndbuf", t(bytesize())} + , {"buffer", t(bytesize())} + , {"high_watermark", t(bytesize(), undefined, "1MB")} + , {"tune_buffer", t(flag())} + , {"nodelay", t(boolean())} + , {"reuseaddr", t(boolean())} + ]; + +fields("tcp_listener_settings") -> + [ {"peer_cert_as_username", t(cn)} + , {"peer_cert_as_clientid", t(cn)} + ] ++ fields("listener_settings"); + +fields("ssl_listener_settings") -> + [ {"peer_cert_as_username", t(union([cn, dn, crt, pem, md5]))} + , {"peer_cert_as_clientid", t(union([cn, dn, crt, pem, md5]))} + ] ++ + ssl(undefined, #{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true}) ++ fields("listener_settings"); + +fields("ws_listener_settings") -> + [ {"mqtt_path", t(string(), undefined, "/mqtt")} + , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} + , {"supported_subprotocols", t(string(), undefined, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} + , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} + , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} + , {"compress", t(boolean())} + , {"deflate_opts", ref("deflate_opts")} + , {"idle_timeout", t(duration())} + , {"max_frame_size", t(integer())} + , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} + , {"check_origin_enable", t(boolean(), undefined, false)} + , {"allow_origin_absence", t(boolean(), undefined, true)} + , {"check_origins", t(comma_separated_list())} + % @fixme + ] ++ lists:keydelete("high_watermark", 1, fields("tcp_listener_settings")); + +fields("wss_listener_settings") -> + % @fixme + Ssl = ssl(undefined, #{depth => 10 + , reuse_sessions => true}) ++ fields("listener_settings"), + Settings = lists:ukeymerge(1, Ssl, fields("ws_listener_settings")), + lists:keydelete("high_watermark", 1, Settings); + +fields("access") -> + [ {"$id", t(string(), undefined, undefined)}]; + +fields("deflate_opts") -> + [ {"level", t(union([none, default, best_compression, best_speed]))} + , {"mem_level", t(range(1, 9))} + , {"strategy", t(union([default, filtered, huffman_only, rle]))} + , {"server_context_takeover", t(union(takeover, no_takeover))} + , {"client_context_takeover", t(union(takeover, no_takeover))} + , {"server_max_window_bits", t(integer())} + , {"client_max_window_bits", t(integer())} + ]; + +fields("module") -> + [ {"loaded_file", t(string(), "emqx.modules_loaded_file", undefined)} + , {"presence", ref("presence")} + , {"subscription", ref("subscription")} + , {"rewrite", ref("rewrite")} + ]; + +fields("presence") -> + [ {"qos", t(range(0, 2), undefined, 1)}]; + +fields("subscription") -> + [ {"$id", ref("subscription_settings")}]; + +fields("subscription_settings") -> + [ {"topic", t(string())} + , {"qos", t(range(0, 2), undefined, 1)} + , {"nl", t(range(0, 1), undefined, 0)} + , {"rap", t(range(0, 1), undefined, 0)} + , {"rh", t(range(0, 2), undefined, 0)} + ]; + + +fields("rewrite") -> + [ {"rule", ref("rule")} + , {"pub_rule", ref("rule")} + , {"sub_rule", ref("rule")} + ]; + +fields("rule") -> + [ {"$id", t(string())}]; + +fields("plugins") -> + [ {"etc_dir", t(string(), "emqx.plugins_etc_dir", undefined)} + , {"loaded_file", t(string(), "emqx.plugins_loaded_file", undefined)} + , {"expand_plugins_dir", t(string(), "emqx.expand_plugins_dir", undefined)} + ]; + +fields("broker") -> + [ {"sys_interval", t(duration(), "emqx.broker_sys_interval", "1m")} + , {"sys_heartbeat", t(duration(), "emqx.broker_sys_heartbeat", "30s")} + , {"enable_session_registry", t(flag(), "emqx.enable_session_registry", true)} + , {"session_locking_strategy", t(union([local, leader, quorum, all]), + "emqx.session_locking_strategy", quorum)} + , {"shared_subscription_strategy", t(union(random, round_robin), + "emqx.shared_subscription_strategy", round_robin)} + , {"shared_dispatch_ack_enabled", t(boolean(), "emqx.shared_dispatch_ack_enabled", false)} + , {"route_batch_clean", t(flag(), "emqx.route_batch_clean", true)} + , {"perf", ref("perf")} + ]; + +fields("perf") -> + [ {"route_lock_type", t(union([key, tab, global]), "emqx.route_lock_type", key)} + , {"trie_compaction", t(boolean(), "emqx.trie_compaction", true)} + ]; + +fields("sysmon") -> + [ {"long_gc", t(duration(), undefined, 0)} + , {"long_schedule", t(duration(), undefined, 240)} + , {"large_heap", t(bytesize(), undefined, "8MB")} + , {"busy_dist_port", t(boolean(), undefined, true)} + , {"busy_port", t(boolean(), undefined, false)} + ]; + +fields("os_mon") -> + [ {"cpu_check_interval", t(duration_s(), undefined, 60)} + , {"cpu_high_watermark", t(percent(), undefined, "80%")} + , {"cpu_low_watermark", t(percent(), undefined, "60%")} + , {"mem_check_interval", t(duration_s(), undefined, 60)} + , {"sysmem_high_watermark", t(percent(), undefined, "70%")} + , {"procmem_high_watermark", t(percent(), undefined, "5%")} + ]; + +fields("vm_mon") -> + [ {"check_interval", t(duration_s(), undefined, 30)} + , {"process_high_watermark", t(percent(), undefined, "80%")} + , {"process_low_watermark", t(percent(), undefined, "60%")} + ]; + +fields("alarm") -> + [ {"actions", t(comma_separated_list(), undefined, "log,publish")} + , {"size_limit", t(integer(), undefined, 1000)} + , {"validity_period", t(duration_s(), undefined, "24h")} + ]; + +fields("telemetry") -> + [ {"enabled", t(boolean(), undefined, false)} + , {"url", t(string(), undefined, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry")} + , {"report_interval", t(duration_s(), undefined, "7d")} + ]. + + +translations() -> ["ekka", "vm_args", "gen_rpc", "kernel", "emqx"]. + +translation("ekka") -> + [ {"cluster_discovery", fun tr_cluster__discovery/1}]; + +translation("vm_args") -> + [ {"+zdbbl", fun tr_zdbbl/1} + , {"-heart", fun tr_heart/1}]; + +translation("gen_rpc") -> + [ {"tcp_client_num", fun tr_tcp_client_num/1} + , {"tcp_client_port", fun tr_tcp_client_port/1}]; + +translation("kernel") -> + [ {"logger_level", fun tr_logger_level/1} + , {"logger", fun tr_logger/1}]; + +translation("emqx") -> + [ {"flapping_detect_policy", fun tr_flapping_detect_policy/1} + , {"zones", fun tr_zones/1} + , {"listeners", fun tr_listeners/1} + , {"modules", fun tr_modules/1} + , {"sysmon", fun tr_sysmon/1} + , {"os_mon", fun tr_os_mon/1} + , {"vm_mon", fun tr_vm_mon/1} + , {"alarm", fun tr_alarm/1} + , {"telemetry", fun tr_telemetry/1} + ]. + +tr_cluster__discovery(Conf) -> + Strategy = conf_get("cluster.discovery", Conf), + {Strategy, filter(options(Strategy, Conf))}. + +tr_heart(Conf) -> + case conf_get("node.heartbeat", Conf) of + true -> ""; + "on" -> ""; + _ -> undefined + end. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +node__dist_buffer_size(type) -> bytesize(); +node__dist_buffer_size(validator) -> + fun(ZDBBL) -> + case ZDBBL >= 1024 andalso ZDBBL =< 2147482624 of + true -> + ok; + false -> + {error, "must be between 1KB and 2097151KB"} + end + end; +node__dist_buffer_size(_) -> undefined. + +tr_zdbbl(Conf) -> + case conf_get("node.dist_buffer_size", Conf) of + undefined -> undefined; + X when is_integer(X) -> ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end. + +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +tr_tcp_client_num(Conf) -> + case conf_get("rpc.tcp_client_num", Conf) of + 0 -> max(1, erlang:system_info(schedulers) div 2); + V -> V + end. + +tr_tcp_client_port(Conf) -> + conf_get("rpc.tcp_server_port", Conf). + +tr_logger_level(Conf) -> conf_get("log.level", Conf). + +tr_logger(Conf) -> + LogTo = conf_get("log.to", Conf), + LogLevel = conf_get("log.level", Conf), + LogType = case conf_get("log.rotation.enable", Conf) of + true -> wrap; + _ -> halt + end, + CharsLimit = case conf_get("log.chars_limit", Conf) of + -1 -> unlimited; + V -> V + end, + SingleLine = conf_get("log.single_line", Conf), + FmtName = conf_get("log.formatter", Conf), + Formatter = formatter(FmtName, CharsLimit, SingleLine), + BurstLimit = conf_get("log.burst_limit", Conf), + {BustLimitOn, {MaxBurstCount, TimeWindow}} = burst_limit(BurstLimit), + FileConf = fun (Filename) -> + BasicConf = + #{type => LogType, + file => filename:join(conf_get("log.dir", Conf), Filename), + max_no_files => conf_get("log.rotation.count", Conf), + sync_mode_qlen => conf_get("log.sync_mode_qlen", Conf), + drop_mode_qlen => conf_get("log.drop_mode_qlen", Conf), + flush_qlen => conf_get("log.flush_qlen", Conf), + overload_kill_enable => conf_get("log.overload_kill", Conf), + overload_kill_qlen => conf_get("log.overload_kill_qlen", Conf), + overload_kill_mem_size => conf_get("log.overload_kill_mem_size", Conf), + overload_kill_restart_after => conf_get("log.overload_kill_restart_after", Conf), + burst_limit_enable => BustLimitOn, + burst_limit_max_count => MaxBurstCount, + burst_limit_window_time => TimeWindow + }, + MaxNoBytes = case LogType of + wrap -> conf_get("log.rotation.size", Conf); + halt -> conf_get("log.size", Conf) + end, + BasicConf#{max_no_bytes => MaxNoBytes} end, + + Filters = case conf_get("log.supervisor_reports", Conf) of + error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; + progress -> [] + end, + + %% For the default logger that outputs to console + DefaultHandler = + if LogTo =:= console orelse LogTo =:= both -> + [{handler, console, logger_std_h, + #{level => LogLevel, + config => #{type => standard_io}, + formatter => Formatter, + filters => Filters + } + }]; + true -> + [{handler, default, undefined}] + end, + + %% For the file logger + FileHandler = + if LogTo =:= file orelse LogTo =:= both -> + [{handler, file, logger_disk_log_h, + #{level => LogLevel, + config => FileConf(conf_get("log.file", Conf)), + formatter => Formatter, + filesync_repeat_interval => no_repeat, + filters => Filters + }}]; + true -> [] + end, + + AdditionalLogFiles = additional_log_files(Conf), + AdditionalHandlers = + [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, + #{level => list_to_atom(Level), + config => FileConf(Filename), + formatter => Formatter, + filesync_repeat_interval => no_repeat}} + || {Level, Filename} <- AdditionalLogFiles], + + DefaultHandler ++ FileHandler ++ AdditionalHandlers. + +tr_flapping_detect_policy(Conf) -> + [Threshold, Duration, Interval] = conf_get("acl.flapping_detect_policy", Conf), + ParseDuration = fun(S, F) -> + case F(S) of + {ok, I} -> I; + {error, Reason} -> error({duration, Reason}) + end end, + #{threshold => list_to_integer(Threshold), + duration => ParseDuration(Duration, fun to_duration/1), + banned_interval => ParseDuration(Interval, fun to_duration_s/1) + }. + +tr_zones(Conf) -> + Names = lists:usort(keys("zone", Conf)), + lists:foldl( + fun(Name, Zones) -> + Zone = keys("zone." ++ Name, Conf), + Mapped = lists:flatten([map_zones(K, conf_get(["zone", Name, K], Conf)) || K <- Zone]), + [{list_to_atom(Name), lists:filter(fun ({K, []}) when K =:= ratelimit; K =:= quota -> false; + ({_, undefined}) -> false; + (_) -> true end, Mapped)} | Zones] + end, [], Names). + +tr_listeners(Conf) -> + Atom = fun(undefined) -> undefined; + (B) when is_binary(B)-> binary_to_atom(B); + (S) when is_list(S) -> list_to_atom(S) end, + + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + + AccOpts = fun(Prefix) -> + case keys(Prefix ++ ".access", Conf) of + [] -> []; + Ids -> + [{access_rules, [Access(conf_get(Prefix ++ ".access." ++ Id, Conf)) || Id <- Ids]}] + end end, + + RateLimit = fun(undefined) -> + undefined; + ([L, D]) -> + Limit = case to_bytesize(L) of + {ok, I0} -> I0; + {error, R0} -> error({bytesize, R0}) + end, + Duration = case to_duration_s(D) of + {ok, I1} -> I1; + {error, R1} -> error({duration, R1}) + end, + {Limit, Duration} + end, + + CheckOrigin = fun(S) -> [ list_to_binary(string:trim(O)) || O <- S] end, + + WsOpts = fun(Prefix) -> + case conf_get(Prefix ++ ".check_origins", Conf) of + undefined -> undefined; + Rules -> lists:flatten(CheckOrigin(Rules)) + end + end, + + LisOpts = fun(Prefix) -> + filter([{acceptors, conf_get(Prefix ++ ".acceptors", Conf)}, + {mqtt_path, conf_get(Prefix ++ ".mqtt_path", Conf)}, + {max_connections, conf_get(Prefix ++ ".max_connections", Conf)}, + {max_conn_rate, conf_get(Prefix ++ ".max_conn_rate", Conf)}, + {active_n, conf_get(Prefix ++ ".active_n", Conf)}, + {tune_buffer, conf_get(Prefix ++ ".tune_buffer", Conf)}, + {zone, Atom(conf_get(Prefix ++ ".zone", Conf))}, + {rate_limit, RateLimit(conf_get(Prefix ++ ".rate_limit", Conf))}, + {proxy_protocol, conf_get(Prefix ++ ".proxy_protocol", Conf)}, + {proxy_address_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_address_header", Conf, <<"">>)))}, + {proxy_port_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_port_header", Conf, <<"">>)))}, + {proxy_protocol_timeout, conf_get(Prefix ++ ".proxy_protocol_timeout", Conf)}, + {fail_if_no_subprotocol, conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf)}, + {supported_subprotocols, string:tokens(conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, + {peer_cert_as_username, conf_get(Prefix ++ ".peer_cert_as_username", Conf)}, + {peer_cert_as_clientid, conf_get(Prefix ++ ".peer_cert_as_clientid", Conf)}, + {compress, conf_get(Prefix ++ ".compress", Conf)}, + {idle_timeout, conf_get(Prefix ++ ".idle_timeout", Conf)}, + {max_frame_size, conf_get(Prefix ++ ".max_frame_size", Conf)}, + {mqtt_piggyback, conf_get(Prefix ++ ".mqtt_piggyback", Conf)}, + {check_origin_enable, conf_get(Prefix ++ ".check_origin_enable", Conf)}, + {allow_origin_absence, conf_get(Prefix ++ ".allow_origin_absence", Conf)}, + {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + end, + DeflateOpts = fun(Prefix) -> + filter([{level, conf_get(Prefix ++ ".deflate_opts.level", Conf)}, + {mem_level, conf_get(Prefix ++ ".deflate_opts.mem_level", Conf)}, + {strategy, conf_get(Prefix ++ ".deflate_opts.strategy", Conf)}, + {server_context_takeover, conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf)}, + {client_context_takeover, conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf)}, + {server_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf)}, + {client_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf)}]) + end, + TcpOpts = fun(Prefix) -> + filter([{backlog, conf_get(Prefix ++ ".backlog", Conf)}, + {send_timeout, conf_get(Prefix ++ ".send_timeout", Conf)}, + {send_timeout_close, conf_get(Prefix ++ ".send_timeout_close", Conf)}, + {recbuf, conf_get(Prefix ++ ".recbuf", Conf)}, + {sndbuf, conf_get(Prefix ++ ".sndbuf", Conf)}, + {buffer, conf_get(Prefix ++ ".buffer", Conf)}, + {high_watermark, conf_get(Prefix ++ ".high_watermark", Conf)}, + {nodelay, conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, conf_get(Prefix ++ ".reuseaddr", Conf)}]) + end, + + SslOpts = fun(Prefix) -> + Opts = tr_ssl(Prefix, Conf), + case lists:keyfind(ciphers, 1, Opts) of + false -> + error(Prefix ++ ".ciphers or " ++ Prefix ++ ".psk_ciphers is absent"); + _ -> + Opts + end end, + + TcpListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + ListenOnN = case conf_get(Prefix ++ ".endpoint", Conf) of + undefined -> []; + ListenOn -> ListenOn + end, + [#{ proto => Atom(Type) + , name => Name + , listen_on => ListenOnN + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end, + SslListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case conf_get(Prefix ++ ".endpoint", Conf) of + undefined -> + []; + ListenOn -> + [#{ proto => Atom(Type) + , name => Name + , listen_on => ListenOn + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + , {ssl_options, SslOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end end, + + + lists:flatten([TcpListeners("tcp", Name) || Name <- keys("listener.tcp", Conf)] + ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] + ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] + ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). + +tr_modules(Conf) -> + Subscriptions = fun() -> + List = keys("module.subscription", Conf), + TopicList = [{N, conf_get(["module", "subscription", N, "topic"], Conf)}|| N <- List], + [{list_to_binary(T), #{ qos => conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = keys("module.rewrite.rule", Conf), + PubRules = keys("module.rewrite.pub_rule", Conf), + SubRules = keys("module.rewrite.sub_rule", Conf), + TotalRules = + [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ + [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.pub_rule", R], Conf)} || R <- PubRules] ++ + [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ + [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.sub_rule", R], Conf)} || R <- SubRules], + lists:map(fun({[_, "rewrite", PubOrSub, "rule", _], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + [{emqx_mod_acl_internal, [{acl_file, conf_get("acl.acl_file", Conf)}]}] + ]). + +tr_sysmon(Conf) -> + Keys = maps:to_list(conf_get("sysmon", Conf, #{})), + [{binary_to_atom(K), maps:get(value, V)} || {K, V} <- Keys]. + +tr_os_mon(Conf) -> + [{cpu_check_interval, conf_get("os_mon.cpu_check_interval", Conf)} + , {cpu_high_watermark, conf_get("os_mon.cpu_high_watermark", Conf) * 100} + , {cpu_low_watermark, conf_get("os_mon.cpu_low_watermark", Conf) * 100} + , {mem_check_interval, conf_get("os_mon.mem_check_interval", Conf)} + , {sysmem_high_watermark, conf_get("os_mon.sysmem_high_watermark", Conf) * 100} + , {procmem_high_watermark, conf_get("os_mon.procmem_high_watermark", Conf) * 100} + ]. + +tr_vm_mon(Conf) -> + [ {check_interval, conf_get("vm_mon.check_interval", Conf)} + , {process_high_watermark, conf_get("vm_mon.process_high_watermark", Conf) * 100} + , {process_low_watermark, conf_get("vm_mon.process_low_watermark", Conf) * 100} + ]. + +tr_alarm(Conf) -> + [ {actions, [list_to_atom(Action) || Action <- conf_get("alarm.actions", Conf)]} + , {size_limit, conf_get("alarm.size_limit", Conf)} + , {validity_period, conf_get("alarm.validity_period", Conf)} + ]. + +tr_telemetry(Conf) -> + [ {enabled, conf_get("telemetry.enabled", Conf)} + , {url, conf_get("telemetry.url", Conf)} + , {report_interval, conf_get("telemetry.report_interval", Conf)} + ]. + +%% helpers + +options(static, Conf) -> + [{seeds, [list_to_atom(S) || S <- conf_get("cluster.static.seeds", Conf, "")]}]; +options(mcast, Conf) -> + {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), + Ports = [list_to_integer(S) || S <- conf_get("cluster.mcast.ports", Conf)], + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, conf_get("cluster.mcast.loop", Conf, true)}]; +options(dns, Conf) -> + [{name, conf_get("cluster.dns.name", Conf)}, + {app, conf_get("cluster.dns.app", Conf)}]; +options(etcd, Conf) -> + Namespace = "cluster.etcd.ssl", + SslOpts = fun(C) -> + Options = keys(Namespace, C), + lists:map(fun(Key) -> {list_to_atom(Key), conf_get([Namespace, Key], Conf)} end, Options) end, + [{server, conf_get("cluster.etcd.server", Conf)}, + {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, + {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, filter(SslOpts(Conf))}]; +options(k8s, Conf) -> + [{apiserver, conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, conf_get("cluster.k8s.service_name", Conf)}, + {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, conf_get("cluster.k8s.app_name", Conf)}, + {namespace, conf_get("cluster.k8s.namespace", Conf)}, + {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; +options(manual, _Conf) -> + []. + +formatter(json, CharsLimit, SingleLine) -> + {emqx_logger_jsonfmt, + #{chars_limit => CharsLimit, + single_line => SingleLine + }}; +formatter(text, CharsLimit, SingleLine) -> + {emqx_logger_textfmt, + #{template => + [time," [",level,"] ", + {clientid, + [{peername, + [clientid,"@",peername," "], + [clientid, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"], + chars_limit => CharsLimit, + single_line => SingleLine + }}. + +burst_limit(["disabled"]) -> + {false, {20000, 1000}}; +burst_limit([Count, Window]) -> + {true, {list_to_integer(Count), + case to_duration(Window) of + {ok, I} -> I; + {error, R} -> error({duration, R}) + end}}. + +%% For creating additional log files for specific log levels. +additional_log_files(Conf) -> + LogLevel = ["debug", "info", "notice", "warning", + "error", "critical", "alert", "emergency"], + additional_log_files(Conf, LogLevel, []). + +additional_log_files(_Conf, [], Acc) -> + Acc; +additional_log_files(Conf, [L | More], Acc) -> + case conf_get(["log", L, "file"], Conf) of + undefined -> additional_log_files(Conf, More, Acc); + F -> additional_log_files(Conf, More, [{L, F} | Acc]) + end. + +rate_limit_byte_dur([L, D]) -> + Limit = case to_bytesize(L) of + {ok, I0} -> I0; + {error, R0} -> error({bytesize, R0}) + end, + Duration = case to_duration_s(D) of + {ok, I1} -> I1; + {error, R1} -> error({duration, R1}) + end, + {Limit, Duration}. + +rate_limit_num_dur([L, D]) -> + Limit = case string:to_integer(L) of + {Int, []} when is_integer(Int) -> Int; + _ -> error("failed to parse bytesize string") + end, + Duration = case to_duration_s(D) of + {ok, I} -> I; + {error, Reason} -> error(Reason) + end, + {Limit, Duration}. + +map_zones(_, undefined) -> + {undefined, undefined}; +map_zones("force_gc_policy", [Count, Bytes]) -> + GcPolicy = case to_bytesize(Bytes) of + {error, Reason} -> + error({bytesize, Reason}); + {ok, Bytes1} -> + #{bytes => Bytes1, + count => list_to_integer(Count)} + end, + {force_gc_policy, GcPolicy}; +map_zones("force_shutdown_policy", ["default"]) -> + WordSize = erlang:system_info(wordsize), + {DefaultLen, DefaultSize} = + case WordSize of + 8 -> % arch_64 + {10000, hocon_postprocess:bytesize("64MB")}; + 4 -> % arch_32 + {1000, hocon_postprocess:bytesize("32MB")} + end, + {force_shutdown_policy, #{message_queue_len => DefaultLen, + max_heap_size => DefaultSize div WordSize + }}; +map_zones("force_shutdown_policy", [Len, Siz]) -> + WordSize = erlang:system_info(wordsize), + MaxSiz = case WordSize of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + ShutdownPolicy = + case to_bytesize(Siz) of + {error, Reason} -> + error(Reason); + {ok, Siz1} when Siz1 > MaxSiz -> + error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + {ok, Siz1} -> + #{message_queue_len => list_to_integer(Len), + max_heap_size => Siz1 div WordSize} + end, + {force_shutdown_policy, ShutdownPolicy}; +map_zones("mqueue_priorities", Val) -> + case Val of + ["none"] -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE + _ -> + MqueuePriorities = lists:foldl(fun(T, Acc) -> + %% NOTE: space in "= " is intended + [Topic, Prio] = string:tokens(T, "= "), + P = list_to_integer(Prio), + (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), + maps:put(iolist_to_binary(Topic), P, Acc) + end, #{}, Val), + {mqueue_priorities, MqueuePriorities} + end; +map_zones("mountpoint", Val) -> + {mountpoint, iolist_to_binary(Val)}; +map_zones("response_information", Val) -> + {response_information, iolist_to_binary(Val)}; +map_zones("rate_limit", Conf) -> + Messages = case conf_get("conn_messages_in", #{value => Conf}) of + undefined -> + []; + M -> + [{conn_messages_in, rate_limit_num_dur(M)}] + end, + Bytes = case conf_get("conn_bytes_in", #{value => Conf}) of + undefined -> + []; + B -> + [{conn_bytes_in, rate_limit_byte_dur(B)}] + end, + {ratelimit, Messages ++ Bytes}; +map_zones("conn_congestion", Conf) -> + Alarm = case conf_get("alarm", #{value => Conf}) of + undefined -> + []; + A -> + [{conn_congestion_alarm_enabled, A}] + end, + MinAlarm = case conf_get("min_alarm_sustain_duration", #{value => Conf}) of + undefined -> + []; + M -> + [{conn_congestion_min_alarm_sustain_duration, M}] + end, + Alarm ++ MinAlarm; +map_zones("quota", Conf) -> + Conn = case conf_get("conn_messages_routing", #{value => Conf}) of + undefined -> + []; + C -> + [{conn_messages_routing, rate_limit_num_dur(C)}] + end, + Overall = case conf_get("overall_messages_routing", #{value => Conf}) of + undefined -> + []; + O -> + [{overall_messages_routing, rate_limit_num_dur(O)}] + end, + {quota, Conn ++ Overall}; +map_zones(Opt, Val) -> + {list_to_atom(Opt), Val}. + + +%% utils + +-spec(conf_get(string() | [string()], hocon:config()) -> term()). +conf_get(Key, Conf) -> + V = hocon_schema:deep_get(Key, Conf, value), + case is_binary(V) of + true -> + binary_to_list(V); + false -> + V + end. + +conf_get(Key, Conf, Default) -> + V = hocon_schema:deep_get(Key, Conf, value, Default), + case is_binary(V) of + true -> + binary_to_list(V); + false -> + V + end. + +filter(Opts) -> + [{K, V} || {K, V} <- Opts, V =/= undefined]. + +%% generate a ssl field. +%% ssl("emqx", #{"verify" => verify_peer}) will return +%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} +%% , {"certfile", t(string(), "emqx.certfile", undefined)} +%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} +%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} +%% , {"server_name_indication", "emqx.server_name_indication", undefined)} +%% ... +ssl(Mapping, Defaults) -> + M = fun (Field) -> + case (Mapping) of + undefined -> undefined; + _ -> Mapping ++ "." ++ Field + end end, + D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, + [ {"enable", t(flag(), M("enable"), D("enable"))} + , {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))} + , {"certfile", t(string(), M("certfile"), D("certfile"))} + , {"keyfile", t(string(), M("keyfile"), D("keyfile"))} + , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} + , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} + , {"secure_renegotiate", t(flag(), M("secure_renegotiate"), D("secure_renegotiate"))} + , {"reuse_sessions", t(flag(), M("reuse_sessions"), D("reuse_sessions"))} + , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} + , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} + , {"depth", t(integer(), M("depth"), D("depth"))} + , {"password", t(string(), M("key_password"), D("key_password"))} + , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} + , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), + D("server_name_indication"))} + , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} + , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} + , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. + +tr_ssl(Field, Conf) -> + Versions = case conf_get([Field, "tls_versions"], Conf) of + undefined -> undefined; + Vs -> [list_to_existing_atom(V) || V <- Vs] + end, + TLSCiphers = conf_get([Field, "ciphers"], Conf), + PSKCiphers = conf_get([Field, "psk_ciphers"], Conf), + Ciphers = ciphers(TLSCiphers, PSKCiphers, Field), + case emqx_schema:conf_get([Field, "enable"], Conf) of + X when X =:= true orelse X =:= undefined -> + filter([{versions, Versions}, + {ciphers, Ciphers}, + {user_lookup_fun, user_lookup_fun(PSKCiphers)}, + {handshake_timeout, conf_get([Field, "handshake_timeout"], Conf)}, + {depth, conf_get([Field, "depth"], Conf)}, + {password, conf_get([Field, "key_password"], Conf)}, + {dhfile, conf_get([Field, "dhfile"], Conf)}, + {keyfile, emqx_schema:conf_get([Field, "keyfile"], Conf)}, + {certfile, emqx_schema:conf_get([Field, "certfile"], Conf)}, + {cacertfile, emqx_schema:conf_get([Field, "cacertfile"], Conf)}, + {verify, emqx_schema:conf_get([Field, "verify"], Conf)}, + {fail_if_no_peer_cert, conf_get([Field, "fail_if_no_peer_cert"], Conf)}, + {secure_renegotiate, conf_get([Field, "secure_renegotiate"], Conf)}, + {reuse_sessions, conf_get([Field, "reuse_sessions"], Conf)}, + {honor_cipher_order, conf_get([Field, "honor_cipher_order"], Conf)}, + {server_name_indication, emqx_schema:conf_get([Field, "server_name_indication"], Conf)} + ]); + _ -> + [] + end. + +map_psk_ciphers(PSKCiphers) -> + lists:map( + fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; + ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; + ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; + ("PSK-RC4-SHA") -> {psk, rc4_128, sha} + end, PSKCiphers). + +ciphers(undefined, undefined, _) -> + undefined; +ciphers(TLSCiphers, undefined, _) -> + TLSCiphers; +ciphers(undefined, PSKCiphers, _) -> + map_psk_ciphers(PSKCiphers); +ciphers(_, _, Field) -> + error(Field ++ ".ciphers and " ++ Field ++ ".psk_ciphers cannot be configured at the same time"). + +user_lookup_fun(undefined) -> + undefined; +user_lookup_fun(_PSKCiphers) -> + {fun emqx_psk:lookup/3, <<>>}. + +tr_password_hash(Field, Conf) -> + case emqx_schema:conf_get([Field, "password_hash"], Conf) of + [Hash] -> list_to_atom(Hash); + [Prefix, Suffix] -> {list_to_atom(Prefix), list_to_atom(Suffix)}; + [Hash, MacFun, Iterations, Dklen] -> {list_to_atom(Hash), list_to_atom(MacFun), + list_to_integer(Iterations), list_to_integer(Dklen)}; + _ -> plain + end. + + +%% @private return a list of keys in a parent field +-spec(keys(string(), hocon:config()) -> [string()]). +keys(Parent, Conf) -> + [binary_to_list(B) || B <- maps:keys(conf_get(Parent, Conf, #{}))]. + +-spec ceiling(float()) -> integer(). +ceiling(X) -> + T = erlang:trunc(X), + case (X - T) of + Neg when Neg < 0 -> T; + Pos when Pos > 0 -> T + 1; + _ -> T + end. + +%% types + +t(T) -> + fun (type) -> T; (_) -> undefined end. + +t(T, M, D) -> + fun (type) -> T; (mapping) -> M; (default) -> D; (_) -> undefined end. + +t(T, M, D, O) -> + fun (type) -> T; + (mapping) -> M; + (default) -> D; + (override_env) -> O; + (_) -> undefined end. + +ref(Field) -> + fun (type) -> Field; (_) -> undefined end. + +to_flag(Str) -> + {ok, hocon_postprocess:onoff(Str)}. + +to_duration(Str) -> + case hocon_postprocess:duration(Str) of + I when is_integer(I) -> {ok, I}; + _ -> {error, Str} + end. + +to_duration_s(Str) -> + case hocon_postprocess:duration(Str) of + I when is_integer(I) -> {ok, ceiling(I / 1000)}; + _ -> {error, Str} + end. + +to_bytesize(Str) -> + case hocon_postprocess:bytesize(Str) of + I when is_integer(I) -> {ok, I}; + _ -> {error, Str} + end. + +to_percent(Str) -> + {ok, hocon_postprocess:percent(Str)}. + +to_comma_separated_list(Str) -> + {ok, string:tokens(Str, ", ")}. + +to_bar_separated_list(Str) -> + {ok, string:tokens(Str, "| ")}. + +to_ip_port(Str) -> + case string:tokens(Str, ":") of + [Ip, Port] -> + case inet:parse_address(Ip) of + {ok, R} -> {ok, {R, list_to_integer(Port)}}; + _ -> {error, Str} + end; + _ -> {error, Str} + end. diff --git a/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl similarity index 100% rename from src/emqx_sequence.erl rename to apps/emqx/src/emqx_sequence.erl diff --git a/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl similarity index 100% rename from src/emqx_session.erl rename to apps/emqx/src/emqx_session.erl diff --git a/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl similarity index 100% rename from src/emqx_shared_sub.erl rename to apps/emqx/src/emqx_shared_sub.erl diff --git a/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl similarity index 100% rename from src/emqx_stats.erl rename to apps/emqx/src/emqx_stats.erl diff --git a/src/emqx_sup.erl b/apps/emqx/src/emqx_sup.erl similarity index 100% rename from src/emqx_sup.erl rename to apps/emqx/src/emqx_sup.erl diff --git a/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl similarity index 100% rename from src/emqx_sys.erl rename to apps/emqx/src/emqx_sys.erl diff --git a/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl similarity index 100% rename from src/emqx_sys_mon.erl rename to apps/emqx/src/emqx_sys_mon.erl diff --git a/src/emqx_sys_sup.erl b/apps/emqx/src/emqx_sys_sup.erl similarity index 100% rename from src/emqx_sys_sup.erl rename to apps/emqx/src/emqx_sys_sup.erl diff --git a/src/emqx_tables.erl b/apps/emqx/src/emqx_tables.erl similarity index 100% rename from src/emqx_tables.erl rename to apps/emqx/src/emqx_tables.erl diff --git a/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl similarity index 100% rename from src/emqx_tls_lib.erl rename to apps/emqx/src/emqx_tls_lib.erl diff --git a/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl similarity index 100% rename from src/emqx_topic.erl rename to apps/emqx/src/emqx_topic.erl diff --git a/src/emqx_tracer.erl b/apps/emqx/src/emqx_tracer.erl similarity index 100% rename from src/emqx_tracer.erl rename to apps/emqx/src/emqx_tracer.erl diff --git a/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl similarity index 100% rename from src/emqx_trie.erl rename to apps/emqx/src/emqx_trie.erl diff --git a/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl similarity index 100% rename from src/emqx_types.erl rename to apps/emqx/src/emqx_types.erl diff --git a/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl similarity index 100% rename from src/emqx_vm.erl rename to apps/emqx/src/emqx_vm.erl diff --git a/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl similarity index 100% rename from src/emqx_vm_mon.erl rename to apps/emqx/src/emqx_vm_mon.erl diff --git a/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl similarity index 100% rename from src/emqx_ws_connection.erl rename to apps/emqx/src/emqx_ws_connection.erl diff --git a/src/emqx_zone.erl b/apps/emqx/src/emqx_zone.erl similarity index 100% rename from src/emqx_zone.erl rename to apps/emqx/src/emqx_zone.erl diff --git a/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl similarity index 100% rename from test/emqx_SUITE.erl rename to apps/emqx/test/emqx_SUITE.erl diff --git a/test/emqx_SUITE_data/acl.conf b/apps/emqx/test/emqx_SUITE_data/acl.conf similarity index 100% rename from test/emqx_SUITE_data/acl.conf rename to apps/emqx/test/emqx_SUITE_data/acl.conf diff --git a/test/emqx_SUITE_data/loaded_modules b/apps/emqx/test/emqx_SUITE_data/loaded_modules similarity index 100% rename from test/emqx_SUITE_data/loaded_modules rename to apps/emqx/test/emqx_SUITE_data/loaded_modules diff --git a/test/emqx_SUITE_data/loaded_plugins b/apps/emqx/test/emqx_SUITE_data/loaded_plugins similarity index 100% rename from test/emqx_SUITE_data/loaded_plugins rename to apps/emqx/test/emqx_SUITE_data/loaded_plugins diff --git a/test/emqx_access_SUITE_data/acl.conf b/apps/emqx/test/emqx_access_SUITE_data/acl.conf similarity index 100% rename from test/emqx_access_SUITE_data/acl.conf rename to apps/emqx/test/emqx_access_SUITE_data/acl.conf diff --git a/test/emqx_access_SUITE_data/acl_deny_action.conf b/apps/emqx/test/emqx_access_SUITE_data/acl_deny_action.conf similarity index 100% rename from test/emqx_access_SUITE_data/acl_deny_action.conf rename to apps/emqx/test/emqx_access_SUITE_data/acl_deny_action.conf diff --git a/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl similarity index 100% rename from test/emqx_access_control_SUITE.erl rename to apps/emqx/test/emqx_access_control_SUITE.erl diff --git a/test/emqx_access_rule_SUITE.erl b/apps/emqx/test/emqx_access_rule_SUITE.erl similarity index 100% rename from test/emqx_access_rule_SUITE.erl rename to apps/emqx/test/emqx_access_rule_SUITE.erl diff --git a/test/emqx_acl_cache_SUITE.erl b/apps/emqx/test/emqx_acl_cache_SUITE.erl similarity index 100% rename from test/emqx_acl_cache_SUITE.erl rename to apps/emqx/test/emqx_acl_cache_SUITE.erl diff --git a/test/emqx_acl_test_mod.erl b/apps/emqx/test/emqx_acl_test_mod.erl similarity index 100% rename from test/emqx_acl_test_mod.erl rename to apps/emqx/test/emqx_acl_test_mod.erl diff --git a/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl similarity index 100% rename from test/emqx_alarm_SUITE.erl rename to apps/emqx/test/emqx_alarm_SUITE.erl diff --git a/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl similarity index 100% rename from test/emqx_banned_SUITE.erl rename to apps/emqx/test/emqx_banned_SUITE.erl diff --git a/test/emqx_batch_SUITE.erl b/apps/emqx/test/emqx_batch_SUITE.erl similarity index 100% rename from test/emqx_batch_SUITE.erl rename to apps/emqx/test/emqx_batch_SUITE.erl diff --git a/test/emqx_boot_SUITE.erl b/apps/emqx/test/emqx_boot_SUITE.erl similarity index 100% rename from test/emqx_boot_SUITE.erl rename to apps/emqx/test/emqx_boot_SUITE.erl diff --git a/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl similarity index 100% rename from test/emqx_broker_SUITE.erl rename to apps/emqx/test/emqx_broker_SUITE.erl diff --git a/test/emqx_broker_helper_SUITE.erl b/apps/emqx/test/emqx_broker_helper_SUITE.erl similarity index 100% rename from test/emqx_broker_helper_SUITE.erl rename to apps/emqx/test/emqx_broker_helper_SUITE.erl diff --git a/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl similarity index 100% rename from test/emqx_channel_SUITE.erl rename to apps/emqx/test/emqx_channel_SUITE.erl diff --git a/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl similarity index 100% rename from test/emqx_client_SUITE.erl rename to apps/emqx/test/emqx_client_SUITE.erl diff --git a/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl similarity index 100% rename from test/emqx_cm_SUITE.erl rename to apps/emqx/test/emqx_cm_SUITE.erl diff --git a/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl similarity index 100% rename from test/emqx_cm_locker_SUITE.erl rename to apps/emqx/test/emqx_cm_locker_SUITE.erl diff --git a/test/emqx_cm_registry_SUITE.erl b/apps/emqx/test/emqx_cm_registry_SUITE.erl similarity index 100% rename from test/emqx_cm_registry_SUITE.erl rename to apps/emqx/test/emqx_cm_registry_SUITE.erl diff --git a/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl similarity index 100% rename from test/emqx_connection_SUITE.erl rename to apps/emqx/test/emqx_connection_SUITE.erl diff --git a/test/emqx_ctl_SUITE.erl b/apps/emqx/test/emqx_ctl_SUITE.erl similarity index 100% rename from test/emqx_ctl_SUITE.erl rename to apps/emqx/test/emqx_ctl_SUITE.erl diff --git a/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl similarity index 100% rename from test/emqx_flapping_SUITE.erl rename to apps/emqx/test/emqx_flapping_SUITE.erl diff --git a/test/emqx_frame_SUITE.erl b/apps/emqx/test/emqx_frame_SUITE.erl similarity index 100% rename from test/emqx_frame_SUITE.erl rename to apps/emqx/test/emqx_frame_SUITE.erl diff --git a/test/emqx_gc_SUITE.erl b/apps/emqx/test/emqx_gc_SUITE.erl similarity index 100% rename from test/emqx_gc_SUITE.erl rename to apps/emqx/test/emqx_gc_SUITE.erl diff --git a/test/emqx_global_gc_SUITE.erl b/apps/emqx/test/emqx_global_gc_SUITE.erl similarity index 100% rename from test/emqx_global_gc_SUITE.erl rename to apps/emqx/test/emqx_global_gc_SUITE.erl diff --git a/test/emqx_guid_SUITE.erl b/apps/emqx/test/emqx_guid_SUITE.erl similarity index 100% rename from test/emqx_guid_SUITE.erl rename to apps/emqx/test/emqx_guid_SUITE.erl diff --git a/test/emqx_hooks_SUITE.erl b/apps/emqx/test/emqx_hooks_SUITE.erl similarity index 100% rename from test/emqx_hooks_SUITE.erl rename to apps/emqx/test/emqx_hooks_SUITE.erl diff --git a/test/emqx_inflight_SUITE.erl b/apps/emqx/test/emqx_inflight_SUITE.erl similarity index 100% rename from test/emqx_inflight_SUITE.erl rename to apps/emqx/test/emqx_inflight_SUITE.erl diff --git a/test/emqx_json_SUITE.erl b/apps/emqx/test/emqx_json_SUITE.erl similarity index 100% rename from test/emqx_json_SUITE.erl rename to apps/emqx/test/emqx_json_SUITE.erl diff --git a/test/emqx_keepalive_SUITE.erl b/apps/emqx/test/emqx_keepalive_SUITE.erl similarity index 100% rename from test/emqx_keepalive_SUITE.erl rename to apps/emqx/test/emqx_keepalive_SUITE.erl diff --git a/test/emqx_limiter_SUITE.erl b/apps/emqx/test/emqx_limiter_SUITE.erl similarity index 100% rename from test/emqx_limiter_SUITE.erl rename to apps/emqx/test/emqx_limiter_SUITE.erl diff --git a/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl similarity index 88% rename from test/emqx_listeners_SUITE.erl rename to apps/emqx/test/emqx_listeners_SUITE.erl index f49d33004..53f388dfa 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -48,7 +48,7 @@ t_restart_listeners(_) -> ok = emqx_listeners:stop(). render_config_file() -> - Path = local_path(["..", "..", "..", "..", "etc", "emqx.conf"]), + Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), Vars0 = mustache_vars(), Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0], @@ -65,16 +65,17 @@ mustache_vars() -> ]. generate_config() -> - Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), ConfFile = render_config_file(), - Conf = conf_parse:file(ConfFile), - cuttlefish_generator:map(Schema, Conf). + {ok, Conf} = hocon:load(ConfFile, #{format => richmap}), + hocon_schema:generate(emqx_schema, Conf). set_app_env({App, Lists}) -> lists:foreach(fun({acl_file, _Var}) -> application:set_env(App, acl_file, local_path(["etc", "acl.conf"])); ({plugins_loaded_file, _Var}) -> - application:set_env(App, plugins_loaded_file, local_path(["test", "emqx_SUITE_data","loaded_plugins"])); + application:set_env(App, + plugins_loaded_file, + local_path(["test", "emqx_SUITE_data","loaded_plugins"])); ({Par, Var}) -> application:set_env(App, Par, Var) end, Lists). @@ -91,4 +92,4 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - + diff --git a/test/emqx_logger_SUITE.erl b/apps/emqx/test/emqx_logger_SUITE.erl similarity index 100% rename from test/emqx_logger_SUITE.erl rename to apps/emqx/test/emqx_logger_SUITE.erl diff --git a/test/emqx_message_SUITE.erl b/apps/emqx/test/emqx_message_SUITE.erl similarity index 100% rename from test/emqx_message_SUITE.erl rename to apps/emqx/test/emqx_message_SUITE.erl diff --git a/test/emqx_metrics_SUITE.erl b/apps/emqx/test/emqx_metrics_SUITE.erl similarity index 100% rename from test/emqx_metrics_SUITE.erl rename to apps/emqx/test/emqx_metrics_SUITE.erl diff --git a/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl similarity index 100% rename from test/emqx_misc_SUITE.erl rename to apps/emqx/test/emqx_misc_SUITE.erl diff --git a/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl similarity index 100% rename from test/emqx_mountpoint_SUITE.erl rename to apps/emqx/test/emqx_mountpoint_SUITE.erl diff --git a/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl similarity index 100% rename from test/emqx_mqtt_SUITE.erl rename to apps/emqx/test/emqx_mqtt_SUITE.erl diff --git a/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl similarity index 100% rename from test/emqx_mqtt_caps_SUITE.erl rename to apps/emqx/test/emqx_mqtt_caps_SUITE.erl diff --git a/test/emqx_mqtt_props_SUITE.erl b/apps/emqx/test/emqx_mqtt_props_SUITE.erl similarity index 100% rename from test/emqx_mqtt_props_SUITE.erl rename to apps/emqx/test/emqx_mqtt_props_SUITE.erl diff --git a/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl similarity index 100% rename from test/emqx_mqtt_protocol_v5_SUITE.erl rename to apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl diff --git a/test/emqx_mqueue_SUITE.erl b/apps/emqx/test/emqx_mqueue_SUITE.erl similarity index 100% rename from test/emqx_mqueue_SUITE.erl rename to apps/emqx/test/emqx_mqueue_SUITE.erl diff --git a/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl similarity index 100% rename from test/emqx_os_mon_SUITE.erl rename to apps/emqx/test/emqx_os_mon_SUITE.erl diff --git a/test/emqx_packet_SUITE.erl b/apps/emqx/test/emqx_packet_SUITE.erl similarity index 100% rename from test/emqx_packet_SUITE.erl rename to apps/emqx/test/emqx_packet_SUITE.erl diff --git a/test/emqx_passwd_SUITE.erl b/apps/emqx/test/emqx_passwd_SUITE.erl similarity index 100% rename from test/emqx_passwd_SUITE.erl rename to apps/emqx/test/emqx_passwd_SUITE.erl diff --git a/test/emqx_pd_SUITE.erl b/apps/emqx/test/emqx_pd_SUITE.erl similarity index 100% rename from test/emqx_pd_SUITE.erl rename to apps/emqx/test/emqx_pd_SUITE.erl diff --git a/test/emqx_plugins_SUITE.erl b/apps/emqx/test/emqx_plugins_SUITE.erl similarity index 100% rename from test/emqx_plugins_SUITE.erl rename to apps/emqx/test/emqx_plugins_SUITE.erl diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl similarity index 100% rename from test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl rename to apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl diff --git a/test/emqx_pmon_SUITE.erl b/apps/emqx/test/emqx_pmon_SUITE.erl similarity index 100% rename from test/emqx_pmon_SUITE.erl rename to apps/emqx/test/emqx_pmon_SUITE.erl diff --git a/test/emqx_pool_SUITE.erl b/apps/emqx/test/emqx_pool_SUITE.erl similarity index 100% rename from test/emqx_pool_SUITE.erl rename to apps/emqx/test/emqx_pool_SUITE.erl diff --git a/test/emqx_pqueue_SUITE.erl b/apps/emqx/test/emqx_pqueue_SUITE.erl similarity index 100% rename from test/emqx_pqueue_SUITE.erl rename to apps/emqx/test/emqx_pqueue_SUITE.erl diff --git a/test/emqx_reason_codes_SUITE.erl b/apps/emqx/test/emqx_reason_codes_SUITE.erl similarity index 100% rename from test/emqx_reason_codes_SUITE.erl rename to apps/emqx/test/emqx_reason_codes_SUITE.erl diff --git a/test/emqx_request_handler.erl b/apps/emqx/test/emqx_request_handler.erl similarity index 100% rename from test/emqx_request_handler.erl rename to apps/emqx/test/emqx_request_handler.erl diff --git a/test/emqx_request_responser_SUITE.erl b/apps/emqx/test/emqx_request_responser_SUITE.erl similarity index 100% rename from test/emqx_request_responser_SUITE.erl rename to apps/emqx/test/emqx_request_responser_SUITE.erl diff --git a/test/emqx_request_sender.erl b/apps/emqx/test/emqx_request_sender.erl similarity index 100% rename from test/emqx_request_sender.erl rename to apps/emqx/test/emqx_request_sender.erl diff --git a/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl similarity index 100% rename from test/emqx_router_SUITE.erl rename to apps/emqx/test/emqx_router_SUITE.erl diff --git a/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl similarity index 100% rename from test/emqx_router_helper_SUITE.erl rename to apps/emqx/test/emqx_router_helper_SUITE.erl diff --git a/test/emqx_sequence_SUITE.erl b/apps/emqx/test/emqx_sequence_SUITE.erl similarity index 100% rename from test/emqx_sequence_SUITE.erl rename to apps/emqx/test/emqx_sequence_SUITE.erl diff --git a/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl similarity index 100% rename from test/emqx_session_SUITE.erl rename to apps/emqx/test/emqx_session_SUITE.erl diff --git a/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl similarity index 100% rename from test/emqx_shared_sub_SUITE.erl rename to apps/emqx/test/emqx_shared_sub_SUITE.erl diff --git a/test/emqx_stats_SUITE.erl b/apps/emqx/test/emqx_stats_SUITE.erl similarity index 100% rename from test/emqx_stats_SUITE.erl rename to apps/emqx/test/emqx_stats_SUITE.erl diff --git a/test/emqx_sup_SUITE.erl b/apps/emqx/test/emqx_sup_SUITE.erl similarity index 100% rename from test/emqx_sup_SUITE.erl rename to apps/emqx/test/emqx_sup_SUITE.erl diff --git a/test/emqx_sys_SUITE.erl b/apps/emqx/test/emqx_sys_SUITE.erl similarity index 100% rename from test/emqx_sys_SUITE.erl rename to apps/emqx/test/emqx_sys_SUITE.erl diff --git a/test/emqx_sys_mon_SUITE.erl b/apps/emqx/test/emqx_sys_mon_SUITE.erl similarity index 100% rename from test/emqx_sys_mon_SUITE.erl rename to apps/emqx/test/emqx_sys_mon_SUITE.erl diff --git a/test/emqx_tables_SUITE.erl b/apps/emqx/test/emqx_tables_SUITE.erl similarity index 100% rename from test/emqx_tables_SUITE.erl rename to apps/emqx/test/emqx_tables_SUITE.erl diff --git a/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl similarity index 100% rename from test/emqx_takeover_SUITE.erl rename to apps/emqx/test/emqx_takeover_SUITE.erl diff --git a/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl similarity index 100% rename from test/emqx_tls_lib_tests.erl rename to apps/emqx/test/emqx_tls_lib_tests.erl diff --git a/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl similarity index 100% rename from test/emqx_topic_SUITE.erl rename to apps/emqx/test/emqx_topic_SUITE.erl diff --git a/test/emqx_tracer_SUITE.erl b/apps/emqx/test/emqx_tracer_SUITE.erl similarity index 100% rename from test/emqx_tracer_SUITE.erl rename to apps/emqx/test/emqx_tracer_SUITE.erl diff --git a/test/emqx_trie_SUITE.erl b/apps/emqx/test/emqx_trie_SUITE.erl similarity index 100% rename from test/emqx_trie_SUITE.erl rename to apps/emqx/test/emqx_trie_SUITE.erl diff --git a/test/emqx_vm_SUITE.erl b/apps/emqx/test/emqx_vm_SUITE.erl similarity index 100% rename from test/emqx_vm_SUITE.erl rename to apps/emqx/test/emqx_vm_SUITE.erl diff --git a/test/emqx_vm_mon_SUITE.erl b/apps/emqx/test/emqx_vm_mon_SUITE.erl similarity index 100% rename from test/emqx_vm_mon_SUITE.erl rename to apps/emqx/test/emqx_vm_mon_SUITE.erl diff --git a/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl similarity index 100% rename from test/emqx_ws_connection_SUITE.erl rename to apps/emqx/test/emqx_ws_connection_SUITE.erl diff --git a/test/emqx_zone_SUITE.erl b/apps/emqx/test/emqx_zone_SUITE.erl similarity index 100% rename from test/emqx_zone_SUITE.erl rename to apps/emqx/test/emqx_zone_SUITE.erl diff --git a/test/props/prop_emqx_base62.erl b/apps/emqx/test/props/prop_emqx_base62.erl similarity index 100% rename from test/props/prop_emqx_base62.erl rename to apps/emqx/test/props/prop_emqx_base62.erl diff --git a/test/props/prop_emqx_frame.erl b/apps/emqx/test/props/prop_emqx_frame.erl similarity index 100% rename from test/props/prop_emqx_frame.erl rename to apps/emqx/test/props/prop_emqx_frame.erl diff --git a/test/props/prop_emqx_json.erl b/apps/emqx/test/props/prop_emqx_json.erl similarity index 100% rename from test/props/prop_emqx_json.erl rename to apps/emqx/test/props/prop_emqx_json.erl diff --git a/test/props/prop_emqx_psk.erl b/apps/emqx/test/props/prop_emqx_psk.erl similarity index 100% rename from test/props/prop_emqx_psk.erl rename to apps/emqx/test/props/prop_emqx_psk.erl diff --git a/test/props/prop_emqx_reason_codes.erl b/apps/emqx/test/props/prop_emqx_reason_codes.erl similarity index 100% rename from test/props/prop_emqx_reason_codes.erl rename to apps/emqx/test/props/prop_emqx_reason_codes.erl diff --git a/test/props/prop_emqx_rpc.erl b/apps/emqx/test/props/prop_emqx_rpc.erl similarity index 100% rename from test/props/prop_emqx_rpc.erl rename to apps/emqx/test/props/prop_emqx_rpc.erl diff --git a/test/props/prop_emqx_sys.erl b/apps/emqx/test/props/prop_emqx_sys.erl similarity index 100% rename from test/props/prop_emqx_sys.erl rename to apps/emqx/test/props/prop_emqx_sys.erl diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 058c86dbf..56e2055c0 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -7,7 +7,7 @@ ## Value: URL ## ## Examples: http://127.0.0.1:80/mqtt/auth, https://[::1]:80/mqtt/auth -auth.http.auth_req.url = http://127.0.0.1:80/mqtt/auth +auth.http.auth_req.url = "http://127.0.0.1:80/mqtt/auth" ## HTTP Request Method for Auth Request ## @@ -18,7 +18,8 @@ auth.http.auth_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.auth_req.headers.accept = */* -auth.http.auth_req.headers.content_type = application/x-www-form-urlencoded + +auth.http.auth_req.headers.content_type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters @@ -35,14 +36,14 @@ auth.http.auth_req.headers.content_type = application/x-www-form-urlencoded ## - %d: subject of client TLS cert ## ## Value: =,=,... -auth.http.auth_req.params = clientid=%c,username=%u,password=%P +auth.http.auth_req.params = "clientid=%c,username=%u,password=%P" ## HTTP URL API path for SuperUser Request ## ## Value: URL ## ## Examples: http://127.0.0.1:80/mqtt/superuser, https://[::1]:80/mqtt/superuser -auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser +auth.http.super_req.url = "http://127.0.0.1:80/mqtt/superuser" ## HTTP Request Method for SuperUser Request ## @@ -53,7 +54,7 @@ auth.http.super_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.super_req.headers.accept = */* -auth.http.super_req.headers.content-type = application/x-www-form-urlencoded +auth.http.super_req.headers.content-type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters @@ -70,7 +71,7 @@ auth.http.super_req.headers.content-type = application/x-www-form-urlencoded ## - %d: subject of client TLS cert ## ## Value: =,=,... -auth.http.super_req.params = clientid=%c,username=%u +auth.http.super_req.params = "clientid=%c,username=%u" ## HTTP URL API path for ACL Request ## Comment out this config to disable ACL checks @@ -78,7 +79,7 @@ auth.http.super_req.params = clientid=%c,username=%u ## Value: URL ## ## Examples: http://127.0.0.1:80/mqtt/acl, https://[::1]:80/mqtt/acl -auth.http.acl_req.url = http://127.0.0.1:80/mqtt/acl +auth.http.acl_req.url = "http://127.0.0.1:80/mqtt/acl" ## HTTP Request Method for ACL Request ## @@ -89,7 +90,7 @@ auth.http.acl_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.acl_req.headers.accept = */* -auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded +auth.http.acl_req.headers.content-type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters @@ -108,7 +109,7 @@ auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded ## - %t: topic ## ## Value: =,=,... -auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m +auth.http.acl_req.params = "access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m" ## Time-out time for the request. ## @@ -143,17 +144,17 @@ auth.http.pool_size = 32 ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## auth.http.ssl.cacertfile = {{ platform_etc_dir }}/certs/ca.pem +## auth.http.ssl.cacertfile = "{{ platform_etc_dir }}/certs/ca.pem" ## The path to a file containing the client's certificate. ## ## Value: File -## auth.http.ssl.certfile = {{ platform_etc_dir }}/certs/client-cert.pem +## auth.http.ssl.certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" ## Path to a file containing the client's private PEM-encoded key. ## ## Value: File -## auth.http.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +## auth.http.ssl.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ## In mode verify_none the default behavior is to allow all x509-path ## validation errors. diff --git a/apps/emqx_auth_http/rebar.config b/apps/emqx_auth_http/rebar.config index 142b7b429..01c0f4209 100644 --- a/apps/emqx_auth_http/rebar.config +++ b/apps/emqx_auth_http/rebar.config @@ -19,7 +19,7 @@ {profiles, [{test, [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + [ {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "v1.2.2"}}} ]} ]} diff --git a/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf b/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf index 748902f9f..e0e6d48dd 100644 --- a/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf +++ b/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf @@ -10,13 +10,13 @@ auth.jwt.secret = emqxsecret ## RSA or ECDSA public key file. ## ## Value: File -#auth.jwt.pubkey = etc/certs/jwt_public_key.pem +#auth.jwt.pubkey = "etc/certs/jwt_public_key.pem" ## The JWKs server address ## ## see: http://self-issued.info/docs/draft-ietf-jose-json-web-key.html ## -#auth.jwt.jwks = https://127.0.0.1:8080/jwks +#auth.jwt.jwks.endpoint = "https://127.0.0.1:8080/jwks" ## The JWKs refresh interval ## @@ -32,7 +32,7 @@ auth.jwt.from = password ## Enable to verify claims fields ## ## Value: on | off -auth.jwt.verify_claims = off +auth.jwt.verify_claims.enable = off ## The checklist of claims to validate ## @@ -46,4 +46,4 @@ auth.jwt.verify_claims = off ## ## For example, to verify that the username in the JWT payload is the same ## as the client (MQTT protocol) username -#auth.jwt.verify_claims.username = %u +#auth.jwt.verify_claims.username = "%u" \ No newline at end of file diff --git a/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema b/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema index 3d8de3678..10b2daa5e 100644 --- a/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema +++ b/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema @@ -4,7 +4,7 @@ {datatype, string} ]}. -{mapping, "auth.jwt.jwks", "emqx_auth_jwt.jwks", [ +{mapping, "auth.jwt.jwks.endpoint", "emqx_auth_jwt.jwks", [ {datatype, string} ]}. @@ -26,7 +26,7 @@ {datatype, {enum, [raw, der]}} ]}. -{mapping, "auth.jwt.verify_claims", "emqx_auth_jwt.verify_claims", [ +{mapping, "auth.jwt.verify_claims.enable", "emqx_auth_jwt.verify_claims", [ {default, off}, {datatype, flag} ]}. @@ -36,7 +36,7 @@ ]}. {translation, "emqx_auth_jwt.verify_claims", fun(Conf) -> - case cuttlefish:conf_get("auth.jwt.verify_claims", Conf) of + case cuttlefish:conf_get("auth.jwt.verify_claims.enable", Conf) of false -> cuttlefish:unset(); true -> lists:foldr( diff --git a/apps/emqx_auth_jwt/rebar.config b/apps/emqx_auth_jwt/rebar.config index 5e7575881..3ec554950 100644 --- a/apps/emqx_auth_jwt/rebar.config +++ b/apps/emqx_auth_jwt/rebar.config @@ -20,6 +20,6 @@ {profiles, [{test, - [{deps, [{emqx_ct_helpers, {git, "http://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + [{deps, []} ]} ]}. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 8db4ffe84..7d784e3b2 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf index 8eebefe97..b457229e3 100644 --- a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf +++ b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf @@ -5,7 +5,7 @@ ## LDAP server list, seperated by ','. ## ## Value: String -auth.ldap.servers = 127.0.0.1 +auth.ldap.servers = "127.0.0.1" ## LDAP server port. ## @@ -20,7 +20,7 @@ auth.ldap.pool = 8 ## LDAP Bind DN. ## ## Value: DN -auth.ldap.bind_dn = cn=root,dc=emqx,dc=io +auth.ldap.bind_dn = "cn=root,dc=emqx,dc=io" ## LDAP Bind Password. ## @@ -37,7 +37,7 @@ auth.ldap.timeout = 30s ## Variables: ## ## Value: DN -auth.ldap.device_dn = ou=device,dc=emqx,dc=io +auth.ldap.device_dn = "ou=device,dc=emqx,dc=io" ## Specified ObjectClass ## @@ -63,14 +63,14 @@ auth.ldap.password.attributetype = userPassword ## Whether to enable SSL. ## ## Value: true | false -auth.ldap.ssl = false +auth.ldap.ssl.enable = false -#auth.ldap.ssl.certfile = etc/certs/cert.pem +#auth.ldap.ssl.certfile = "etc/certs/cert.pem" -#auth.ldap.ssl.keyfile = etc/certs/key.pem +#auth.ldap.ssl.keyfile = "etc/certs/key.pem" -#auth.ldap.ssl.cacertfile = etc/certs/cacert.pem +#auth.ldap.ssl.cacertfile = "etc/certs/cacert.pem" -#auth.ldap.ssl.verify = verify_peer +#auth.ldap.ssl.verify = "verify_peer" #auth.ldap.ssl.server_name_indication = your_server_name diff --git a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema index b3d3de1a2..f9c3bf16b 100644 --- a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema +++ b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema @@ -31,7 +31,7 @@ {datatype, {duration, ms}} ]}. -{mapping, "auth.ldap.ssl", "emqx_auth_ldap.ldap", [ +{mapping, "auth.ldap.ssl.enable", "emqx_auth_ldap.ldap", [ {default, false}, {datatype, {enum, [true, false]}} ]}. @@ -83,7 +83,7 @@ {bind_password, BindPassword}, {pool, Pool}, {auto_reconnect, 2}], - case cuttlefish:conf_get("auth.ldap.ssl", Conf) of + case cuttlefish:conf_get("auth.ldap.ssl.enable", Conf) of true -> [{ssl, true}, {sslopts, Filter(SslOpts())}|Opts]; false -> [{ssl, false}|Opts] end diff --git a/apps/emqx_auth_ldap/rebar.config b/apps/emqx_auth_ldap/rebar.config index 48eaf812f..811468a7b 100644 --- a/apps/emqx_auth_ldap/rebar.config +++ b/apps/emqx_auth_ldap/rebar.config @@ -4,7 +4,7 @@ {profiles, [{test, - [{deps, [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + [{deps, []} ]} ]}. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 8635c4834..1b76c32c8 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf b/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf index ff74656cb..758df1a9c 100644 --- a/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf +++ b/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf @@ -10,12 +10,12 @@ auth.mnesia.password_hash = sha256 ## Examples ##auth.client.1.clientid = id ##auth.client.1.password = passwd -##auth.client.2.clientid = dev:devid +##auth.client.2.clientid = "dev:devid" ##auth.client.2.password = passwd2 -##auth.client.3.clientid = app:appid +##auth.client.3.clientid = "app:appid" ##auth.client.3.password = passwd3 -##auth.client.4.clientid = client~!@#$%^&*()_+ -##auth.client.4.password = passwd~!@#$%^&*()_+ +##auth.client.4.clientid = "client~!@#$%^&*()_+" +##auth.client.4.password = "passwd~!@#$%^&*()_+" ##-------------------------------------------------------------------- ## Username Authentication @@ -26,5 +26,5 @@ auth.mnesia.password_hash = sha256 ##auth.user.1.password = public ##auth.user.2.username = feng@emqtt.io ##auth.user.2.password = public -##auth.user.3.username = name~!@#$%^&*()_+ -##auth.user.3.password = pwsswd~!@#$%^&*()_+ +##auth.user.3.username = "name~!@#$%^&*()_+" +##auth.user.3.password = "pwsswd~!@#$%^&*()_+" diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 2a3d038f0..c59c80643 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -16,8 +16,8 @@ auth.mongo.type = single ## ## Value: String ## -## Examples: 127.0.0.1:27017,127.0.0.2:27017... -auth.mongo.server = 127.0.0.1:27017 +## Examples: "127.0.0.1:27017,127.0.0.2:27017,..." +auth.mongo.server = "127.0.0.1:27017" ## MongoDB pool size ## @@ -53,7 +53,7 @@ auth.mongo.database = mqtt ## Whether to enable SSL connection. ## ## Value: on | off -## auth.mongo.ssl = off +## auth.mongo.ssl.enable = off ## SSL keyfile. ## @@ -117,17 +117,17 @@ auth.mongo.topology.max_overflow = 0 auth.mongo.auth_query.password_hash = sha256 ## sha256 with salt suffix -## auth.mongo.auth_query.password_hash = sha256,salt +## auth.mongo.auth_query.password_hash = "sha256,salt" ## sha256 with salt prefix -## auth.mongo.auth_query.password_hash = salt,sha256 +## auth.mongo.auth_query.password_hash = "salt,sha256" ## bcrypt with salt prefix -## auth.mongo.auth_query.password_hash = salt,bcrypt +## auth.mongo.auth_query.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.mongo.auth_query.password_hash = pbkdf2,sha256,1000,20 +## auth.mongo.auth_query.password_hash = "pbkdf2,sha256,1000,20" ## Authentication query. auth.mongo.auth_query.collection = mqtt_user @@ -146,15 +146,15 @@ auth.mongo.auth_query.password_field = password ## - %d: subject of client TLS cert ## ## auth.mongo.auth_query.selector = {Field}={Placeholder} -auth.mongo.auth_query.selector = username=%u +auth.mongo.auth_query.selector = "username=%u" ## ------------------------------------------------- ## Super User Query ## ------------------------------------------------- auth.mongo.super_query.collection = mqtt_user auth.mongo.super_query.super_field = is_superuser -#auth.mongo.super_query.selector = username=%u, clientid=%c -auth.mongo.super_query.selector = username=%u +#auth.mongo.super_query.selector.1 = username=%u, clientid=%c +auth.mongo.super_query.selector = "username=%u" ## ACL Selector. ## @@ -165,8 +165,8 @@ auth.mongo.super_query.selector = username=%u ## ## With following 2 selectors configured: ## -## auth.mongo.acl_query.selector.1 = username=%u -## auth.mongo.acl_query.selector.2 = username=$all +## auth.mongo.acl_query.selector.1 = "username=%u" +## auth.mongo.acl_query.selector.2 = "username=$all" ## ## And if a client connected using username 'ilyas', ## then the following mongo command will be used to @@ -180,8 +180,8 @@ auth.mongo.super_query.selector = username=%u ## ## Examples: ## -## auth.mongo.acl_query.selector.1 = username=%u,clientid=%c -## auth.mongo.acl_query.selector.2 = username=$all -## auth.mongo.acl_query.selector.3 = clientid=$all +## auth.mongo.acl_query.selector.1 = "username=%u,clientid=%c" +## auth.mongo.acl_query.selector.2 = "username=$all" +## auth.mongo.acl_query.selector.3 = "clientid=$all" auth.mongo.acl_query.collection = mqtt_acl -auth.mongo.acl_query.selector = username=%u +auth.mongo.acl_query.selector = "username=%u" diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 8a2ff98b3..cd8c03015 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -45,7 +45,7 @@ {datatype, string} ]}. -{mapping, "auth.mongo.ssl", "emqx_auth_mongo.server", [ +{mapping, "auth.mongo.ssl.enable", "emqx_auth_mongo.server", [ {default, off}, {datatype, {enum, [on, off, true, false]}} %% FIXME: ture/false is compatible with 4.0-4.2 version format, plan to delete in 5.0 ]}. @@ -130,8 +130,6 @@ true -> []; false -> [{r_mode, R}] end, - - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> Verify = case cuttlefish:conf_get(Prefix ++ ".verify", Conf, false) of @@ -149,8 +147,14 @@ end, %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 - Ssl = case cuttlefish:conf_get("auth.mongo.ssl", Conf) of - on -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl")}]; + GenSsl = case cuttlefish:conf_get("auth.mongo.ssl.cacertfile", Conf, undefined) of + undefined -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl_opts")}]; + _ -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl")}] + end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + Ssl = case cuttlefish:conf_get("auth.mongo.ssl.enable", Conf) of + on -> GenSsl; off -> []; true -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl_opts")}]; false -> [] diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index cc4e72ef3..ab0b4ff56 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index 6014329b3..1c3d40059 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -7,7 +7,7 @@ ## Value: Port | IP:Port ## ## Examples: 3306, 127.0.0.1:3306, localhost:3306 -auth.mysql.server = 127.0.0.1:3306 +auth.mysql.server = "127.0.0.1:3306" ## MySQL pool size. ## @@ -50,7 +50,7 @@ auth.mysql.database = mqtt ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.mysql.auth_query = select password from mqtt_user where username = '%u' limit 1 +auth.mysql.auth_query = "select password from mqtt_user where username = '%u' limit 1" ## auth.mysql.auth_query = select password_hash as password from mqtt_user where username = '%u' limit 1 ## Password hash. @@ -59,17 +59,17 @@ auth.mysql.auth_query = select password from mqtt_user where username = '%u' lim auth.mysql.password_hash = sha256 ## sha256 with salt prefix -## auth.mysql.password_hash = salt,sha256 +## auth.mysql.password_hash = "salt,sha256" ## bcrypt with salt only prefix -## auth.mysql.password_hash = salt,bcrypt +## auth.mysql.password_hash = "salt,bcrypt" ## sha256 with salt suffix -## auth.mysql.password_hash = sha256,salt +## auth.mysql.password_hash = "sha256,salt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.mysql.password_hash = pbkdf2,sha256,1000,20 +## auth.mysql.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query. ## @@ -81,7 +81,7 @@ auth.mysql.password_hash = sha256 ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.mysql.super_query = select is_superuser from mqtt_user where username = '%u' limit 1 +auth.mysql.super_query = "select is_superuser from mqtt_user where username = '%u' limit 1" ## ACL query. ## @@ -93,12 +93,12 @@ auth.mysql.super_query = select is_superuser from mqtt_user where username = '%u ## - %c: clientid ## ## Note: You can add the 'ORDER BY' statement to control the rules match order -auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c' +auth.mysql.acl_query = "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" ## Mysql ssl configuration. ## ## Value: on | off -#auth.mysql.ssl = off +## auth.mysql.ssl.enable = off ## CA certificate. ## diff --git a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema index 5e4619301..fba25f41f 100644 --- a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema +++ b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema @@ -30,7 +30,7 @@ {datatype, string} ]}. -{mapping, "auth.mysql.ssl", "emqx_auth_mysql.server", [ +{mapping, "auth.mysql.ssl.enable", "emqx_auth_mysql.server", [ {default, off}, {datatype, flag} ]}. @@ -94,7 +94,7 @@ {keep_alive, true}], Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, Options1 = - case cuttlefish:conf_get("auth.mysql.ssl", Conf) of + case cuttlefish:conf_get("auth.mysql.ssl.enable", Conf) of true -> %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 CA = cuttlefish:conf_get( diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 221061f03..8a0d116cc 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/README.md b/apps/emqx_auth_pgsql/README.md index 2dccd6f53..a8f5d723f 100644 --- a/apps/emqx_auth_pgsql/README.md +++ b/apps/emqx_auth_pgsql/README.md @@ -49,7 +49,7 @@ auth.pgsql.encoding = utf8 ## Whether to enable SSL connection. ## ## Value: true | false -auth.pgsql.ssl = false +auth.pgsql.ssl.enable = false ## SSL keyfile. ## diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index e39d0c78a..6f7018210 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -6,8 +6,8 @@ ## ## Value: Port | IP:Port ## -## Examples: 5432, 127.0.0.1:5432, localhost:5432 -auth.pgsql.server = 127.0.0.1:5432 +## Examples: 5432, "127.0.0.1:5432", "localhost:5432" +auth.pgsql.server = "127.0.0.1:5432" ## PostgreSQL pool size. ## @@ -37,7 +37,7 @@ auth.pgsql.encoding = utf8 ## Whether to enable SSL connection. ## ## Value: on | off -auth.pgsql.ssl = off +auth.pgsql.ssl.enable = off ## TLS version. ## @@ -87,7 +87,7 @@ auth.pgsql.ssl = off ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.pgsql.auth_query = select password from mqtt_user where username = '%u' limit 1 +auth.pgsql.auth_query = "select password from mqtt_user where username = '%u' limit 1" ## Password hash. ## @@ -95,17 +95,17 @@ auth.pgsql.auth_query = select password from mqtt_user where username = '%u' lim auth.pgsql.password_hash = sha256 ## sha256 with salt prefix -## auth.pgsql.password_hash = salt,sha256 +## auth.pgsql.password_hash = "salt,sha256" ## sha256 with salt suffix -## auth.pgsql.password_hash = sha256,salt +## auth.pgsql.password_hash = "sha256,salt" ## bcrypt with salt prefix -## auth.pgsql.password_hash = salt,bcrypt +## auth.pgsql.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.pgsql.password_hash = pbkdf2,sha256,1000,20 +## auth.pgsql.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query. ## @@ -117,7 +117,7 @@ auth.pgsql.password_hash = sha256 ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.pgsql.super_query = select is_superuser from mqtt_user where username = '%u' limit 1 +auth.pgsql.super_query = "select is_superuser from mqtt_user where username = '%u' limit 1" ## ACL query. Comment this query, the ACL will be disabled. ## @@ -129,4 +129,4 @@ auth.pgsql.super_query = select is_superuser from mqtt_user where username = '%u ## - %c: clientid ## ## Note: You can add the 'ORDER BY' statement to control the rules match order -auth.pgsql.acl_query = select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c' +auth.pgsql.acl_query = "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index f78bcc597..2be9b0670 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -30,7 +30,7 @@ {datatype, atom} ]}. -{mapping, "auth.pgsql.ssl", "emqx_auth_pgsql.server", [ +{mapping, "auth.pgsql.ssl.enable", "emqx_auth_pgsql.server", [ {default, off}, {datatype, {enum, [on, off, true, false]}} %% FIXME: true/fasle is compatible with 4.0-4.2 version format, plan to delete in 5.0 ]}. @@ -116,8 +116,14 @@ end, %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 - Ssl = case cuttlefish:conf_get("auth.pgsql.ssl", Conf) of - on -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl")}]; + GenSsl = case cuttlefish:conf_get("auth.pgsql.ssl.cacertfile", Conf, undefined) of + undefined -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl_opts")}]; + _ -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl")}] + end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + Ssl = case cuttlefish:conf_get("auth.pgsql.ssl.enable", Conf) of + on -> GenSsl; off -> []; true -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl_opts")}]; false -> [] diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index f70612262..e97487e21 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 5a56c5dce..62a6e4fe1 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -12,9 +12,9 @@ auth.redis.type = single ## Value: Port | IP:Port ## ## Single Redis Server: 127.0.0.1:6379, localhost:6379 -## Redis Sentinel: 127.0.0.1:26379,127.0.0.2:26379,127.0.0.3:26379 -## Redis Cluster: 127.0.0.1:6379,127.0.0.2:6379,127.0.0.3:6379 -auth.redis.server = 127.0.0.1:6379 +## Redis Sentinel: "127.0.0.1:26379,127.0.0.2:26379,127.0.0.3:26379" +## Redis Cluster: "127.0.0.1:6379,127.0.0.2:6379,127.0.0.3:6379" +auth.redis.server = "127.0.0.1:6379" ## Redis sentinel cluster name. ## @@ -52,10 +52,10 @@ auth.redis.database = 0 ## - %d: subject of client TLS cert ## ## Examples: -## - HGET mqtt_user:%u password -## - HMGET mqtt_user:%u password -## - HMGET mqtt_user:%u password salt -auth.redis.auth_cmd = HMGET mqtt_user:%u password +## - "HGET mqtt_user:%u password" +## - "HMGET mqtt_user:%u password" +## - "HMGET mqtt_user:%u password salt" +auth.redis.auth_cmd = "HMGET mqtt_user:%u password" ## Password hash. ## @@ -63,17 +63,17 @@ auth.redis.auth_cmd = HMGET mqtt_user:%u password auth.redis.password_hash = plain ## sha256 with salt prefix -## auth.redis.password_hash = salt,sha256 +## auth.redis.password_hash = "salt,sha256" ## sha256 with salt suffix -## auth.redis.password_hash = sha256,salt +## auth.redis.password_hash = "sha256,salt" ## bcrypt with salt prefix -## auth.redis.password_hash = salt,bcrypt +## auth.redis.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.redis.password_hash = pbkdf2,sha256,1000,20 +## auth.redis.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query command. ## @@ -84,7 +84,7 @@ auth.redis.password_hash = plain ## - %c: clientid ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert -auth.redis.super_cmd = HGET mqtt_user:%u is_superuser +auth.redis.super_cmd = "HGET mqtt_user:%u is_superuser" ## ACL query command. ## @@ -93,12 +93,12 @@ auth.redis.super_cmd = HGET mqtt_user:%u is_superuser ## Variables: ## - %u: username ## - %c: clientid -auth.redis.acl_cmd = HGETALL mqtt_acl:%u +auth.redis.acl_cmd = "HGETALL mqtt_acl:%u" ## Redis ssl configuration. ## ## Value: on | off -#auth.redis.ssl = off +# auth.redis.ssl.enable = off ## CA certificate. ## @@ -108,12 +108,12 @@ auth.redis.acl_cmd = HGETALL mqtt_acl:%u ## Client ssl certificate. ## ## Value: File -#auth.redis.ssl.certfile = path/to/your/certfile +# auth.redis.ssl.certfile = path/to/your/certfile ## Client ssl keyfile. ## ## Value: File -#auth.redis.ssl.keyfile = path/to/your/keyfile +# auth.redis.ssl.keyfile = path/to/your/keyfile ## In mode verify_none the default behavior is to allow all x509-path ## validation errors. diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index 3ce34d597..db758d8c0 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -33,7 +33,7 @@ hidden ]}. -{mapping, "auth.redis.ssl", "emqx_auth_redis.options", [ +{mapping, "auth.redis.ssl.enable", "emqx_auth_redis.options", [ {default, off}, {datatype, flag} ]}. @@ -75,7 +75,7 @@ ]}. {translation, "emqx_auth_redis.options", fun(Conf) -> - Ssl = cuttlefish:conf_get("auth.redis.ssl", Conf, false), + Ssl = cuttlefish:conf_get("auth.redis.ssl.enable", Conf, false), Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, case Ssl of true -> diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 6a38af2ce..419131566 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md index 456fae584..812645627 100644 --- a/apps/emqx_bridge_mqtt/README.md +++ b/apps/emqx_bridge_mqtt/README.md @@ -53,13 +53,13 @@ The following is the basic configuration of RPC bridging. A simplest RPC bridgin ``` ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection -bridge.mqtt.emqx2.address = emqx2@192.168.1.2 +bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" ## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# +bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## bridged mountpoint -bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ +bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ``` If the messages received by the local node emqx1 matches the topic `sersor1/#` or `sensor2/#`, these messages will be forwarded to the `sensor1/#` or `sensor2/#` topic of the remote node emqx2. @@ -82,66 +82,66 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co ``` ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection -bridge.mqtt.emqx2.address = 192.168.1.2:1883 +bridge.mqtt.emqx2.address = "192.168.1.2:1883" ## Bridged Protocol Version ## Enumeration value: mqttv3 | mqttv4 | mqttv5 -bridge.mqtt.emqx2.proto_ver = mqttv4 +bridge.mqtt.emqx2.proto_ver = "mqttv4" ## mqtt client's clientid -bridge.mqtt.emqx2.clientid = bridge_emq +bridge.mqtt.emqx2.clientid = "bridge_emq" ## mqtt client's clean_start field ## Note: Some MQTT Brokers need to set the clean_start value as `true` bridge.mqtt.emqx2.clean_start = true ## mqtt client's username field -bridge.mqtt.emqx2.username = user +bridge.mqtt.emqx2.username = "user" ## mqtt client's password field -bridge.mqtt.emqx2.password = passwd +bridge.mqtt.emqx2.password = "passwd" ## Whether the mqtt client uses ssl to connect to a remote serve or not bridge.mqtt.emqx2.ssl = off ## CA Certificate of Client SSL Connection (PEM format) -bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem +bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" ## SSL certificate of Client SSL connection -bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem +bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" ## Key file of Client SSL connection -bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem +bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" ## SSL encryption -bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 +bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" ## TTLS PSK password ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time ## ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Client's heartbeat interval bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version -bridge.mqtt.emqx2.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +bridge.mqtt.emqx2.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# +bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## Bridged mountpoint -bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ +bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1 +bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.1.qos = 1 ## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2 +bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.2.qos = 1 diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst index 47235aa7d..a1c2a9126 100644 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ b/apps/emqx_bridge_mqtt/docs/guide.rst @@ -39,7 +39,7 @@ In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguish .. code-block:: ## Bridge address: node name for local bridge, host:port for remote. - bridge.mqtt.aws.address = 127.0.0.1:1883 + bridge.mqtt.aws.address = "127.0.0.1:1883" This configuration declares a bridge named ``aws`` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. @@ -69,13 +69,13 @@ The following is the basic configuration of RPC bridging. A simplest RPC bridgin .. code-block:: ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection - bridge.mqtt.emqx2.address = emqx2@192.168.1.2 + bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# + bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## bridged mountpoint - bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ + bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" If the messages received by the local node emqx1 matches the topic ``sersor1/#`` or ``sensor2/#``\ , these messages will be forwarded to the ``sensor1/#`` or ``sensor2/#`` topic of the remote node emqx2. @@ -86,10 +86,10 @@ If the messages received by the local node emqx1 matches the topic ``sersor1/#`` Limitations of RPC bridging: -#. +#. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; -#. +#. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. EMQ X MQTT Bridge Configuration @@ -102,66 +102,66 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co .. code-block:: ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection - bridge.mqtt.emqx2.address = 192.168.1.2:1883 + bridge.mqtt.emqx2.address = "192.168.1.2:1883" ## Bridged Protocol Version ## Enumeration value: mqttv3 | mqttv4 | mqttv5 - bridge.mqtt.emqx2.proto_ver = mqttv4 + bridge.mqtt.emqx2.proto_ver = "mqttv4" ## mqtt client's clientid - bridge.mqtt.emqx2.clientid = bridge_emq + bridge.mqtt.emqx2.clientid = "bridge_emq" ## mqtt client's clean_start field ## Note: Some MQTT Brokers need to set the clean_start value as `true` bridge.mqtt.emqx2.clean_start = true ## mqtt client's username field - bridge.mqtt.emqx2.username = user + bridge.mqtt.emqx2.username = "user" ## mqtt client's password field - bridge.mqtt.emqx2.password = passwd + bridge.mqtt.emqx2.password = "passwd" ## Whether the mqtt client uses ssl to connect to a remote serve or not bridge.mqtt.emqx2.ssl = off ## CA Certificate of Client SSL Connection (PEM format) - bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem + bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" ## SSL certificate of Client SSL connection - bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem + bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" ## Key file of Client SSL connection - bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem + bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" ## TTLS PSK password ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time ## ## See 'https://tools.ietf.org/html/rfc4279#section-2'. - ## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA + ## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Client's heartbeat interval bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version - bridge.mqtt.emqx2.tls_versions = tlsv1.2 + bridge.mqtt.emqx2.tls_versions = "tlsv1.2" ## SSL encryption - bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# + bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## Bridged mountpoint - bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ + bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1 + bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.1.qos = 1 ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2 + bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.2.qos = 1 @@ -190,7 +190,7 @@ The bridge of EMQ X has a message caching mechanism. The caching mechanism is ap bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB ## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. - bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/ + bridge.mqtt.emqx2.queue.replayq_dir = "data/emqx_emqx2_bridge/" ## Replayq data segment size bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index db59270c0..3cc719e40 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -9,8 +9,8 @@ ## Bridge address: node name for local bridge, host:port for remote. ## ## Value: String -## Example: emqx@127.0.0.1, 127.0.0.1:1883 -bridge.mqtt.aws.address = 127.0.0.1:1883 +## Example: emqx@127.0.0.1, "127.0.0.1:1883" +bridge.mqtt.aws.address = "127.0.0.1:1883" ## Protocol version of the bridge. ## @@ -65,18 +65,18 @@ bridge.mqtt.aws.password = passwd ## Topics that need to be forward to AWS IoTHUB ## ## Value: String -## Example: topic1/#,topic2/# -bridge.mqtt.aws.forwards = topic1/#,topic2/# +## Example: "topic1/#,topic2/#" +bridge.mqtt.aws.forwards = "topic1/#,topic2/#" ## Forward messages to the mountpoint of an AWS IoTHUB ## ## Value: String -bridge.mqtt.aws.forward_mountpoint = bridge/aws/${node}/ +bridge.mqtt.aws.forward_mountpoint = "bridge/aws/${node}/" ## Need to subscribe to AWS topics ## ## Value: String -## bridge.mqtt.aws.subscription.1.topic = cmd/topic1 +## bridge.mqtt.aws.subscription.1.topic = "cmd/topic1" ## Need to subscribe to AWS topics QoS. ## @@ -86,7 +86,7 @@ bridge.mqtt.aws.forward_mountpoint = bridge/aws/${node}/ ## A mountpoint that receives messages from AWS IoTHUB ## ## Value: String -## bridge.mqtt.aws.receive_mountpoint = receive/aws/ +## bridge.mqtt.aws.receive_mountpoint = "receive/aws/" ## Bribge to remote server via SSL. @@ -97,28 +97,28 @@ bridge.mqtt.aws.ssl = off ## PEM-encoded CA certificates of the bridge. ## ## Value: File -bridge.mqtt.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +bridge.mqtt.aws.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Client SSL Certfile of the bridge. ## ## Value: File -bridge.mqtt.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem +bridge.mqtt.aws.certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" ## Client SSL Keyfile of the bridge. ## ## Value: File -bridge.mqtt.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +bridge.mqtt.aws.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ## SSL Ciphers used by the bridge. ## ## Value: String -bridge.mqtt.aws.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +bridge.mqtt.aws.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'bridge.${BridgeName}.ciphers' and 'bridge.${BridgeName}.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#bridge.mqtt.aws.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#bridge.mqtt.aws.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Ping interval of a down bridge. ## @@ -130,7 +130,7 @@ bridge.mqtt.aws.keepalive = 60s ## ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier ## Value: String -bridge.mqtt.aws.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +bridge.mqtt.aws.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## Bridge reconnect time. ## @@ -160,7 +160,7 @@ bridge.mqtt.aws.max_inflight_size = 32 ## replayq works in a mem-only manner. ## ## Value: String -bridge.mqtt.aws.queue.replayq_dir = {{ platform_data_dir }}/replayq/emqx_aws_bridge/ +bridge.mqtt.aws.queue.replayq_dir = "{{ platform_data_dir }}/replayq/emqx_aws_bridge/" ## Replayq segment size ## diff --git a/apps/emqx_coap/etc/emqx_coap.conf b/apps/emqx_coap/etc/emqx_coap.conf index 0590a348e..d6dfa1a6f 100644 --- a/apps/emqx_coap/etc/emqx_coap.conf +++ b/apps/emqx_coap/etc/emqx_coap.conf @@ -4,13 +4,13 @@ ## The IP and UDP port that CoAP bind with. ## -## Default: 0.0.0.0:5683 +## Default: "0.0.0.0:5683" ## ## Examples: -## coap.bind.udp.x = 0.0.0.0:5683 | :::5683 | 127.0.0.1:5683 | ::1:5683 +## coap.bind.udp.x = "0.0.0.0:5683" | ":::5683" | "127.0.0.1:5683" | "::1:5683" ## -coap.bind.udp.1 = 0.0.0.0:5683 -##coap.bind.udp.2 = 0.0.0.0:6683 +coap.bind.udp.1 = "0.0.0.0:5683" +##coap.bind.udp.2 = "0.0.0.0:6683" ## Whether to enable statistics for CoAP clients. ## @@ -23,13 +23,13 @@ coap.enable_stats = off ## The DTLS port that CoAP is listening on. ## -## Default: 0.0.0.0:5684 +## Default: "0.0.0.0:5684" ## ## Examples: -## coap.bind.dtls.x = 0.0.0.0:5684 | :::5684 | 127.0.0.1:5684 | ::1:5684 +## coap.bind.dtls.x = "0.0.0.0:5684" | ":::5684" | "127.0.0.1:5684" | "::1:5684" ## -coap.bind.dtls.1 = 0.0.0.0:5684 -##coap.bind.dtls.2 = 0.0.0.0:6684 +coap.bind.dtls.1 = "0.0.0.0:5684" +##coap.bind.dtls.2 = "0.0.0.0:6684" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -43,17 +43,17 @@ coap.bind.dtls.1 = 0.0.0.0:5684 ## Private key file for DTLS ## ## Value: File -coap.dtls.keyfile = {{ platform_etc_dir }}/certs/key.pem +coap.dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Server certificate for DTLS. ## ## Value: File -coap.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem +coap.dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## PEM-encoded CA certificates for DTLS ## ## Value: File -## coap.dtls.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## coap.dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Used together with {verify, verify_peer} by an SSL server. If set to true, ## the server fails if the client does not have a certificate to send, that is, @@ -79,4 +79,4 @@ coap.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -coap.dtls.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +coap.dtls.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md new file mode 100644 index 000000000..879669f93 --- /dev/null +++ b/apps/emqx_connector/README.md @@ -0,0 +1,27 @@ +# emqx_connector + +This application is a collection of `connectors`. + +A `connector` is a callback module of `emqx_resource` that maintains the data related to +external resources. Put all resource related callback modules in a single application is good as +we can put some util functions/modules here for reusing purpose. + +For example, a mysql connector is an emqx resource that maintains all the mysql connection +related parameters (configs) and the TCP connections to the mysql server. + +An mysql connector can be used as following: + +``` +(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). +[#{config => + #{auto_reconnect => true,cacertfile => [],certfile => [], + database => "mqtt",keyfile => [],password => "public", + pool_size => 1, + server => {{127,0,0,1},3306}, + ssl => false,user => "root",verify => false}, + id => <<"mysql-abc">>,mod => emqx_connector_mysql, + state => #{poolname => 'mysql-abc'}, + status => started}] +(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf new file mode 100644 index 000000000..db4402d47 --- /dev/null +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -0,0 +1,4 @@ +##-------------------------------------------------------------------- +## EMQ X CONNECTOR Plugin +##-------------------------------------------------------------------- + diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema new file mode 100644 index 000000000..b8476c4d9 --- /dev/null +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config new file mode 100644 index 000000000..53b5c63e8 --- /dev/null +++ b/apps/emqx_connector/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_connector]} +]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src new file mode 100644 index 000000000..a821b8f13 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -0,0 +1,17 @@ +{application, emqx_connector, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_connector_app, []}}, + {applications, + [kernel, + stdlib, + emqx_resource, + ecpool + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl new file mode 100644 index 000000000..dd0359348 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl new file mode 100644 index 000000000..4bbad75cf --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl new file mode 100644 index 000000000..6fbbe11c6 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -0,0 +1,106 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_mysql). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +-export([ schema/0 + ]). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + , on_jsonify/1 + ]). + +-export([connect/1]). + +-export([do_health_check/1]). + +%%===================================================================== +schema() -> + emqx_connector_schema_lib:relational_db_fields() ++ + emqx_connector_schema_lib:ssl_fields(). + +on_jsonify(#{<<"server">> := Server, <<"user">> := User, <<"database">> := DB, + <<"password">> := Passwd, <<"cacertfile">> := CAFile, + <<"keyfile">> := KeyFile, <<"certfile">> := CertFile} = Config) -> + Config#{ + <<"user">> => list_to_binary(User), + <<"database">> => list_to_binary(DB), + <<"password">> => list_to_binary(Passwd), + <<"server">> => emqx_connector_schema_lib:ip_port_to_string(Server), + <<"cacertfile">> => list_to_binary(CAFile), + <<"keyfile">> => list_to_binary(KeyFile), + <<"certfile">> => list_to_binary(CertFile) + }. + +%% =================================================================== +on_start(InstId, #{<<"server">> := {Host, Port}, + <<"database">> := DB, + <<"user">> := User, + <<"password">> := Password, + <<"auto_reconnect">> := AutoReconn, + <<"pool_size">> := PoolSize} = Config) -> + logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]), + SslOpts = case maps:get(<<"ssl">>, Config) of + true -> + [{ssl, [{server_name_indication, disable} | + emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}]; + false -> + [] + end, + Options = [{host, Host}, + {port, Port}, + {user, User}, + {password, Password}, + {database, DB}, + {auto_reconnect, reconn_interval(AutoReconn)}, + {pool_size, PoolSize}], + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts), + {ok, #{poolname => PoolName}}. + +on_stop(InstId, #{poolname := PoolName}) -> + logger:info("stopping mysql connector: ~p", [InstId]), + emqx_plugin_libs_pool:stop_pool(PoolName). + +on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), + case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL]}, no_handover) of + {error, Reason} -> + logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), + emqx_resource:query_failed(AfterQuery); + _ -> + emqx_resource:query_success(AfterQuery) + end, + Result. + +on_health_check(_InstId, #{poolname := PoolName} = State) -> + emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State). + +do_health_check(Conn) -> + ok == element(1, mysql:query(Conn, <<"SELECT count(1) AS T">>)). + +%% =================================================================== +reconn_interval(true) -> 15; +reconn_interval(false) -> false. + +connect(Options) -> + mysql:start_link(Options). diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl new file mode 100644 index 000000000..02bed956b --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -0,0 +1,111 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_schema_lib). +-include_lib("typerefl/include/types.hrl"). + +-export([ relational_db_fields/0 + , ssl_fields/0 + ]). + +-export([ to_ip_port/1 + , ip_port_to_string/1 + ]). + +-typerefl_from_string({ip_port/0, emqx_connector_schema_lib, to_ip_port}). + +-reflect_type([ip_port/0]). + +-type ip_port() :: tuple(). + +-define(VALID, emqx_resource_validator). +-define(REQUIRED(MSG), ?VALID:required(MSG)). +-define(MAX(MAXV), ?VALID:max(number, MAXV)). +-define(MIN(MINV), ?VALID:min(number, MINV)). + +relational_db_fields() -> + [ {server, fun server/1} + , {database, fun database/1} + , {pool_size, fun pool_size/1} + , {user, fun user/1} + , {password, fun password/1} + , {auto_reconnect, fun auto_reconnect/1} + ]. + +ssl_fields() -> + [ {ssl, fun ssl/1} + , {cacertfile, fun cacertfile/1} + , {keyfile, fun keyfile/1} + , {certfile, fun certfile/1} + , {verify, fun verify/1} + ]. + +server(type) -> ip_port(); +server(validator) -> [?REQUIRED("the field 'server' is required")]; +server(_) -> undefined. + +database(type) -> string(); +database(validator) -> [?REQUIRED("the field 'server' is required")]; +database(_) -> undefined. + +pool_size(type) -> integer(); +pool_size(default) -> 8; +pool_size(validator) -> [?MIN(1), ?MAX(64)]; +pool_size(_) -> undefined. + +user(type) -> string(); +user(default) -> "root"; +user(_) -> undefined. + +password(type) -> string(); +password(default) -> ""; +password(_) -> undefined. + +auto_reconnect(type) -> boolean(); +auto_reconnect(default) -> true; +auto_reconnect(_) -> undefined. + +ssl(type) -> boolean(); +ssl(default) -> false; +ssl(_) -> undefined. + +cacertfile(type) -> string(); +cacertfile(default) -> ""; +cacertfile(_) -> undefined. + +keyfile(type) -> string(); +keyfile(default) -> ""; +keyfile(_) -> undefined. + +certfile(type) -> string(); +certfile(default) -> ""; +certfile(_) -> undefined. + +verify(type) -> boolean(); +verify(default) -> false; +verify(_) -> undefined. + +to_ip_port(Str) -> + case string:tokens(Str, ":") of + [Ip, Port] -> + case inet:parse_address(Ip) of + {ok, R} -> {ok, {R, list_to_integer(Port)}}; + _ -> {error, Str} + end; + _ -> {error, Str} + end. + +ip_port_to_string({Ip, Port}) -> + iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]). diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl new file mode 100644 index 000000000..603b9a8ad --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_data_bridge/.gitignore b/apps/emqx_data_bridge/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_data_bridge/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_data_bridge/README.md b/apps/emqx_data_bridge/README.md new file mode 100644 index 000000000..8f76f17a5 --- /dev/null +++ b/apps/emqx_data_bridge/README.md @@ -0,0 +1,10 @@ +# emqx_data_bridge + +EMQ X Data Bridge is an application that managing the resources (see emqx_resource) used by emqx +rule engine. + +It provides CRUD HTTP APIs of the resources, and is also responsible for loading the resources at +startup, and saving configs of resources to `data/` after configs updated. + +The application depends on `emqx_connector` as that's where all the callback modules of `connector` +resources placed. diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf new file mode 100644 index 000000000..295173e6c --- /dev/null +++ b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf @@ -0,0 +1,30 @@ +##-------------------------------------------------------------------- +## EMQ X Bridge Plugin +##-------------------------------------------------------------------- + +emqx_data_bridge.bridges: [ + {name: "mysql-abc" + type: mysql + config: { + server: "127.0.0.1:3306" + database: mqtt + pool_size: 1 + user: root + password: public + auto_reconnect: true + ssl: false + } + }, + {name: "mysql-def" + type: mysql + config: { + server: "127.0.0.1:3306" + database: mqtt + pool_size: 1 + user: root + password: public + auto_reconnect: true + ssl: false + } + } +] diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema new file mode 100644 index 000000000..c9cb3f2c0 --- /dev/null +++ b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema @@ -0,0 +1,16 @@ +%%-*- mode: erlang -*- +%% emqx_data_bridge config mapping + +{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ + {default, []}, + {datatype, string} +]}. + +% fields("emqx_data_bridge") -> +% [ +% {bridges, +% [fun(mapping) -> "emqx_data_bridge.bridges"; +% (type) -> list(); +% (_) -> undefined +% end]} +% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config new file mode 100644 index 000000000..cf4cfcf1b --- /dev/null +++ b/apps/emqx_data_bridge/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_data_bridge]} +]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src new file mode 100644 index 000000000..360511d9b --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -0,0 +1,15 @@ +{application, emqx_data_bridge, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_data_bridge_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl new file mode 100644 index 000000000..2adc48111 --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge). + +-export([ load_bridges/0 + , resource_type/1 + , bridge_type/1 + , name_to_resource_id/1 + , resource_id_to_name/1 + , list_bridges/0 + , is_bridge/1 + ]). + +load_bridges() -> + Bridges = proplists:get_value(bridges, + application:get_all_env(emqx_data_bridge), []), + emqx_data_bridge_monitor:ensure_all_started(Bridges). + +resource_type(<<"mysql">>) -> emqx_connector_mysql. + +bridge_type(emqx_connector_mysql) -> <<"mysql">>. + +name_to_resource_id(BridgeName) -> + <<"bridge:", BridgeName/binary>>. + +resource_id_to_name(<<"bridge:", BridgeName/binary>> = _ResourceId) -> + BridgeName. + +list_bridges() -> + emqx_resource_api:list_instances(fun emqx_data_bridge:is_bridge/1). + +is_bridge(#{id := <<"bridge:", _/binary>>}) -> + true; +is_bridge(_Data) -> + false. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl new file mode 100644 index 000000000..1a27351eb --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -0,0 +1,114 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_api). + +-rest_api(#{ name => list_data_bridges + , method => 'GET' + , path => "/data_bridges" + , func => list_bridges + , descr => "List all data bridges" + }). + +-rest_api(#{ name => get_data_bridge + , method => 'GET' + , path => "/data_bridges/:bin:name" + , func => get_bridge + , descr => "Get a data bridge by name" + }). + +-rest_api(#{ name => create_data_bridge + , method => 'POST' + , path => "/data_bridges/:bin:name" + , func => create_bridge + , descr => "Create a new data bridge" + }). + +-rest_api(#{ name => update_data_bridge + , method => 'PUT' + , path => "/data_bridges/:bin:name" + , func => update_bridge + , descr => "Update an existing data bridge" + }). + +-rest_api(#{ name => delete_data_bridge + , method => 'DELETE' + , path => "/data_bridges/:bin:name" + , func => delete_bridge + , descr => "Delete an existing data bridge" + }). + +-export([ list_bridges/2 + , get_bridge/2 + , create_bridge/2 + , update_bridge/2 + , delete_bridge/2 + ]). + +list_bridges(_Binding, _Params) -> + {200, #{code => 0, data => [format_api_reply(Data) || + Data <- emqx_data_bridge:list_bridges()]}}. + +get_bridge(#{name := Name}, _Params) -> + case emqx_resource:get_instance(emqx_data_bridge:name_to_resource_id(Name)) of + {ok, Data} -> + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; + {error, not_found} -> + {404, #{code => 102, message => <<"not_found: ", Name/binary>>}} + end. + +create_bridge(#{name := Name}, Params) -> + Config = proplists:get_value(<<"config">>, Params), + BridgeType = proplists:get_value(<<"type">>, Params), + case emqx_resource:check_and_create( + emqx_data_bridge:name_to_resource_id(Name), + emqx_data_bridge:resource_type(BridgeType), Config) of + {ok, Data} -> + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; + {error, already_created} -> + {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; + {error, Reason0} -> + Reason = emqx_resource_api:stringnify(Reason0), + {500, #{code => 102, message => <<"create bridge ", Name/binary, + " failed:", Reason/binary>>}} + end. + +update_bridge(#{name := Name}, Params) -> + Config = proplists:get_value(<<"config">>, Params), + BridgeType = proplists:get_value(<<"type">>, Params), + case emqx_resource:check_and_update( + emqx_data_bridge:name_to_resource_id(Name), + emqx_data_bridge:resource_type(BridgeType), Config, []) of + {ok, Data} -> + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; + {error, not_found} -> + {400, #{code => 102, message => <<"bridge not_found: ", Name/binary>>}}; + {error, Reason0} -> + Reason = emqx_resource_api:stringnify(Reason0), + {500, #{code => 102, message => <<"update bridge ", Name/binary, + " failed:", Reason/binary>>}} + end. + +delete_bridge(#{name := Name}, _Params) -> + case emqx_resource:remove(emqx_data_bridge:name_to_resource_id(Name)) of + ok -> {200, #{code => 0, data => #{}}}; + {error, Reason} -> + {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} + end. + +format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> + #{type => emqx_data_bridge:bridge_type(Type), + name => emqx_data_bridge:resource_id_to_name(Id), + config => Conf, status => Status}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl new file mode 100644 index 000000000..4ff96e34e --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_data_bridge_sup:start_link(), + ok = emqx_data_bridge:load_bridges(), + {ok, Sup}. + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl new file mode 100644 index 000000000..ed21b4ce3 --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_config_handler). + +-behaviour(gen_server). + +%% API functions +-export([ start_link/0 + , notify_updated/0 + ]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% TODO: trigger the `updated` message from emqx_resource. +notify_updated() -> + gen_server:cast(?MODULE, updated). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +%% TODO: change the config handler as a behavoir that calls back the Mod:format_config/1 +handle_cast(updated, State) -> + Configs = [format_conf(Data) || Data <- emqx_data_bridge:list_bridges()], + emqx_config_handler ! {emqx_data_bridge, Configs}, + {noreply, State}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%============================================================================ + +format_conf(#{resource_type := Type, id := Id, config := Conf}) -> + #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), + config => Conf}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl new file mode 100644 index 000000000..521d9721c --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl @@ -0,0 +1,81 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +%% This process monitors all the data bridges, and try to restart a bridge +%% when one of it stopped. +-module(emqx_data_bridge_monitor). + +-behaviour(gen_server). + +%% API functions +-export([ start_link/0 + , ensure_all_started/1 + ]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +ensure_all_started(Configs) -> + gen_server:cast(?MODULE, {start_and_monitor, Configs}). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast({start_and_monitor, Configs}, State) -> + ok = load_bridges(Configs), + {noreply, State}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%============================================================================ +load_bridges(Configs) -> + lists:foreach(fun load_bridge/1, Configs). + +%% TODO: move this monitor into emqx_resource +%% emqx_resource:check_and_create_local(ResourceId, ResourceType, Config, #{keep_retry => true}). +load_bridge(#{<<"name">> := Name, <<"type">> := Type, + <<"config">> := Config}) -> + case emqx_resource:check_and_create_local( + emqx_data_bridge:name_to_resource_id(Name), + emqx_data_bridge:resource_type(Type), Config) of + {ok, _} -> ok; + {error, already_created} -> ok; + {error, Reason} -> + error({load_bridge, Reason}) + end. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl new file mode 100644 index 000000000..516d9b5ab --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl @@ -0,0 +1,41 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 10}, + ChildSpecs = [ + #{id => emqx_data_bridge_monitor, + start => {emqx_data_bridge_monitor, start_link, []}, + restart => permanent, + type => worker, + modules => [emqx_data_bridge_monitor]}], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index f6f5213f7..b2758e705 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -8,8 +8,8 @@ ## The gRPC server url ## ## exhook.server.$name.url = url() -exhook.server.default.url = http://127.0.0.1:9000 +exhook.server.default.url = "http://127.0.0.1:9000" -#exhook.server.default.ssl.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -#exhook.server.default.ssl.certfile = {{ platform_etc_dir }}/certs/cert.pem -#exhook.server.default.ssl.keyfile = {{ platform_etc_dir }}/certs/key.pem +#exhook.server.default.ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +#exhook.server.default.ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" +#exhook.server.default.ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 883aad9bd..eafa20d85 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -43,7 +43,6 @@ {profiles, [{test, [{deps, - [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}} - ]} + []} ]} ]}. diff --git a/apps/emqx_exproto/etc/emqx_exproto.conf b/apps/emqx_exproto/etc/emqx_exproto.conf index 713685734..7a7667271 100644 --- a/apps/emqx_exproto/etc/emqx_exproto.conf +++ b/apps/emqx_exproto/etc/emqx_exproto.conf @@ -5,9 +5,9 @@ exproto.server.http.port = 9100 exproto.server.https.port = 9101 -exproto.server.https.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -exproto.server.https.certfile = {{ platform_etc_dir }}/certs/cert.pem -exproto.server.https.keyfile = {{ platform_etc_dir }}/certs/key.pem +exproto.server.https.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +exproto.server.https.certfile = "{{ platform_etc_dir }}/certs/cert.pem" +exproto.server.https.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ##-------------------------------------------------------------------- ## Listeners @@ -20,12 +20,12 @@ exproto.server.https.keyfile = {{ platform_etc_dir }}/certs/key.pem ## ## Value: ://: ## -## Examples: tcp://0.0.0.0:7993 | ssl://127.0.0.1:7994 -exproto.listener.protoname = tcp://0.0.0.0:7993 +## Examples: "tcp://0.0.0.0:7993" | "ssl://127.0.0.1:7994" +exproto.listener.protoname.endpoint = "tcp://0.0.0.0:7993" ## The ConnectionHandler server address ## -exproto.listener.protoname.connection_handler_url = http://127.0.0.1:9001 +exproto.listener.protoname.connection_handler_url = "http://127.0.0.1:9001" #exproto.listener.protoname.connection_handler_certfile = #exproto.listener.protoname.connection_handler_cacertfile = @@ -62,8 +62,8 @@ exproto.listener.protoname.idle_timeout = 30s ## ## Value: ACL Rule ## -## Example: allow 192.168.0.0/24 -exproto.listener.protoname.access.1 = allow all +## Example: "allow 192.168.0.0/24" +exproto.listener.protoname.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed ## behind HAProxy or Nginx. @@ -146,27 +146,27 @@ exproto.listener.protoname.reuseaddr = true ## See: http://erlang.org/doc/man/ssl.html ## ## Value: String, seperated by ',' -#exproto.listener.protoname.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +#exproto.listener.protoname.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## Path to the file containing the user's private PEM-encoded key. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -#exproto.listener.protoname.keyfile = {{ platform_etc_dir }}/certs/key.pem +#exproto.listener.protoname.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -#exproto.listener.protoname.certfile = {{ platform_etc_dir }}/certs/cert.pem +#exproto.listener.protoname.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -#exproto.listener.protoname.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +#exproto.listener.protoname.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit @@ -183,7 +183,7 @@ exproto.listener.protoname.reuseaddr = true ## openssl dhparam -out dh-params.pem 2048 ## ## Value: File -#exproto.listener.protoname.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +#exproto.listener.protoname.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -218,13 +218,13 @@ exproto.listener.protoname.reuseaddr = true ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -#exproto.listener.protoname.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +#exproto.listener.protoname.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#exproto.listener.protoname.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#exproto.listener.protoname.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## SSL parameter renegotiation is a feature that allows a client and a server ## to renegotiate the parameters of the SSL connection on the fly. diff --git a/apps/emqx_exproto/priv/emqx_exproto.schema b/apps/emqx_exproto/priv/emqx_exproto.schema index fb114dc77..4bd215847 100644 --- a/apps/emqx_exproto/priv/emqx_exproto.schema +++ b/apps/emqx_exproto/priv/emqx_exproto.schema @@ -44,7 +44,7 @@ end}. %%-------------------------------------------------------------------- %% Listeners -{mapping, "exproto.listener.$proto", "emqx_exproto.listeners", [ +{mapping, "exproto.listener.$proto.endpoint", "emqx_exproto.listeners", [ {datatype, string} ]}. @@ -340,7 +340,7 @@ end}. Listeners = fun(Proto) -> Prefix = string:join(["exproto","listener", Proto], "."), Opts = HandlerOpts(Prefix) ++ ConnOpts(Prefix) ++ LisOpts(Prefix), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".endpoint", Conf, undefined) of undefined -> []; ListenOn0 -> case ParseListenOn(ListenOn0) of @@ -359,6 +359,6 @@ end}. end end end, - lists:flatten([Listeners(Proto) || {[_, "listener", Proto], ListenOn} + lists:flatten([Listeners(Proto) || {[_, "listener", Proto, "endpoint"], ListenOn} <- cuttlefish_variable:filter_by_prefix("exproto.listener", Conf)]) end}. diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 61677f0e3..3fa9f6f8a 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -46,7 +46,6 @@ {profiles, [{test, [{deps, - [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.0"}}} - ]} + []} ]} ]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 52ce96a54..9841f5bcb 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.0"}, %% strict semver + {vsn, "4.4.0"}, %% strict semver {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf index 968b8fd19..b244fa385 100644 --- a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf +++ b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf @@ -25,25 +25,25 @@ lwm2m.lifetime_max = 86400s # Placeholders supported: # '%e': Endpoint Name # '%a': IP Address -lwm2m.mountpoint = lwm2m/%e/ +lwm2m.mountpoint = "lwm2m/%e/" # The topic subscribed by the lwm2m client after it is connected # Placeholders supported: # '%e': Endpoint Name # '%a': IP Address -lwm2m.topics.command = dn/# +lwm2m.topics.command = "dn/#" # The topic to which the lwm2m client's response is published -lwm2m.topics.response = up/resp +lwm2m.topics.response = "up/resp" # The topic to which the lwm2m client's notify message is published -lwm2m.topics.notify = up/notify +lwm2m.topics.notify = "up/notify" # The topic to which the lwm2m client's register message is published -lwm2m.topics.register = up/resp +lwm2m.topics.register = "up/resp" # The topic to which the lwm2m client's update message is published -lwm2m.topics.update = up/resp +lwm2m.topics.update = "up/resp" # When publish the update message. # @@ -55,18 +55,18 @@ lwm2m.topics.update = up/resp #lwm2m.update_msg_publish_condition = contains_object_list # Dir where the object definition files can be found -lwm2m.xml_dir = {{ platform_etc_dir }}/lwm2m_xml +lwm2m.xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" ##-------------------------------------------------------------------- ## UDP Listener options ## The IP and port of the LwM2M Gateway ## -## Default: 0.0.0.0:5683 +## Default: "0.0.0.0:5683" ## Examples: -## lwm2m.bind.udp.x = 0.0.0.0:5683 | :::5683 | 127.0.0.1:5683 | ::1:5683 -lwm2m.bind.udp.1 = 0.0.0.0:5683 -#lwm2m.bind.udp.2 = 0.0.0.0:6683 +## lwm2m.bind.udp.x = "0.0.0.0:5683" | ":::5683" | "127.0.0.1:5683" | "::1:5683" +lwm2m.bind.udp.1 = "0.0.0.0:5683" +#lwm2m.bind.udp.2 = "0.0.0.0:6683" ## Socket options, used for performance tuning ## @@ -83,13 +83,13 @@ lwm2m.opts.read_packets = 20 ## The DTLS port that LwM2M is listening on. ## -## Default: 0.0.0.0:5684 +## Default: "0.0.0.0:5684" ## ## Examples: -## lwm2m.bind.dtls.x = 0.0.0.0:5684 | :::5684 | 127.0.0.1:5684 | ::1:5684 +## lwm2m.bind.dtls.x = "0.0.0.0:5684" | ":::5684" | "127.0.0.1:5684" | "::1:5684" ## -lwm2m.bind.dtls.1 = 0.0.0.0:5684 -#lwm2m.bind.dtls.2 = 0.0.0.0:6684 +lwm2m.bind.dtls.1 = "0.0.0.0:5684" +#lwm2m.bind.dtls.2 = "0.0.0.0:6684" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -103,17 +103,17 @@ lwm2m.bind.dtls.1 = 0.0.0.0:5684 ## Private key file for DTLS ## ## Value: File -lwm2m.dtls.keyfile = {{ platform_etc_dir }}/certs/key.pem +lwm2m.dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Server certificate for DTLS. ## ## Value: File -lwm2m.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem +lwm2m.dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## PEM-encoded CA certificates for DTLS ## ## Value: File -#lwm2m.dtls.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +#lwm2m.dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Used together with {verify, verify_peer} by an SSL server. If set to true, ## the server fails if the client does not have a certificate to send, that is, @@ -139,11 +139,11 @@ lwm2m.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -lwm2m.dtls.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +lwm2m.dtls.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## ## Note that 'lwm2m.dtls.ciphers' and 'lwm2m.dtls.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#lwm2m.dtls.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#lwm2m.dtls.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index b9dea9bb8..fbfcdc02f 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -5,7 +5,6 @@ {profiles, [{test, [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, - {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} ]} ]} diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index aa737add9..05a3008fa 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -23,7 +23,7 @@ management.default_application.secret = public ##-------------------------------------------------------------------- ## HTTP Listener -management.listener.http = 8081 +management.listener.http.port = 8081 management.listener.http.acceptors = 2 management.listener.http.max_clients = 512 management.listener.http.backlog = 512 @@ -35,19 +35,19 @@ management.listener.http.ipv6_v6only = false ##-------------------------------------------------------------------- ## HTTPS Listener -## management.listener.https = 8081 +## management.listener.https.port = 8081 ## management.listener.https.acceptors = 2 ## management.listener.https.max_clients = 512 ## management.listener.https.backlog = 512 ## management.listener.https.send_timeout = 15s ## management.listener.https.send_timeout_close = on -## management.listener.https.certfile = etc/certs/cert.pem -## management.listener.https.keyfile = etc/certs/key.pem -## management.listener.https.cacertfile = etc/certs/cacert.pem +## management.listener.https.certfile = "etc/certs/cert.pem" +## management.listener.https.keyfile = "etc/certs/key.pem" +## management.listener.https.cacertfile = "etc/certs/cacert.pem" ## management.listener.https.verify = verify_peer ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## management.listener.https.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 -## management.listener.https.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## management.listener.https.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" +## management.listener.https.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## management.listener.https.fail_if_no_peer_cert = true ## management.listener.https.inet6 = false ## management.listener.https.ipv6_v6only = false diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema index 343a70de6..4e887809e 100644 --- a/apps/emqx_management/priv/emqx_management.schema +++ b/apps/emqx_management/priv/emqx_management.schema @@ -21,7 +21,7 @@ {datatype, string} ]}. -{mapping, "management.listener.http", "emqx_management.listeners", [ +{mapping, "management.listener.http.port", "emqx_management.listeners", [ {datatype, [integer, ip]} ]}. @@ -85,7 +85,7 @@ {datatype, {enum, [true, false]}} ]}. -{mapping, "management.listener.https", "emqx_management.listeners", [ +{mapping, "management.listener.https.port", "emqx_management.listeners", [ {datatype, [integer, ip]} ]}. @@ -225,7 +225,7 @@ end}. lists:foldl( fun(Proto, Acc) -> Prefix = "management.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of undefined -> Acc; Port -> [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 43fb5ba53..fe68fef44 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index bae633045..5667cb73f 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -181,4 +181,4 @@ remove_resources() -> lists:foreach(fun(#resource{id = Id}) -> emqx_rule_engine:delete_resource(Id) end, emqx_rule_registry:get_resources()), - timer:sleep(500). \ No newline at end of file + timer:sleep(500). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 2749edc15..d4d284e69 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -563,7 +563,10 @@ t_data_import_content(_) -> Dir = emqx:get_env(data_dir), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), - ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + %% TODO: enable when 5.0 if we are still using data export/import + %?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + ?assertMatch({ok, "{\"message\":\"5.0\",\"code\":\"unsupported_version\"}"}, + request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), application:stop(emqx_rule_engine), application:stop(emqx_dahboard). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl new file mode 100644 index 000000000..dddc44f4e --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugin_libs_pool). + +-export([ start_pool/3 + , stop_pool/1 + , pool_name/1 + , health_check/3 + ]). + +pool_name(ID) when is_binary(ID) -> + list_to_atom(binary_to_list(ID)). + +start_pool(Name, Mod, Options) -> + case ecpool:start_sup_pool(Name, Mod, Options) of + {ok, _} -> logger:log(info, "Initiated ~0p Successfully", [Name]); + {error, {already_started, _Pid}} -> + stop_pool(Name), + start_pool(Name, Mod, Options); + {error, Reason} -> + logger:log(error, "Initiate ~0p failed ~0p", [Name, Reason]), + error({start_pool_failed, Name}) + end. + +stop_pool(Name) -> + case ecpool:stop_sup_pool(Name) of + ok -> logger:log(info, "Destroyed ~0p Successfully", [Name]); + {error, not_found} -> ok; + {error, Reason} -> + logger:log(error, "Destroy ~0p failed, ~0p", [Name, Reason]), + error({stop_pool_failed, Name}) + end. + +health_check(PoolName, CheckFunc, State) when is_function(CheckFunc) -> + Status = [begin + case ecpool_worker:client(Worker) of + {ok, Conn} -> CheckFunc(Conn); + _ -> false + end + end || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + case length(Status) > 0 andalso lists:all(fun(St) -> St =:= true end, Status) of + true -> {ok, State}; + false -> {error, test_query_failed} + end. diff --git a/apps/emqx_prometheus/etc/emqx_prometheus.conf b/apps/emqx_prometheus/etc/emqx_prometheus.conf index 7bfa22095..92e0d850d 100644 --- a/apps/emqx_prometheus/etc/emqx_prometheus.conf +++ b/apps/emqx_prometheus/etc/emqx_prometheus.conf @@ -5,7 +5,7 @@ ## The Prometheus Push Gateway URL address ## ## Note: You can comment out this line to disable it -prometheus.push.gateway.server = http://127.0.0.1:9091 +prometheus.push.gateway.server = "http://127.0.0.1:9091" ## The metrics data push interval (millisecond) ## diff --git a/apps/emqx_psk_file/etc/emqx_psk_file.conf b/apps/emqx_psk_file/etc/emqx_psk_file.conf index 3cee1c926..88c5bbdb1 100644 --- a/apps/emqx_psk_file/etc/emqx_psk_file.conf +++ b/apps/emqx_psk_file/etc/emqx_psk_file.conf @@ -1,2 +1,2 @@ -psk.file.path = {{ platform_etc_dir }}/psk.txt -psk.file.delimiter = : \ No newline at end of file +psk.file.path = "{{ platform_etc_dir }}/psk.txt" +psk.file.delimiter = ":" diff --git a/apps/emqx_recon/src/emqx_recon.app.src b/apps/emqx_recon/src/emqx_recon.app.src index 061ed8e93..e25e2bbc5 100644 --- a/apps/emqx_recon/src/emqx_recon.app.src +++ b/apps/emqx_recon/src/emqx_recon.app.src @@ -1,6 +1,6 @@ {application, emqx_recon, [{description, "EMQ X Recon Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,recon]}, diff --git a/apps/emqx_resource/Makefile b/apps/emqx_resource/Makefile new file mode 100644 index 000000000..596b9b2a1 --- /dev/null +++ b/apps/emqx_resource/Makefile @@ -0,0 +1,43 @@ +REBAR := rebar3 + +.PHONY: all +all: es + +.PHONY: compile +compile: + $(REBAR) compile + +.PHONY: clean +clean: distclean + +.PHONY: distclean +distclean: + @rm -rf _build erl_crash.dump rebar3.crashdump + +.PHONY: xref +xref: + $(REBAR) xref + +.PHONY: eunit +eunit: compile + $(REBAR) eunit -v -c + $(REBAR) cover + +.PHONY: ct +ct: compile + $(REBAR) as test ct -v + +cover: + $(REBAR) cover + +.PHONY: dialyzer +dialyzer: + $(REBAR) dialyzer + +.PHONY: es +es: compile + $(REBAR) escriptize + +.PHONY: elvis +elvis: + ./scripts/elvis-check.sh diff --git a/apps/emqx_resource/README.md b/apps/emqx_resource/README.md new file mode 100644 index 000000000..77e1b607b --- /dev/null +++ b/apps/emqx_resource/README.md @@ -0,0 +1,48 @@ +# emqx_resource + +The `emqx_resource` is a behavior that manages configuration specs and runtime states +for resources like mysql or redis backends. + +It is intended to be used by the emqx_data_bridges and all other resources that need CRUD operations +to their configs, and need to initialize the states when creating. + +There can be foreign references between resource instances via resource-id. +So they may find each other via this Id. + +The main idea of the emqx resource is to put all the `general` code in a common lib, including +the config operations (like config validation, config dump back to files), and the state management. +And we put all the `specific` codes to the callback modules. + +## Try it out + + $ ./demo.sh + Eshell V11.1.8 (abort with ^G) + 1> == the demo log tracer <<"log_tracer_clientid_shawn">> started. + config: #{<<"config">> => + #{<<"bulk">> => <<"10KB">>,<<"cache_log_dir">> => <<"/tmp">>, + <<"condition">> => #{<<"clientid">> => <<"abc">>}, + <<"level">> => <<"debug">>}, + <<"id">> => <<"log_tracer_clientid_shawn">>, + <<"resource_type">> => <<"log_tracer">>} + 1> emqx_resource_instance:health_check(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> is working well + state: #{health_checked => 1,logger_handler_id => abc} + ok + + 2> emqx_resource_instance:health_check(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> is working well + state: #{health_checked => 2,logger_handler_id => abc} + ok + + 3> emqx_resource_instance:query(<<"log_tracer_clientid_shawn">>, get_log). + == the demo log tracer <<"log_tracer_clientid_shawn">> received request: get_log + state: #{health_checked => 2,logger_handler_id => abc} + "this is a demo log messages..." + + 4> emqx_resource_instance:remove(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> stopped. + state: #{health_checked => 0,logger_handler_id => abc} + ok + + 5> emqx_resource_instance:query(<<"log_tracer_clientid_shawn">>, get_log). + ** exception error: {get_instance,{<<"log_tracer_clientid_shawn">>,not_found}} diff --git a/apps/emqx_resource/demo.sh b/apps/emqx_resource/demo.sh new file mode 100755 index 000000000..19cbab809 --- /dev/null +++ b/apps/emqx_resource/demo.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e + +rebar3 compile + +erl -sname abc -pa _build/default/lib/*/ebin _build/default/lib/emqx_resource/examples -s demo diff --git a/apps/emqx_resource/elvis.config b/apps/emqx_resource/elvis.config new file mode 100644 index 000000000..59aa13fbe --- /dev/null +++ b/apps/emqx_resource/elvis.config @@ -0,0 +1,14 @@ +[{elvis, [{config, [ + +#{dirs => ["src"], + filter => "*.erl", + %ignore => [], + ruleset => erl_files, + rules => [{elvis_style, operator_spaces, #{ + rules => [{right, ","}, + {right, "|"}, + {left, "|"}, + {right, "||"}, + {left, "||"}]}}, + {elvis_style, god_modules, #{limit => 100}}]} +]}]}]. diff --git a/apps/emqx_resource/etc/emqx_resource.conf b/apps/emqx_resource/etc/emqx_resource.conf new file mode 100644 index 000000000..1f038ef8b --- /dev/null +++ b/apps/emqx_resource/etc/emqx_resource.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQ X Resource Plugin +##-------------------------------------------------------------------- diff --git a/apps/emqx_resource/examples/demo.erl b/apps/emqx_resource/examples/demo.erl new file mode 100644 index 000000000..171a80b61 --- /dev/null +++ b/apps/emqx_resource/examples/demo.erl @@ -0,0 +1,13 @@ +-module(demo). + +-export([start/0]). + +start() -> + code:load_file(log_tracer), + code:load_file(log_tracer_schema), + {ok, _} = application:ensure_all_started(minirest), + {ok, _} = application:ensure_all_started(emqx_resource), + emqx_resource:load_instances("./_build/default/lib/emqx_resource/examples"), + Handlers = [{"/", minirest:handler(#{modules => [log_tracer]})}], + Dispatch = [{"/[...]", minirest, Handlers}], + minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 9900}]}, Dispatch). diff --git a/apps/emqx_resource/examples/demo.md b/apps/emqx_resource/examples/demo.md new file mode 100644 index 000000000..c8842578d --- /dev/null +++ b/apps/emqx_resource/examples/demo.md @@ -0,0 +1,154 @@ +--- +theme: gaia +color: #000 +colorSecondary: #333 +backgroundColor: #fff +backgroundImage: url('https://marp.app/assets/hero-background.jpg') +paginate: true +marp: true +--- + + + +# EMQ X Resource + +--- + +## What is it for + +The [emqx_resource](https://github.com/emqx/emqx/tree/master/apps/emqx_resource) is a behavior that manages configuration specs and runtime states for resources like mysql or redis backends. + +It is intended to be used by the emqx_data_bridges and all other resources that need CRUD operations to their configs, and need to initialize the states when creating. + +--- + + + +# The Demo + +The data_bridge for mysql + +--- +## The callback module 'emqx_mysql_connector' + +1. include the emqx_resource_behaviour.hrl: +``` +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +``` +--- +2. provide the hocon schema for validating the configs: +``` +schema() -> + emqx_connector_schema_lib:relational_db_fields() ++ + emqx_connector_schema_lib:ssl_fields(). +... +``` + +--- +3. write the callback functions for starting or stopping the resource instance: + +``` +on_start/2, +on_stop/2, +on_query/4, +on_health_check/2 + +``` +--- +## Start the emqx_data_bridge + +``` +application:ensure_all_started(emqx_data_bridge). +``` + +--- + +## To use the mysql resource from code: + +``` +emqx_resource:query(ResourceID, {sql, SQL}). +``` + +``` +(emqx@127.0.0.1)2> emqx_resource:list_instances_verbose(). +[#{config => + #{<<"auto_reconnect">> => true,<<"cacertfile">> => [], + <<"certfile">> => [],<<"database">> => "mqtt", + <<"keyfile">> => [],<<"password">> => "public", + <<"pool_size">> => 1, + <<"server">> => {{127,0,0,1},3306}, + <<"ssl">> => false,<<"user">> => "root", + <<"verify">> => false}, + id => <<"bridge:mysql-def">>,mod => emqx_connector_mysql, + state => #{poolname => 'bridge:mysql-def'}, + status => started}] + +(emqx@127.0.0.1)3> emqx_resource:query(<<"bridge:mysql-def">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` + +--- + +## To get all available data bridges: + +``` +curl -q --basic -u admin:public -X GET "http://localhost:8081/api/v4/data_bridges/" | jq . +``` + +--- + +## Create + +To create a mysql data bridge: + +``` +BridgeMySQL='{ + "type": "mysql", + "status": "started", + "name": "mysql-def", + "config": { + "verify": false, + "user": "root", + "ssl": false, + "server": "127.0.0.1:3306", + "pool_size": 1, + "password": "public", + "keyfile": "", + "database": "mqtt", + "certfile": "", + "cacertfile": "", + "auto_reconnect": true + } + }' + +curl -q --basic -u admin:public -X POST "http://localhost:8081/api/v4/data_bridges/mysql-aaaa" -d $BridgeMySQL | jq . +``` + +--- + +## Update + +To update an existing data bridge: + +``` +BridgeMySQL='{ + "type": "mysql", + "status": "started", + "name": "mysql-def", + "config": { + "verify": false, + "user": "root", + "ssl": false, + "server": "127.0.0.1:3306", + "pool_size": 2, + "password": "public", + "keyfile": "", + "database": "mqtt", + "certfile": "", + "cacertfile": "", + "auto_reconnect": true + } + }' + +curl -q --basic -u admin:public -X PUT "http://localhost:8081/api/v4/data_bridges/mysql-aaaa" -d $BridgeMySQL | jq . +``` diff --git a/apps/emqx_resource/examples/log_tracer.conf b/apps/emqx_resource/examples/log_tracer.conf new file mode 100644 index 000000000..7b438ec1f --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer.conf @@ -0,0 +1,11 @@ +{ + "id": "log_tracer_clientid_shawn" + "resource_type": "log_tracer" + "config": { + "condition": {"app": "emqx"} + "level": "debug" + "cache_log_dir": "/tmp" + "bulk": "10KB" + "chars_limit": 1024 + } +} \ No newline at end of file diff --git a/apps/emqx_resource/examples/log_tracer.erl b/apps/emqx_resource/examples/log_tracer.erl new file mode 100644 index 000000000..ed3a1a84c --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer.erl @@ -0,0 +1,43 @@ +-module(log_tracer). + +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + , on_api_reply_format/1 + , on_config_merge/3 + ]). + +%% callbacks for emqx_resource config schema +-export([schema/0]). + +schema() -> + log_tracer_schema:schema(). + +on_start(InstId, Config) -> + io:format("== the demo log tracer ~p started.~nconfig: ~p~n", [InstId, Config]), + {ok, #{logger_handler_id => abc, health_checked => 0}}. + +on_stop(InstId, State) -> + io:format("== the demo log tracer ~p stopped.~nstate: ~p~n", [InstId, State]), + ok. + +on_query(InstId, Request, AfterQuery, State) -> + io:format("== the demo log tracer ~p received request: ~p~nstate: ~p~n", + [InstId, Request, State]), + emqx_resource:query_success(AfterQuery), + "this is a demo log messages...". + +on_health_check(InstId, State = #{health_checked := Checked}) -> + NState = State#{health_checked => Checked + 1}, + io:format("== the demo log tracer ~p is working well~nstate: ~p~n", [InstId, NState]), + {ok, NState}. + +on_api_reply_format(#{id := Id, status := Status, state := #{health_checked := NChecked}}) -> + #{id => Id, status => Status, checked_count => NChecked}. + +on_config_merge(OldConfig, NewConfig, _Params) -> + maps:merge(OldConfig, NewConfig). diff --git a/apps/emqx_resource/examples/log_tracer_schema.erl b/apps/emqx_resource/examples/log_tracer_schema.erl new file mode 100644 index 000000000..a8fc55411 --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer_schema.erl @@ -0,0 +1,44 @@ +-module(log_tracer_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([schema/0]). + +-reflect_type([t_level/0, t_cache_logs_in/0]). + +-type t_level() :: debug | info | notice | warning | error | critical | alert | emergency. + +-type t_cache_logs_in() :: memory | file. + +schema() -> + [ {condition, fun condition/1} + , {level, fun level/1} + , {enable_cache, fun enable_cache/1} + , {cache_logs_in, fun cache_logs_in/1} + , {cache_log_dir, fun cache_log_dir/1} + , {bulk, fun bulk/1} + ]. + +condition(mapping) -> "config.condition"; +condition(type) -> map(); +condition(_) -> undefined. + +level(mapping) -> "config.level"; +level(type) -> t_level(); +level(_) -> undefined. + +enable_cache(mapping) -> "config.enable_cache"; +enable_cache(type) -> boolean(); +enable_cache(_) -> undefined. + +cache_logs_in(mapping) -> "config.cache_logs_in"; +cache_logs_in(type) -> t_cache_logs_in(); +cache_logs_in(_) -> undefined. + +cache_log_dir(mapping) -> "config.cache_log_dir"; +cache_log_dir(type) -> typerefl:regexp_string("^(.*)$"); +cache_log_dir(_) -> undefined. + +bulk(mapping) -> "config.bulk"; +bulk(type) -> typerefl:regexp_string("^[. 0-9]+(B|KB|MB|GB)$"); +bulk(_) -> undefined. diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl new file mode 100644 index 000000000..123854bc9 --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -0,0 +1,34 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-type resource_type() :: module(). +-type instance_id() :: binary(). +-type resource_config() :: term(). +-type resource_spec() :: map(). +-type resource_state() :: term(). +-type resource_data() :: #{ + id => instance_id(), + mod => module(), + config => resource_config(), + state => resource_state(), + status => started | stopped +}. + +-type after_query() :: {OnSuccess :: after_query_fun(), OnFailed :: after_query_fun()} | + undefined. + +%% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback +%% actions upon query failure +-type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}. diff --git a/apps/emqx_resource/include/emqx_resource_behaviour.hrl b/apps/emqx_resource/include/emqx_resource_behaviour.hrl new file mode 100644 index 000000000..bb4f18b55 --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource_behaviour.hrl @@ -0,0 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-behaviour(emqx_resource). +-compile({parse_transform, emqx_resource_transform}). diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl new file mode 100644 index 000000000..a20a17e89 --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource_utils.hrl @@ -0,0 +1,53 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-define(CLUSTER_CALL(Func, Args), ?CLUSTER_CALL(Func, Args, ok)). + +-define(CLUSTER_CALL(Func, Args, ResParttern), +%% ekka_mnesia:running_nodes() + fun() -> + case LocalResult = erlang:apply(?MODULE, Func, Args) of + ResParttern -> + case rpc:multicall(nodes(), ?MODULE, Func, Args, 5000) of + {ResL, []} -> + Filter = fun + (ResParttern) -> false; + ({badrpc, {'EXIT', {undef, [{?MODULE, Func0, _, []}]}}}) + when Func0 =:= Func -> false; + (_) -> true + end, + case lists:filter(Filter, ResL) of + [] -> LocalResult; + ErrL -> {error, ErrL} + end; + {ResL, BadNodes} -> + {error, {failed_on_nodes, BadNodes, ResL}} + end; + ErrorResult -> + {error, ErrorResult} + end + end()). + +-define(SAFE_CALL(_EXP_), + ?SAFE_CALL(_EXP_, _ = do_nothing)). + +-define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_), + fun() -> + try (_EXP_) + catch _EXCLASS_:_EXCPTION_:_ST_ -> + _EXP_ON_FAIL_, + {error, {_EXCLASS_, _EXCPTION_, _ST_}} + end + end()). \ No newline at end of file diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema new file mode 100644 index 000000000..8246dc6a7 --- /dev/null +++ b/apps/emqx_resource/priv/emqx_resource.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx-resource config mapping diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config new file mode 100644 index 000000000..4e88043de --- /dev/null +++ b/apps/emqx_resource/rebar.config @@ -0,0 +1,17 @@ +{erl_opts, [ debug_info + , nowarn_unused_import + %, {d, 'RESOURCE_DEBUG'} + ]}. + +{erl_first_files, ["src/emqx_resource_transform.erl"]}. + +{extra_src_dirs, ["examples"]}. + +%% try to override the dialyzer 'race_conditions' defined in the top-level dir, +%% but it doesn't work +{dialyzer, [{warnings, [unmatched_returns, error_handling]} + ]}. + +{deps, [ {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} + ]}. + diff --git a/apps/emqx_resource/scripts/elvis-check.sh b/apps/emqx_resource/scripts/elvis-check.sh new file mode 100755 index 000000000..3fae0f191 --- /dev/null +++ b/apps/emqx_resource/scripts/elvis-check.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +ELVIS_VERSION='1.0.0-emqx-2' + +elvis_version="${2:-$ELVIS_VERSION}" + +echo "elvis -v: $elvis_version" + +if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx-[0-9]+')" != "$elvis_version" ]; then + curl -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis" + chmod +x ./elvis +fi + +./elvis rock --config elvis.config + diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src new file mode 100644 index 000000000..13330b061 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -0,0 +1,18 @@ +{application, emqx_resource, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_resource_app, []}}, + {applications, + [kernel, + stdlib, + gproc, + hocon, + jsx + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl new file mode 100644 index 000000000..5a9b21408 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -0,0 +1,338 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource). + +-include("emqx_resource.hrl"). +-include("emqx_resource_utils.hrl"). + +%% APIs for resource types + +-export([ get_type/1 + , list_types/0 + , list_types_verbose/0 + ]). + +-export([ discover_resource_mods/0 + , is_resource_mod/1 + , call_instance/2 + ]). + +-export([ query_success/1 + , query_failed/1 + ]). + +%% APIs for instances + +-export([ check_config/2 + , check_and_create/3 + , check_and_create_local/3 + , check_and_update/4 + , check_and_update_local/4 + , resource_type_from_str/1 + ]). + +%% Sync resource instances and files +%% provisional solution: rpc:multical to all the nodes for creating/updating/removing +%% todo: replicate operations +-export([ create/3 %% store the config and start the instance + , create_local/3 + , create_dry_run/3 %% run start/2, health_check/2 and stop/1 sequentially + , create_dry_run_local/3 + , update/4 %% update the config, stop the old instance and start the new one + , update_local/4 + , remove/1 %% remove the config and stop the instance + , remove_local/1 + ]). + +%% Calls to the callback module with current resource state +%% They also save the state after the call finished (except query/2,3). +-export([ restart/1 %% restart the instance. + , health_check/1 %% verify if the resource is working normally + , stop/1 %% stop the instance + , query/2 %% query the instance + , query/3 %% query the instance with after_query() + ]). + +%% Direct calls to the callback module +-export([ call_start/3 %% start the instance + , call_health_check/3 %% verify if the resource is working normally + , call_stop/3 %% stop the instance + , call_config_merge/4 %% merge the config when updating + , call_jsonify/2 + ]). + +-export([ list_instances/0 %% list all the instances, id only. + , list_instances_verbose/0 %% list all the instances + , get_instance/1 %% return the data of the instance + , list_instances_by_type/1 %% return all the instances of the same resource type + % , dependents/1 + % , inc_counter/2 %% increment the counter of the instance + % , inc_counter/3 %% increment the counter by a given integer + ]). + +-define(EXT, "*.spec"). + +-optional_callbacks([ on_query/4 + , on_health_check/2 + , on_config_merge/3 + , on_jsonify/1 + , on_api_reply_format/1 + ]). + +-callback on_api_reply_format(resource_data()) -> jsx:json_term(). + +-callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). + +-callback on_jsonify(resource_config()) -> jsx:json_term(). + +%% when calling emqx_resource:start/1 +-callback on_start(instance_id(), resource_config()) -> + {ok, resource_state()} | {error, Reason :: term()}. + +%% when calling emqx_resource:stop/1 +-callback on_stop(instance_id(), resource_state()) -> term(). + +%% when calling emqx_resource:query/3 +-callback on_query(instance_id(), Request :: term(), after_query(), resource_state()) -> term(). + +%% when calling emqx_resource:health_check/2 +-callback on_health_check(instance_id(), resource_state()) -> + {ok, resource_state()} | {error, Reason:: term(), resource_state()}. + +%% load specs and return the loaded resources this time. +-spec list_types_verbose() -> [resource_spec()]. +list_types_verbose() -> + [get_spec(Mod) || Mod <- list_types()]. + +-spec list_types() -> [module()]. +list_types() -> + discover_resource_mods(). + +-spec get_type(module()) -> {ok, resource_spec()} | {error, not_found}. +get_type(Mod) -> + case is_resource_mod(Mod) of + true -> {ok, get_spec(Mod)}; + false -> {error, not_found} + end. + +-spec get_spec(module()) -> resource_spec(). +get_spec(Mod) -> + maps:put(<<"resource_type">>, Mod, Mod:emqx_resource_schema()). + +-spec discover_resource_mods() -> [module()]. +discover_resource_mods() -> + [Mod || {Mod, _} <- code:all_loaded(), is_resource_mod(Mod)]. + +-spec is_resource_mod(module()) -> boolean(). +is_resource_mod(Mod) -> + erlang:function_exported(Mod, emqx_resource_schema, 0). + +-spec query_success(after_query()) -> ok. +query_success(undefined) -> ok; +query_success({{OnSucc, Args}, _}) -> + safe_apply(OnSucc, Args). + +-spec query_failed(after_query()) -> ok. +query_failed(undefined) -> ok; +query_failed({_, {OnFailed, Args}}) -> + safe_apply(OnFailed, Args). + +%% ================================================================================= +%% APIs for resource instances +%% ================================================================================= +-spec create(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +create(InstId, ResourceType, Config) -> + ?CLUSTER_CALL(create_local, [InstId, ResourceType, Config], {ok, _}). + +-spec create_local(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +create_local(InstId, ResourceType, Config) -> + call_instance(InstId, {create, InstId, ResourceType, Config}). + +-spec create_dry_run(instance_id(), resource_type(), resource_config()) -> + ok | {error, Reason :: term()}. +create_dry_run(InstId, ResourceType, Config) -> + ?CLUSTER_CALL(create_dry_run_local, [InstId, ResourceType, Config]). + +-spec create_dry_run_local(instance_id(), resource_type(), resource_config()) -> + ok | {error, Reason :: term()}. +create_dry_run_local(InstId, ResourceType, Config) -> + call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). + +-spec update(instance_id(), resource_type(), resource_config(), term()) -> + {ok, resource_data()} | {error, Reason :: term()}. +update(InstId, ResourceType, Config, Params) -> + ?CLUSTER_CALL(update_local, [InstId, ResourceType, Config, Params], {ok, _}). + +-spec update_local(instance_id(), resource_type(), resource_config(), term()) -> + {ok, resource_data()} | {error, Reason :: term()}. +update_local(InstId, ResourceType, Config, Params) -> + call_instance(InstId, {update, InstId, ResourceType, Config, Params}). + +-spec remove(instance_id()) -> ok | {error, Reason :: term()}. +remove(InstId) -> + ?CLUSTER_CALL(remove_local, [InstId]). + +-spec remove_local(instance_id()) -> ok | {error, Reason :: term()}. +remove_local(InstId) -> + call_instance(InstId, {remove, InstId}). + +%% ================================================================================= +-spec query(instance_id(), Request :: term()) -> Result :: term(). +query(InstId, Request) -> + query(InstId, Request, undefined). + +%% same to above, also defines what to do when the Module:on_query success or failed +%% it is the duty of the Moudle to apply the `after_query()` functions. +-spec query(instance_id(), Request :: term(), after_query()) -> Result :: term(). +query(InstId, Request, AfterQuery) -> + case get_instance(InstId) of + {ok, #{mod := Mod, state := ResourceState}} -> + %% the resource state is readonly to Moudle:on_query/4 + %% and the `after_query()` functions should be thread safe + Mod:on_query(InstId, Request, AfterQuery, ResourceState); + {error, Reason} -> + error({get_instance, {InstId, Reason}}) + end. + +-spec restart(instance_id()) -> ok | {error, Reason :: term()}. +restart(InstId) -> + call_instance(InstId, {restart, InstId}). + +-spec stop(instance_id()) -> ok | {error, Reason :: term()}. +stop(InstId) -> + call_instance(InstId, {stop, InstId}). + +-spec health_check(instance_id()) -> ok | {error, Reason :: term()}. +health_check(InstId) -> + call_instance(InstId, {health_check, InstId}). + +-spec get_instance(instance_id()) -> {ok, resource_data()} | {error, Reason :: term()}. +get_instance(InstId) -> + emqx_resource_instance:lookup(InstId). + +-spec list_instances() -> [instance_id()]. +list_instances() -> + [Id || #{id := Id} <- list_instances_verbose()]. + +-spec list_instances_verbose() -> [resource_data()]. +list_instances_verbose() -> + emqx_resource_instance:list_all(). + +-spec list_instances_by_type(module()) -> [resource_data()]. +list_instances_by_type(ResourceType) -> + emqx_resource_instance:lookup_by_type(ResourceType). + +-spec call_start(instance_id(), module(), resource_config()) -> + {ok, resource_state()} | {error, Reason :: term()}. +call_start(InstId, Mod, Config) -> + ?SAFE_CALL(Mod:on_start(InstId, Config)). + +-spec call_health_check(instance_id(), module(), resource_state()) -> + {ok, resource_state()} | {error, Reason:: term(), resource_state()}. +call_health_check(InstId, Mod, ResourceState) -> + ?SAFE_CALL(Mod:on_health_check(InstId, ResourceState)). + +-spec call_stop(instance_id(), module(), resource_state()) -> term(). +call_stop(InstId, Mod, ResourceState) -> + ?SAFE_CALL(Mod:on_stop(InstId, ResourceState)). + +-spec call_config_merge(module(), resource_config(), resource_config(), term()) -> + resource_config(). +call_config_merge(Mod, OldConfig, NewConfig, Params) -> + case erlang:function_exported(Mod, on_config_merge, 3) of + true -> ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)); + false -> NewConfig + end. + +-spec call_jsonify(module(), resource_config()) -> jsx:json_term(). +call_jsonify(Mod, Config) -> + case erlang:function_exported(Mod, on_jsonify, 1) of + false -> Config; + true -> ?SAFE_CALL(Mod:on_jsonify(Config)) + end. + +-spec check_config(resource_type(), binary() | term()) -> + {ok, resource_config()} | {error, term()}. +check_config(ResourceType, RawConfig) when is_binary(RawConfig) -> + case hocon:binary(RawConfig, #{format => richmap}) of + {ok, MapConfig} -> + do_check_config(ResourceType, MapConfig); + Error -> Error + end; +check_config(ResourceType, RawConfigTerm) -> + check_config(ResourceType, jsx:encode(#{config => RawConfigTerm})). + +-spec do_check_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. +do_check_config(ResourceType, MapConfig) -> + case ?SAFE_CALL(emqx_resource_schema:check(ResourceType, MapConfig)) of + {error, Reason} -> {error, Reason}; + Config -> {ok, maps:get(<<"config">>, hocon_schema:richmap_to_map(Config))} + end. + +-spec check_and_create(instance_id(), resource_type(), binary() | term()) -> + {ok, resource_data()} | {error, term()}. +check_and_create(InstId, ResourceType, Config) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> create(InstId, ResourceType, InstConf) end). + +-spec check_and_create_local(instance_id(), resource_type(), binary() | term()) -> + {ok, resource_data()} | {error, term()}. +check_and_create_local(InstId, ResourceType, Config) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> create_local(InstId, ResourceType, InstConf) end). + +-spec check_and_update(instance_id(), resource_type(), binary() | term(), term()) -> + {ok, resource_data()} | {error, term()}. +check_and_update(InstId, ResourceType, Config, Params) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> update(InstId, ResourceType, InstConf, Params) end). + +-spec check_and_update_local(instance_id(), resource_type(), binary() | term(), term()) -> + {ok, resource_data()} | {error, term()}. +check_and_update_local(InstId, ResourceType, Config, Params) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> update_local(InstId, ResourceType, InstConf, Params) end). + +check_and_do(ResourceType, Config, Do) when is_function(Do) -> + case check_config(ResourceType, Config) of + {ok, InstConf} -> Do(InstConf); + Error -> Error + end. + +%% ================================================================================= + +-spec resource_type_from_str(string()) -> {ok, resource_type()} | {error, term()}. +resource_type_from_str(ResourceType) -> + try Mod = list_to_existing_atom(str(ResourceType)), + case emqx_resource:is_resource_mod(Mod) of + true -> {ok, Mod}; + false -> {error, {invalid_resource, Mod}} + end + catch error:badarg -> + {error, {resource_not_found, ResourceType}} + end. + +call_instance(InstId, Query) -> + emqx_resource_instance:hash_call(InstId, Query). + +safe_apply(Func, Args) -> + ?SAFE_CALL(erlang:apply(Func, Args)). + +str(S) when is_binary(S) -> binary_to_list(S); +str(S) when is_list(S) -> S. diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl new file mode 100644 index 000000000..fe1ca4509 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_api). + +-export([ list_instances/1 + , format_data/1 + , stringnify/1 + ]). + +list_instances(Filter) -> + [format_data(Data) || Data <- emqx_resource:list_instances_verbose(), Filter(Data)]. + +format_data(#{id := Id, mod := Mod, status := Status, config := Config}) -> + #{id => Id, status => Status, resource_type => Mod, + config => emqx_resource:call_jsonify(Mod, Config)}. + +stringnify(Bin) when is_binary(Bin) -> Bin; +stringnify(Str) when is_list(Str) -> list_to_binary(Str); +stringnify(Reason) -> + iolist_to_binary(io_lib:format("~p", [Reason])). diff --git a/apps/emqx_resource/src/emqx_resource_app.erl b/apps/emqx_resource/src/emqx_resource_app.erl new file mode 100644 index 000000000..d2c499490 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_app.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_app). + +-behaviour(application). + +-include("emqx_resource.hrl"). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_resource_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl new file mode 100644 index 000000000..1e924e249 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -0,0 +1,276 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_instance). + +-behaviour(gen_server). + +-include("emqx_resource.hrl"). +-include("emqx_resource_utils.hrl"). + +-export([start_link/2]). + +%% load resource instances from *.conf files +-export([ lookup/1 + , list_all/0 + , lookup_by_type/1 + , create_local/3 + ]). + +-export([ hash_call/2 + , hash_call/3 + ]). + +%% gen_server Callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, {worker_pool, worker_id}). + +-type state() :: #state{}. + +%%------------------------------------------------------------------------------ +%% Start the registry +%%------------------------------------------------------------------------------ + +start_link(Pool, Id) -> + gen_server:start_link({local, proc_name(?MODULE, Id)}, + ?MODULE, {Pool, Id}, []). + +%% call the worker by the hash of resource-instance-id, to make sure we always handle +%% operations on the same instance in the same worker. +hash_call(InstId, Request) -> + hash_call(InstId, Request, infinity). + +hash_call(InstId, Request, Timeout) -> + gen_server:call(pick(InstId), Request, Timeout). + +-spec lookup(instance_id()) -> {ok, resource_data()} | {error, Reason :: term()}. +lookup(InstId) -> + case ets:lookup(emqx_resource_instance, InstId) of + [] -> {error, not_found}; + [{_, Data}] -> {ok, Data#{id => InstId}} + end. + +force_lookup(InstId) -> + {ok, Data} = lookup(InstId), + Data. + +-spec list_all() -> [resource_data()]. +list_all() -> + [Data#{id => Id} || {Id, Data} <- ets:tab2list(emqx_resource_instance)]. + +-spec lookup_by_type(module()) -> [resource_data()]. +lookup_by_type(ResourceType) -> + [Data || #{mod := Mod} = Data <- list_all() + , Mod =:= ResourceType]. + +-spec create_local(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, term()}. +create_local(InstId, ResourceType, InstConf) -> + case hash_call(InstId, {create, InstId, ResourceType, InstConf}, 15000) of + {ok, Data} -> {ok, Data}; + Error -> Error + end. + +save_config_to_disk(InstId, ResourceType, Config) -> + %% TODO: send an event to the config handler, and the hander (single process) + %% will dump configs for all instances (from an ETS table) to a file. + file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), + jsx:encode(#{id => InstId, resource_type => ResourceType, + config => emqx_resource:call_jsonify(ResourceType, Config)})). + +emqx_data_dir() -> + "data". + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +-spec init({atom(), integer()}) -> + {ok, State :: state()} | {ok, State :: state(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term()} | ignore. +init({Pool, Id}) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #state{worker_pool = Pool, worker_id = Id}}. + +handle_call({create, InstId, ResourceType, Config}, _From, State) -> + {reply, do_create(InstId, ResourceType, Config), State}; + +handle_call({create_dry_run, InstId, ResourceType, Config}, _From, State) -> + {reply, do_create_dry_run(InstId, ResourceType, Config), State}; + +handle_call({update, InstId, ResourceType, Config, Params}, _From, State) -> + {reply, do_update(InstId, ResourceType, Config, Params), State}; + +handle_call({remove, InstId}, _From, State) -> + {reply, do_remove(InstId), State}; + +handle_call({restart, InstId}, _From, State) -> + {reply, do_restart(InstId), State}; + +handle_call({stop, InstId}, _From, State) -> + {reply, do_stop(InstId), State}; + +handle_call({health_check, InstId}, _From, State) -> + {reply, do_health_check(InstId), State}; + +handle_call(Req, _From, State) -> + logger:error("Received unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #state{worker_pool = Pool, worker_id = Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ + +%% suppress the race condition check, as these functions are protected in gproc workers +-dialyzer({nowarn_function, [do_update/4, do_create/3, do_restart/1, do_stop/1, do_health_check/1]}). +do_update(InstId, ResourceType, NewConfig, Params) -> + case lookup(InstId) of + {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> + Config = emqx_resource:call_config_merge(ResourceType, OldConfig, + NewConfig, Params), + case do_create_dry_run(InstId, ResourceType, Config) of + ok -> + do_remove(ResourceType, InstId, ResourceState), + do_create(InstId, ResourceType, Config); + Error -> + Error + end; + {ok, #{mod := Mod}} when Mod =/= ResourceType -> + {error, updating_to_incorrect_resource_type}; + {error, not_found} -> + {error, not_found} + end. + +do_create(InstId, ResourceType, Config) -> + case lookup(InstId) of + {ok, _} -> {error, already_created}; + _ -> + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState} -> + ets:insert(emqx_resource_instance, {InstId, + #{mod => ResourceType, config => Config, + state => ResourceState, status => stopped}}), + _ = do_health_check(InstId), + case save_config_to_disk(InstId, ResourceType, Config) of + ok -> {ok, force_lookup(InstId)}; + {error, Reason} -> + logger:error("save config for ~p resource ~p to disk failed: ~p", + [ResourceType, InstId, Reason]), + {error, Reason} + end; + {error, Reason} -> + logger:error("start ~s resource ~s failed: ~p", [ResourceType, InstId, Reason]), + {error, Reason} + end + end. + +do_create_dry_run(InstId, ResourceType, Config) -> + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState0} -> + Return = case emqx_resource:call_health_check(InstId, ResourceType, ResourceState0) of + {ok, ResourceState1} -> ok; + {error, Reason, ResourceState1} -> + {error, Reason} + end, + _ = emqx_resource:call_stop(InstId, ResourceType, ResourceState1), + Return; + {error, Reason} -> + {error, Reason} + end. + +do_remove(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState}} -> + do_remove(Mod, InstId, ResourceState); + Error -> + Error + end. + +do_remove(Mod, InstId, ResourceState) -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ets:delete(emqx_resource_instance, InstId), + ok. + +do_restart(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState, config := Config} = Data} -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + case emqx_resource:call_start(InstId, Mod, Config) of + {ok, ResourceState} -> + ets:insert(emqx_resource_instance, + {InstId, Data#{state => ResourceState, status => started}}), + ok; + {error, Reason} -> + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), + {error, Reason} + end; + Error -> + Error + end. + +do_stop(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState} = Data} -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), + ok; + Error -> + Error + end. + +do_health_check(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState0} = Data} -> + case emqx_resource:call_health_check(InstId, Mod, ResourceState0) of + {ok, ResourceState1} -> + ets:insert(emqx_resource_instance, + {InstId, Data#{status => started, state => ResourceState1}}), + ok; + {error, Reason, ResourceState1} -> + logger:error("health check for ~p failed: ~p", [InstId, Reason]), + ets:insert(emqx_resource_instance, + {InstId, Data#{status => stopped, state => ResourceState1}}), + {error, Reason} + end; + Error -> + Error + end. + +%%------------------------------------------------------------------------------ +%% internal functions +%%------------------------------------------------------------------------------ + +proc_name(Mod, Id) -> + list_to_atom(lists:concat([Mod, "_", Id])). + +pick(InstId) -> + gproc_pool:pick_worker(emqx_resource_instance, InstId). diff --git a/apps/emqx_resource/src/emqx_resource_schema.erl b/apps/emqx_resource/src/emqx_resource_schema.erl new file mode 100644 index 000000000..62d9cb6bc --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_schema.erl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_schema). + +-export([check/2]). + +-export([structs/0, fields/1]). + +-behaviour(hocon_schema). + +check(SchemaMod, Conf) -> + _ = erlang:erase(res_schema_mod), + erlang:put(res_schema_mod, SchemaMod), + hocon_schema:check(?MODULE, Conf). + +structs() -> ["config"]. + +fields("config") -> + [fun(type) -> "schema"; + (_) -> undefined + end]; +fields("schema") -> + SchemaMod = erlang:get(res_schema_mod), + SchemaMod:schema(). diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl new file mode 100644 index 000000000..22984b940 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(RESOURCE_INST_MOD, emqx_resource_instance). +-define(POOL_SIZE, 64). %% set a very large pool size in case all the workers busy + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + TabOpts = [named_table, set, public, {read_concurrency, true}], + _ = ets:new(emqx_resource_instance, TabOpts), + + SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, + Pool = ?RESOURCE_INST_MOD, + Mod = ?RESOURCE_INST_MOD, + ensure_pool(Pool, hash, [{size, ?POOL_SIZE}]), + {ok, {SupFlags, [ + begin + ensure_pool_worker(Pool, {Pool, Idx}, Idx), + #{id => {Mod, Idx}, + start => {Mod, start_link, [Pool, Idx]}, + restart => transient, + shutdown => 5000, type => worker, modules => [Mod]} + end || Idx <- lists:seq(1, ?POOL_SIZE)]}}. + +%% internal functions +ensure_pool(Pool, Type, Opts) -> + try gproc_pool:new(Pool, Type, Opts) + catch + error:exists -> ok + end. + +ensure_pool_worker(Pool, Name, Slot) -> + try gproc_pool:add_worker(Pool, Name, Slot) + catch + error:exists -> ok + end. \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl new file mode 100644 index 000000000..a738a4ca6 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_transform). + +-include_lib("syntax_tools/include/merl.hrl"). + +-export([parse_transform/2]). + +parse_transform(Forms, _Opts) -> + Mod = hd([M || {attribute, _, module, M} <- Forms]), + AST = trans(Mod, proplists:delete(eof, Forms)), + _ = debug_print(Mod, AST), + AST. + +-ifdef(RESOURCE_DEBUG). + +debug_print(Mod, Ts) -> + {ok, Io} = file:open("./" ++ atom_to_list(Mod) ++ ".trans.erl", [write]), + _ = do_debug_print(Io, Ts), + file:close(Io). + +do_debug_print(Io, Ts) when is_list(Ts) -> + lists:foreach(fun(T) -> do_debug_print(Io, T) end, Ts); +do_debug_print(Io, T) -> + io:put_chars(Io, erl_prettypr:format(merl:tree(T))), + io:nl(Io). +-else. +debug_print(_Mod, _AST) -> + ok. +-endif. + +trans(Mod, Forms) -> + forms(Mod, Forms) ++ [erl_syntax:revert(erl_syntax:eof_marker())]. + +forms(Mod, [F0 | Fs0]) -> + case form(Mod, F0) of + {CurrForms, AppendedForms} -> + CurrForms ++ forms(Mod, Fs0) ++ AppendedForms; + {CurrForms, FollowerForms, AppendedForms} -> + CurrForms ++ FollowerForms ++ forms(Mod, Fs0) ++ AppendedForms + end; +forms(_, []) -> []. + +form(Mod, Form) -> + case Form of + ?Q("-module('@_').") -> + {[Form], fix_spec_attrs(), fix_spec_funcs(Mod)}; + _ -> + %io:format("---other form: ~p~n", [Form]), + {[Form], [], []} + end. + +fix_spec_attrs() -> + [ ?Q("-export([emqx_resource_schema/0]).") + ]. +fix_spec_funcs(_Mod) -> + [ ?Q("emqx_resource_schema() -> <<\"demo_swagger_schema\">>.") + ]. diff --git a/apps/emqx_resource/src/emqx_resource_uitils.erl b/apps/emqx_resource/src/emqx_resource_uitils.erl new file mode 100644 index 000000000..ab3f6dd1e --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_uitils.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_uitils). \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl new file mode 100644 index 000000000..e9517f160 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -0,0 +1,63 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_validator). + +-export([ min/2 + , max/2 + , equals/2 + , enum/1 + , required/1 + ]). + +max(Type, Max) -> + limit(Type, '=<', Max). + +min(Type, Min) -> + limit(Type, '>=', Min). + +equals(Type, Expected) -> + limit(Type, '==', Expected). + +enum(Items) -> + fun(Value) -> + return(lists:member(Value, Items), + err_limit({enum, {is_member_of, Items}, {got, Value}})) + end. + +required(ErrMsg) -> + fun(undefined) -> {error, ErrMsg}; + (_) -> ok + end. + +limit(Type, Op, Expected) -> + L = len(Type), + fun(Value) -> + Got = L(Value), + return(erlang:Op(Got, Expected), + err_limit({Type, {Op, Expected}, {got, Got}})) + end. + +len(array) -> fun erlang:length/1; +len(string) -> fun string:length/1; +len(_Type) -> fun(Val) -> Val end. + +err_limit({Type, {Op, Expected}, {got, Got}}) -> + io_lib:format("Expect the ~s value ~s ~p but got: ~p", [Type, Op, Expected, Got]). + +return(true, _) -> ok; +return(false, Error) -> + {error, Error}. diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index 0a883cee5..4db438a98 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -37,5 +37,5 @@ retainer.max_payload_size = 1MB ## - 30m: 30 minutes ## - 20s: 20 seconds ## -## Defaut: 0 +## Default: 0 retainer.expiry_interval = 0 diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 9557780e8..7e762cb72 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -18,7 +18,7 @@ {profiles, [{test, [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + [ {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} ]} ]}. diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index 2fe946779..556c59970 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -32,7 +32,7 @@ rule_engine.ignore_sys_message = on ## ## QoS-Level: qos0/qos1/qos2 -#rule_engine.events.client_connected = on, qos1 +#rule_engine.events.client_connected = "on, qos1" rule_engine.events.client_connected = off rule_engine.events.client_disconnected = off rule_engine.events.session_subscribed = off diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index d424c9f8a..d8244c018 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -2553,33 +2553,20 @@ stop_apps() -> start_apps() -> [start_apps(App, SchemaFile, ConfigFile) || {App, SchemaFile, ConfigFile} - <- [{emqx, deps_path(emqx, "priv/emqx.schema"), - deps_path(emqx, "etc/emqx.conf")}, + <- [{emqx, emqx_schema, deps_path(emqx, "etc/emqx.conf")}, {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), local_path("etc/emqx_rule_engine.conf")}]]. -start_apps(App, SchemaFile, ConfigFile) -> - read_schema_configs(App, SchemaFile, ConfigFile), - set_special_configs(App), - {ok, _} = application:ensure_all_started(App). - -read_schema_configs(App, SchemaFile, ConfigFile) -> - ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = conf_parse:file(ConfigFile), - NewConfig = cuttlefish_generator:map(Schema, Conf), - Vals = proplists:get_value(App, NewConfig, []), - [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. +start_apps(App, Schema, ConfigFile) -> + emqx_ct_helpers:start_app(App, Schema, ConfigFile, fun set_special_configs/1). deps_path(App, RelativePath) -> - %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir - %% but priv dir is - Path0 = code:priv_dir(App), + Path0 = code:lib_dir(App), Path = case file:read_link(Path0) of {ok, Resolved} -> Resolved; {error, _} -> Path0 end, - filename:join([Path, "..", RelativePath]). + filename:join([Path, RelativePath]). local_path(RelativePath) -> deps_path(emqx_rule_engine, RelativePath). diff --git a/apps/emqx_sn/etc/emqx_sn.conf b/apps/emqx_sn/etc/emqx_sn.conf index 6572812c1..e05f1e7be 100644 --- a/apps/emqx_sn/etc/emqx_sn.conf +++ b/apps/emqx_sn/etc/emqx_sn.conf @@ -6,7 +6,7 @@ ## ## Value: IP:Port | Port ## -## Examples: 1884, 127.0.0.1:1884, ::1:1884 +## Examples: 1884, "127.0.0.1:1884", "::1:1884" mqtt.sn.port = 1884 ## The duration that emqx-sn broadcast ADVERTISE message through. @@ -37,8 +37,8 @@ mqtt.sn.idle_timeout = 30s ## The pre-defined topic name corresponding to the pre-defined topic id of N. ## Note that the pre-defined topic id of 0 is reserved. mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = /predefined/topic/name/hello -mqtt.sn.predefined.topic.2 = /predefined/topic/name/nice +mqtt.sn.predefined.topic.1 = "/predefined/topic/name/hello" +mqtt.sn.predefined.topic.2 = "/predefined/topic/name/nice" ## Default username for MQTT-SN. This parameter is optional. If specified, ## emq-sn will connect EMQ core with this username. It is useful if any auth diff --git a/apps/emqx_sn/priv/emqx_sn.schema b/apps/emqx_sn/priv/emqx_sn.schema index a585c1037..edc76db37 100644 --- a/apps/emqx_sn/priv/emqx_sn.schema +++ b/apps/emqx_sn/priv/emqx_sn.schema @@ -1,23 +1,19 @@ %%-*- mode: erlang -*- %% emqx_sn config mapping {mapping, "mqtt.sn.port", "emqx_sn.port", [ - {default, "1884"}, - {datatype, string} + {default, 1884}, + {datatype, [integer, ip]} ]}. {translation, "emqx_sn.port", fun(Conf) -> - case re:split(cuttlefish:conf_get("mqtt.sn.port", Conf, ""), ":", [{return, list}]) of - [Port] -> - {{0,0,0,0}, list_to_integer(Port)}; - Tokens -> - Port = lists:last(Tokens), - IP = case inet:parse_address(lists:flatten(lists:join(":", Tokens -- [Port]))) of - {error, Reason} -> - throw({invalid_ip_address, Reason}); - {ok, X} -> X - end, - Port1 = list_to_integer(Port), - {IP, Port1} + case cuttlefish:conf_get("mqtt.sn.port", Conf, undefined) of + Port when is_integer(Port) -> + {{0,0,0,0}, Port}; + {Ip, Port} -> + case inet:parse_address(Ip) of + {ok ,R} -> {R, Port}; + _ -> {Ip, Port} + end end end}. diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 5fecbb815..cbdac78f6 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -2,8 +2,7 @@ {plugins, [rebar3_proper]}. {deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}}, - {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index 82ec3b9fb..0e4e53dc8 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_stomp/etc/emqx_stomp.conf b/apps/emqx_stomp/etc/emqx_stomp.conf index b404e99ee..832b50655 100644 --- a/apps/emqx_stomp/etc/emqx_stomp.conf +++ b/apps/emqx_stomp/etc/emqx_stomp.conf @@ -8,7 +8,7 @@ ## The Port that stomp listener will bind. ## ## Value: Port -stomp.listener = 61613 +stomp.listener.port = 61613 ## The acceptor pool for stomp listener. ## @@ -28,22 +28,22 @@ stomp.listener.max_connections = 512 ## Path to the file containing the user's private PEM-encoded key. ## ## Value: File -## stomp.listener.keyfile = etc/certs/key.pem +## stomp.listener.keyfile = "etc/certs/key.pem" ## Path to a file containing the user certificate. ## ## Value: File -## stomp.listener.certfile = etc/certs/cert.pem +## stomp.listener.certfile = "etc/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## Value: File -## stomp.listener.cacertfile = etc/certs/cacert.pem +## stomp.listener.cacertfile = "etc/certs/cacert.pem" ## See: 'listener.ssl..dhfile' in emq.conf ## ## Value: File -## stomp.listener.dhfile = etc/certs/dh-params.pem +## stomp.listener.dhfile = "etc/certs/dh-params.pem" ## See: 'listener.ssl..verify' in emq.conf ## @@ -59,7 +59,7 @@ stomp.listener.max_connections = 512 ## ## Value: String, seperated by ',' ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## stomp.listener.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +## stomp.listener.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## SSL Handshake timeout. ## @@ -69,7 +69,7 @@ stomp.listener.max_connections = 512 ## See: 'listener.ssl..ciphers' in emq.conf ## ## Value: Ciphers -## stomp.listener.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## stomp.listener.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## See: 'listener.ssl..secure_renegotiate' in emq.conf ## diff --git a/apps/emqx_stomp/priv/emqx_stomp.schema b/apps/emqx_stomp/priv/emqx_stomp.schema index c77cc297e..32a3c272b 100644 --- a/apps/emqx_stomp/priv/emqx_stomp.schema +++ b/apps/emqx_stomp/priv/emqx_stomp.schema @@ -1,7 +1,7 @@ %%-*- mode: erlang -*- %% emqx_stomp config mapping -{mapping, "stomp.listener", "emqx_stomp.listener", [ +{mapping, "stomp.listener.port", "emqx_stomp.listener", [ {default, 61613}, {datatype, [integer, ip]} ]}. @@ -72,7 +72,7 @@ ]}. {translation, "emqx_stomp.listener", fun(Conf) -> - Port = cuttlefish:conf_get("stomp.listener", Conf), + Port = cuttlefish:conf_get("stomp.listener.port", Conf), Acceptors = cuttlefish:conf_get("stomp.listener.acceptors", Conf), MaxConnections = cuttlefish:conf_get("stomp.listener.max_connections", Conf), Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index b03abdac1..2e66734ec 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf index 7b9d32dfb..6707e4673 100644 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ b/apps/emqx_web_hook/etc/emqx_web_hook.conf @@ -5,16 +5,16 @@ ## Webhook URL ## ## Value: String -web.hook.url = http://127.0.0.1:80 +web.hook.url = "http://127.0.0.1:80" ## HTTP Headers ## ## Example: -## 1. web.hook.headers.content-type = application/json -## 2. web.hook.headers.accept = * +## 1. web.hook.headers.content-type = "application/json" +## 2. web.hook.headers.accept = "*" ## ## Value: String -web.hook.headers.content-type = application/json +web.hook.headers.content-type = "application/json" ## The encoding format of the payload field in the HTTP body ## The payload field only appears in the on_message_publish and on_message_delivered actions @@ -63,15 +63,15 @@ web.hook.pool_size = 32 ## ## Format: ## web.hook.rule.. = -#web.hook.rule.client.connect.1 = {"action": "on_client_connect"} -#web.hook.rule.client.connack.1 = {"action": "on_client_connack"} -#web.hook.rule.client.connected.1 = {"action": "on_client_connected"} -#web.hook.rule.client.disconnected.1 = {"action": "on_client_disconnected"} -#web.hook.rule.client.subscribe.1 = {"action": "on_client_subscribe"} -#web.hook.rule.client.unsubscribe.1 = {"action": "on_client_unsubscribe"} -#web.hook.rule.session.subscribed.1 = {"action": "on_session_subscribed"} -#web.hook.rule.session.unsubscribed.1 = {"action": "on_session_unsubscribed"} -#web.hook.rule.session.terminated.1 = {"action": "on_session_terminated"} -#web.hook.rule.message.publish.1 = {"action": "on_message_publish"} -#web.hook.rule.message.delivered.1 = {"action": "on_message_delivered"} -#web.hook.rule.message.acked.1 = {"action": "on_message_acked"} +#web.hook.rule.client.connect.1 = "{"action": "on_client_connect"}" +#web.hook.rule.client.connack.1 = "{"action": "on_client_connack"}" +#web.hook.rule.client.connected.1 = "{"action": "on_client_connected"}" +#web.hook.rule.client.disconnected.1 = "{"action": "on_client_disconnected"}" +#web.hook.rule.client.subscribe.1 = "{"action": "on_client_subscribe"}" +#web.hook.rule.client.unsubscribe.1 = "{"action": "on_client_unsubscribe"}" +#web.hook.rule.session.subscribed.1 = "{"action": "on_session_subscribed"}" +#web.hook.rule.session.unsubscribed.1 = "{"action": "on_session_unsubscribed"}" +#web.hook.rule.session.terminated.1 = "{"action": "on_session_terminated"}" +#web.hook.rule.message.publish.1 = "{"action": "on_message_publish"}" +#web.hook.rule.message.delivered.1 = "{"action": "on_message_delivered"}" +#web.hook.rule.message.acked.1 = ""{"action": "on_message_acked"}" diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config index 5a1c77868..387972c9f 100644 --- a/apps/emqx_web_hook/rebar.config +++ b/apps/emqx_web_hook/rebar.config @@ -15,4 +15,4 @@ warnings_as_errors, deprecated_functions]}. {cover_enabled, true}. {cover_opts, [verbose]}. -{cover_export_enabled, true}. +{cover_export_enabled, true}. \ No newline at end of file diff --git a/bin/emqx b/bin/emqx index fd6986614..96f6ec42f 100755 --- a/bin/emqx +++ b/bin/emqx @@ -20,8 +20,8 @@ mkdir -p "$RUNNER_LOG_DIR" # Make sure data directory exists mkdir -p "$RUNNER_DATA_DIR" -# cuttlefish try to read environment variables starting with "EMQX_" -export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' +# hocon try to read environment variables starting with "EMQX_" +export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' relx_usage() { command="$1" @@ -123,9 +123,6 @@ fi # Echo to stderr on errors echoerr() { echo "$@" 1>&2; } -# By default, use cuttlefish to generate app.config and vm.args -CUTTLEFISH="${USE_CUTTLEFISH:-yes}" - SED_REPLACE="sed -i " case $(sed --help 2>&1) in *GNU*) SED_REPLACE="sed -i ";; @@ -202,54 +199,46 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - if [ "$CUTTLEFISH" != "yes" ]; then - # Note: we have added a parameter '-vm_args' to this. It - # appears redundant but it is not! the erlang vm allows us to - # access all arguments to the erl command EXCEPT '-args_file', - # so in order to get access to this file location from within - # the vm, we need to pass it in twice. - CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args " - else - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" - # shellcheck disable=SC2181 - RESULT=$? - set -e - if [ $RESULT -gt 0 ]; then - echo "$CUTTLEFISH_OUTPUT" - exit $RESULT - fi - # print override from environment variables (EMQX_*) - echo "$CUTTLEFISH_OUTPUT" | sed -e '$d' - CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1) - - ## Merge cuttlefish generated *.args into the vm.args - CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') - TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" - cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" - echo "" >> "$TMP_ARG_FILE" - echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" - sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do - ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') - ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') - if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then - if [ -n "$TMP_ARG_VALUE" ]; then - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" - else - echo "$ARG_LINE" >> "$TMP_ARG_FILE" - fi - fi - done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + EMQX_LICENSE_CONF_OPTION="" + if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then + EMQX_LICENSE_CONF_OPTION="-c ${EMQX_LICENSE_CONF}" fi + set +e + # disable shellcheck; let EMQX_LICENSE_CONF_OPTION split + # shellcheck disable=SC2086 + HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf $EMQX_LICENSE_CONF_OPTION -d "$RUNNER_DATA_DIR"/configs generate)" + # shellcheck disable=SC2181 + RESULT=$? + set -e + if [ $RESULT -gt 0 ]; then + echo "$HOCON_OUTPUT" + exit $RESULT + fi + # print override from environment variables (EMQX_*) + echo "$HOCON_OUTPUT" | sed -e '$d' + CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1) + + ## Merge hocon generated *.args into the vm.args + HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') + TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" + cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" + echo "" >> "$TMP_ARG_FILE" + echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" + sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do + ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') + ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then + if [ -n "$TMP_ARG_VALUE" ]; then + sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" + else + echo "$ARG_LINE" >> "$TMP_ARG_FILE" + fi + fi + done + mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE" + # shellcheck disable=SC2086 if ! relx_nodetool chkconfig $CONFIG_ARGS; then echoerr "Error reading $CONFIG_ARGS" @@ -303,7 +292,8 @@ if [ -z "$NAME_ARG" ]; then NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')" else # for boot commands, inspect emqx.conf for node name - NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name) + # todo: use get command from hocon escript + NODENAME="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")" fi fi if [ -z "$NODENAME" ]; then @@ -329,7 +319,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" COOKIE="${EMQX_NODE_COOKIE:-}" if [ -z "$COOKIE" ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then - COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie) + COOKIE="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")" else # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" diff --git a/etc/BUILT_ON b/data/BUILT_ON similarity index 100% rename from etc/BUILT_ON rename to data/BUILT_ON diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..5ac46e0e3 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,4 +5,7 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. +{emqx_resource, {{enable_plugin_emqx_resource}}}. +{emqx_connector, {{enable_plugin_emqx_connector}}}. +{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index 933b9885d..4f28ec84e 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -20,7 +20,7 @@ dashboard.default_user.password = public ## Value: Port ## ## Examples: 18083 -dashboard.listener.http = 18083 +dashboard.listener.http.port = 18083 ## The acceptor pool for external Dashboard HTTP listener. ## @@ -50,7 +50,7 @@ dashboard.listener.http.ipv6_v6only = false ## Value: Port ## ## Examples: 18084 -## dashboard.listener.https = 18084 +## dashboard.listener.https.port = 18084 ## The acceptor pool for external Dashboard HTTPS listener. ## @@ -75,22 +75,22 @@ dashboard.listener.http.ipv6_v6only = false ## Path to the file containing the user's private PEM-encoded key. ## ## Value: File -## dashboard.listener.https.keyfile = etc/certs/key.pem +## dashboard.listener.https.keyfile = "etc/certs/key.pem" ## Path to a file containing the user certificate. ## ## Value: File -## dashboard.listener.https.certfile = etc/certs/cert.pem +## dashboard.listener.https.certfile = "etc/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## Value: File -## dashboard.listener.https.cacertfile = etc/certs/cacert.pem +## dashboard.listener.https.cacertfile = "etc/certs/cacert.pem" ## See: 'listener.ssl..dhfile' in emq.conf ## ## Value: File -## dashboard.listener.https.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## dashboard.listener.https.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## See: 'listener.ssl..verify' in emq.conf ## @@ -106,12 +106,12 @@ dashboard.listener.http.ipv6_v6only = false ## ## Value: String, seperated by ',' ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## dashboard.listener.https.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +## dashboard.listener.https.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## See: 'listener.ssl..ciphers' in emq.conf ## ## Value: Ciphers -## dashboard.listener.https.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## dashboard.listener.https.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## See: 'listener.ssl..secure_renegotiate' in emq.conf ## diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index d517a6e97..d485ecdf9 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -10,7 +10,7 @@ {override_env, "ADMIN_PASSWORD"} ]}. -{mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ +{mapping, "dashboard.listener.http.port", "emqx_dashboard.listeners", [ {datatype, integer} ]}. @@ -38,7 +38,7 @@ {datatype, {enum, [true, false]}} ]}. -{mapping, "dashboard.listener.https", "emqx_dashboard.listeners", [ +{mapping, "dashboard.listener.https.port", "emqx_dashboard.listeners", [ {datatype, integer} ]}. @@ -139,7 +139,7 @@ lists:map( fun(Proto) -> Prefix = "dashboard.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of undefined -> []; Port -> [{Proto, Port, case Proto of diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 92bd59a7a..1604198dc 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_telemetry/etc/emqx_telemetry.conf b/lib-ce/emqx_telemetry/etc/emqx_telemetry.conf index 041b54f60..326ef0508 100644 --- a/lib-ce/emqx_telemetry/etc/emqx_telemetry.conf +++ b/lib-ce/emqx_telemetry/etc/emqx_telemetry.conf @@ -13,8 +13,8 @@ telemetry.enabled = true ## ## Value: String ## -## Default: https://telemetry.emqx.io/api/telemetry -telemetry.url = https://telemetry.emqx.io/api/telemetry +## Default: "https://telemetry.emqx.io/api/telemetry" +telemetry.url = "https://telemetry.emqx.io/api/telemetry" ## Interval for reporting telemetry data ## @@ -25,4 +25,4 @@ telemetry.url = https://telemetry.emqx.io/api/telemetry ## -s: second ## ## Default: 7d -telemetry.report_interval = 7d \ No newline at end of file +telemetry.report_interval = 7d diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 904690ad1..72bdb2850 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -13,7 +13,7 @@ else fi ## emqx_release.hrl is the single source of truth for release version -RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" include/emqx_release.hrl | cut -d '"' -f2)" +RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" ## git commit hash is added as suffix in case the git tag and release version is not an exact match if [ -d .git ] && ! git describe --tags --match "[e|v]${RELEASE}" --exact >/dev/null 2>&1; then diff --git a/priv/emqx.schema b/priv/emqx.schema deleted file mode 100644 index 7c6bf0298..000000000 --- a/priv/emqx.schema +++ /dev/null @@ -1,2481 +0,0 @@ -%%-*- mode: erlang -*- -%% EMQ X R4.0 config mapping - -%%-------------------------------------------------------------------- -%% Cluster -%%-------------------------------------------------------------------- - -%% @doc Cluster name -{mapping, "cluster.name", "ekka.cluster_name", [ - {default, emqxcl}, - {datatype, atom} -]}. - -%% @doc Cluster discovery -{mapping, "cluster.discovery", "ekka.cluster_discovery", [ - {default, manual}, - {datatype, atom} -]}. - -%% @doc Clean down node from the cluster -{mapping, "cluster.autoclean", "ekka.cluster_autoclean", [ - {datatype, {duration, ms}} -]}. - -%% @doc Cluster autoheal -{mapping, "cluster.autoheal", "ekka.cluster_autoheal", [ - {datatype, flag}, - {default, off} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by static node list - -{mapping, "cluster.static.seeds", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by UDP Multicast - -{mapping, "cluster.mcast.addr", "ekka.cluster_discovery", [ - {default, "239.192.0.1"}, - {datatype, string} -]}. - -{mapping, "cluster.mcast.ports", "ekka.cluster_discovery", [ - {default, "4369"}, - {datatype, string} -]}. - -{mapping, "cluster.mcast.iface", "ekka.cluster_discovery", [ - {datatype, string}, - {default, "0.0.0.0"} -]}. - -{mapping, "cluster.mcast.ttl", "ekka.cluster_discovery", [ - {datatype, integer}, - {default, 255} -]}. - -{mapping, "cluster.mcast.loop", "ekka.cluster_discovery", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "cluster.mcast.sndbuf", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "16KB"} -]}. - -{mapping, "cluster.mcast.recbuf", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "16KB"} -]}. - -{mapping, "cluster.mcast.buffer", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "32KB"} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by DNS A Record - -{mapping, "cluster.dns.name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%% @doc The erlang distributed protocol -{mapping, "cluster.proto_dist", "ekka.proto_dist", [ - {default, "inet_tcp"}, - {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, - hidden -]}. - -{mapping, "cluster.dns.app", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Cluster using etcd - -{mapping, "cluster.etcd.server", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.node_ttl", "ekka.cluster_discovery", [ - {datatype, {duration, ms}}, - {default, "1m"} -]}. - -{mapping, "cluster.etcd.ssl.keyfile", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.ssl.certfile", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.ssl.cacertfile", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Cluster on K8s - -{mapping, "cluster.k8s.apiserver", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.service_name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.address_type", "ekka.cluster_discovery", [ - {datatype, {enum, [ip, dns, hostname]}} -]}. - -{mapping, "cluster.k8s.app_name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.namespace", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.suffix", "ekka.cluster_discovery", [ - {datatype, string}, - {default, ""} - ]}. - -{translation, "ekka.cluster_discovery", fun(Conf) -> - Strategy = cuttlefish:conf_get("cluster.discovery", Conf), - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - IpPort = fun(S) -> - [Addr, Port] = string:tokens(S, ":"), - {ok, Ip} = inet:parse_address(Addr), - {Ip, Port} - end, - Options = fun(static) -> - [{seeds, [list_to_atom(S) || S <- string:tokens(cuttlefish:conf_get("cluster.static.seeds", Conf, ""), ",")]}]; - (mcast) -> - {ok, Addr} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.addr", Conf)), - {ok, Iface} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.iface", Conf)), - Ports = [list_to_integer(S) || S <- string:tokens(cuttlefish:conf_get("cluster.mcast.ports", Conf), ",")], - [{addr, Addr}, {ports, Ports}, {iface, Iface}, - {ttl, cuttlefish:conf_get("cluster.mcast.ttl", Conf, 1)}, - {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; - (dns) -> - [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, - {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; - (etcd) -> - SslOpts = fun(Conf) -> - Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), - lists:map(fun({["cluster", "etcd", "ssl", Name], Value}) -> - {list_to_atom(Name), Value} - end, Options) - end, - [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, - {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, - {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}, - {ssl_options, SslOpts(Conf)}]; - (k8s) -> - [{apiserver, cuttlefish:conf_get("cluster.k8s.apiserver", Conf)}, - {service_name, cuttlefish:conf_get("cluster.k8s.service_name", Conf)}, - {address_type, cuttlefish:conf_get("cluster.k8s.address_type", Conf, ip)}, - {app_name, cuttlefish:conf_get("cluster.k8s.app_name", Conf)}, - {namespace, cuttlefish:conf_get("cluster.k8s.namespace", Conf)}, - {suffix, cuttlefish:conf_get("cluster.k8s.suffix", Conf, "")}]; - (manual) -> - [ ] - end, - {Strategy, Filter(Options(Strategy))} -end}. - -%%-------------------------------------------------------------------- -%% Node -%%-------------------------------------------------------------------- - -%% @doc Node name -{mapping, "node.name", "vm_args.-name", [ - {default, "emqx@127.0.0.1"}, - {override_env, "NODE_NAME"} -]}. - -%% @doc Specify SSL Options in the file if using SSL for erlang distribution -{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ - {datatype, string}, - hidden -]}. - -%% @doc Secret cookie for distributed erlang node -{mapping, "node.cookie", "vm_args.-setcookie", [ - {default, "emqxsecretcookie"}, - {override_env, "NODE_COOKIE"} -]}. - -{mapping, "node.data_dir", "emqx.data_dir", [ - {datatype, string} -]}. - -%% @doc http://erlang.org/doc/man/heart.html -{mapping, "node.heartbeat", "vm_args.-heart", [ - {datatype, flag}, - hidden -]}. - -{translation, "vm_args.-heart", fun(Conf) -> - case cuttlefish:conf_get("node.heartbeat", Conf) of - true -> ""; - false -> cuttlefish:invalid("should be 'on' or comment the line!") - end -end}. - -%% @doc More information at: http://erlang.org/doc/man/erl.html -{mapping, "node.async_threads", "vm_args.+A", [ - {datatype, integer}, - {validators, ["range:0-1024"]} -]}. - -%% @doc Erlang Process Limit -{mapping, "node.process_limit", "vm_args.+P", [ - {datatype, integer}, - hidden -]}. - -%% @doc The maximum number of concurrent ports/sockets. -%% Valid range is 1024-134217727 -{mapping, "node.max_ports", "vm_args.+Q", [ - {datatype, integer}, - {validators, ["range4ports"]}, - {override_env, "MAX_PORTS"} -]}. - -{validator, "range4ports", "must be 1024 to 134217727", - fun(X) -> X >= 1024 andalso X =< 134217727 end}. - -%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl -{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ - {datatype, bytesize}, - {commented, "32MB"}, - hidden, - {validators, ["zdbbl_range"]} -]}. - -{translation, "vm_args.+zdbbl", - fun(Conf) -> - ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), - case ZDBBL of - undefined -> undefined; - X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; - _ -> undefined - end - end}. - -{validator, "zdbbl_range", "must be between 1KB and 2097151KB", - fun(ZDBBL) -> - %% 2097151KB = 2147482624 - ZDBBL >= 1024 andalso ZDBBL =< 2147482624 - end -}. - -%% @doc Global GC Interval -{mapping, "node.global_gc_interval", "emqx.global_gc_interval", [ - {datatype, {duration, s}} -]}. - -%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 -{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ - {default, 1000}, - {datatype, integer}, - hidden, - {validators, ["positive_integer"]} -]}. - -{validator, "positive_integer", "must be a positive integer", - fun(X) -> X >= 0 end}. - -%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, -%% R16+ uses +e -%% @doc The ETS table limit -{mapping, "node.max_ets_tables", - cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ - {default, 256000}, - {datatype, integer}, - hidden -]}. - -%% @doc Set the location of crash dumps -{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ - {default, "{{crash_dump}}"}, - {datatype, file}, - hidden -]}. - -%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime -{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ - {datatype, integer}, - hidden -]}. - -%% @doc http://www.erlang.org/doc/man/kernel_app.html -{mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ - {commented, 6369}, - {datatype, integer}, - hidden -]}. - -%% @see node.dist_listen_min -{mapping, "node.dist_listen_max", "kernel.inet_dist_listen_max", [ - {commented, 6369}, - {datatype, integer}, - hidden -]}. - -{mapping, "node.backtrace_depth", "emqx.backtrace_depth", [ - {default, 16}, - {datatype, integer} -]}. - -%%-------------------------------------------------------------------- -%% RPC -%%-------------------------------------------------------------------- - -%% RPC Mode. -{mapping, "rpc.mode", "emqx.rpc_mode", [ - {default, async}, - {datatype, {enum, [sync, async]}} -]}. - -{mapping, "rpc.async_batch_size", "gen_rpc.max_batch_size", [ - {default, 256}, - {datatype, integer} -]}. - -{mapping, "rpc.port_discovery", "gen_rpc.port_discovery", [ - {default, stateless}, - {datatype, {enum, [manual, stateless]}} -]}. - -%% RPC server port. -{mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ - {default, 5369}, - {datatype, integer} -]}. - -%% Number of tcp connections when connecting to RPC server -{mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:gt_0_lt_256"]} -]}. - -{translation, "gen_rpc.tcp_client_num", fun(Conf) -> - case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of - 0 -> max(1, erlang:system_info(schedulers) div 2); - V -> V - end -end}. - -%% Client connect timeout -{mapping, "rpc.connect_timeout", "gen_rpc.connect_timeout", [ - {default, "5s"}, - {datatype, {duration, ms}} -]}. - -%% Client and Server send timeout -{mapping, "rpc.send_timeout", "gen_rpc.send_timeout", [ - {default, 5000}, - {datatype, {duration, ms}} -]}. - -%% Authentication timeout -{mapping, "rpc.authentication_timeout", "gen_rpc.authentication_timeout", [ - {default, 5000}, - {datatype, {duration, ms}} -]}. - -%% Default receive timeout for call() functions -{mapping, "rpc.call_receive_timeout", "gen_rpc.call_receive_timeout", [ - {default, 15000}, - {datatype, {duration, ms}} -]}. - -%% Socket keepalive configuration -{mapping, "rpc.socket_keepalive_idle", "gen_rpc.socket_keepalive_idle", [ - {default, 7200}, - {datatype, {duration, s}} -]}. - -%% Seconds between probes -{mapping, "rpc.socket_keepalive_interval", "gen_rpc.socket_keepalive_interval", [ - {default, 75}, - {datatype, {duration, s}} -]}. - -%% Probes lost to close the connection -{mapping, "rpc.socket_keepalive_count", "gen_rpc.socket_keepalive_count", [ - {default, 9}, - {datatype, integer} -]}. - -%% Size of TCP send buffer -{mapping, "rpc.socket_sndbuf", "gen_rpc.socket_sndbuf", [ - {default, "1MB"}, - {datatype, bytesize} -]}. - -%% Size of TCP receive buffer -{mapping, "rpc.socket_recbuf", "gen_rpc.socket_recbuf", [ - {default, "1MB"}, - {datatype, bytesize} -]}. - -%% Size of TCP receive buffer -{mapping, "rpc.socket_buffer", "gen_rpc.socket_buffer", [ - {default, "1MB"}, - {datatype, bytesize} -]}. - -{validator, "range:gt_0_lt_256", "must greater than 0 and less than 256", - fun(X) -> X >= 0 andalso X < 256 end -}. - -%% Force client to use server listening port, because we do no provide -%% per-node listening port manual mapping from configs. -%% i.e. all nodes in the cluster should agree to the same -%% listening port number. -{translation, "gen_rpc.tcp_client_port", fun(_, _, Conf) -> - cuttlefish:conf_get("rpc.tcp_server_port", Conf) -end}. - -%%-------------------------------------------------------------------- -%% Log -%%-------------------------------------------------------------------- - -{mapping, "log.to", "kernel.logger", [ - {default, file}, - {datatype, {enum, [file, console, both]}} -]}. - -{mapping, "log.level", "kernel.logger", [ - {default, warning}, - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} -]}. - -{mapping, "log.primary_log_level", "kernel.logger_level", [ - {default, warning}, - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} -]}. - -{mapping, "log.dir", "kernel.logger", [ - {default, "log"}, - {datatype, string} -]}. - -{mapping, "log.file", "kernel.logger", [ - {default, "emqx.log"}, - {datatype, file} -]}. - -{mapping, "log.chars_limit", "kernel.logger", [ - {default, -1}, - {datatype, integer} -]}. - -{mapping, "log.supervisor_reports", "kernel.logger", [ - {default, error}, - {datatype, {enum, [error, progress]}}, - hidden -]}. - -%% @doc Maximum depth in Erlang term log formatting -%% and message queue inspection. -{mapping, "log.max_depth", "kernel.error_logger_format_depth", [ - {default, 20}, - {datatype, [{enum, [unlimited]}, integer]} -]}. - -%% @doc format logs as JSON objects -{mapping, "log.formatter", "kernel.logger", [ - {default, text}, - {datatype, {enum, [text, json]}} -]}. - -%% @doc format logs in a single line. -{mapping, "log.single_line", "kernel.logger", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "log.rotation", "kernel.logger", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "log.rotation.size", "kernel.logger", [ - {default, "10MB"}, - {datatype, bytesize} -]}. - -{mapping, "log.size", "kernel.logger", [ - {default, infinity}, - {datatype, [bytesize, atom]} -]}. - -{mapping, "log.rotation.count", "kernel.logger", [ - {default, 5}, - {datatype, integer} -]}. - -{mapping, "log.$level.file", "kernel.logger", [ - {datatype, file} -]}. - -{mapping, "log.sync_mode_qlen", "kernel.logger", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "log.drop_mode_qlen", "kernel.logger", [ - {default, 3000}, - {datatype, integer} -]}. - -{mapping, "log.flush_qlen", "kernel.logger", [ - {default, 8000}, - {datatype, integer} -]}. - -{mapping, "log.overload_kill", "kernel.logger", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "log.overload_kill_mem_size", "kernel.logger", [ - {default, "30MB"}, - {datatype, bytesize} -]}. - -{mapping, "log.overload_kill_qlen", "kernel.logger", [ - {default, 20000}, - {datatype, integer} -]}. - -{mapping, "log.overload_kill_restart_after", "kernel.logger", [ - {default, "5s"}, - {datatype, [{duration, ms}, atom]} -]}. - -{mapping, "log.burst_limit", "kernel.logger", [ - {default, "disabled"}, - {datatype, string} -]}. - -{mapping, "log.error_logger", "kernel.error_logger", [ - {default, silent}, - {datatype, {enum, [silent]}}, - hidden -]}. - -%% disable lager -{mapping, "lager.handlers", "lager.handlers", [ - {default, []}, - hidden -]}. -{mapping, "lager.crash_log", "lager.crash_log", [ - {default, off}, - {datatype, flag}, - hidden -]}. - -{translation, "kernel.logger_level", fun(_, _, Conf) -> - cuttlefish:conf_get("log.level", Conf) -end}. - -{translation, "kernel.logger", fun(Conf) -> - LogTo = cuttlefish:conf_get("log.to", Conf), - LogLevel = cuttlefish:conf_get("log.level", Conf), - LogType = case cuttlefish:conf_get("log.rotation", Conf) of - true -> wrap; - false -> halt - end, - CharsLimit = case cuttlefish:conf_get("log.chars_limit", Conf) of - -1 -> unlimited; - V -> V - end, - SingleLine = cuttlefish:conf_get("log.single_line", Conf), - FmtName = cuttlefish:conf_get("log.formatter", Conf), - Formatter = - case FmtName of - json -> - {emqx_logger_jsonfmt, - #{chars_limit => CharsLimit, - single_line => SingleLine - }}; - text -> - {emqx_logger_textfmt, - #{template => - [time," [",level,"] ", - {clientid, - [{peername, - [clientid,"@",peername," "], - [clientid, " "]}], - [{peername, - [peername," "], - []}]}, - msg,"\n"], - chars_limit => CharsLimit, - single_line => SingleLine - }} - end, - {BustLimitOn, {MaxBurstCount, TimeWindow}} = - case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of - ["disabled"] -> {false, {20000, 1000}}; - [Count, Window] -> - {true, {list_to_integer(Count), - case cuttlefish_duration:parse(Window, ms) of - Secs when is_integer(Secs) -> Secs; - {error, Reason1} -> error(Reason1) - end}} - end, - FileConf = fun(Filename) -> - BasicConf = - #{type => LogType, - file => filename:join(cuttlefish:conf_get("log.dir", Conf), Filename), - max_no_files => cuttlefish:conf_get("log.rotation.count", Conf), - sync_mode_qlen => cuttlefish:conf_get("log.sync_mode_qlen", Conf), - drop_mode_qlen => cuttlefish:conf_get("log.drop_mode_qlen", Conf), - flush_qlen => cuttlefish:conf_get("log.flush_qlen", Conf), - overload_kill_enable => cuttlefish:conf_get("log.overload_kill", Conf), - overload_kill_qlen => cuttlefish:conf_get("log.overload_kill_qlen", Conf), - overload_kill_mem_size => cuttlefish:conf_get("log.overload_kill_mem_size", Conf), - overload_kill_restart_after => cuttlefish:conf_get("log.overload_kill_restart_after", Conf), - burst_limit_enable => BustLimitOn, - burst_limit_max_count => MaxBurstCount, - burst_limit_window_time => TimeWindow - }, - MaxNoBytes = case LogType of - wrap -> cuttlefish:conf_get("log.rotation.size", Conf); - halt -> cuttlefish:conf_get("log.size", Conf) - end, - BasicConf#{max_no_bytes => MaxNoBytes} - end, - - Filters = case cuttlefish:conf_get("log.supervisor_reports", Conf) of - error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; - progress -> [] - end, - - %% For the default logger that outputs to console - DefaultHandler = - if LogTo =:= console orelse LogTo =:= both -> - [{handler, console, logger_std_h, - #{level => LogLevel, - config => #{type => standard_io}, - formatter => Formatter, - filters => Filters - } - }]; - true -> - [{handler, default, undefined}] - end, - - %% For the file logger - FileHandler = - if LogTo =:= file orelse LogTo =:= both -> - [{handler, file, logger_disk_log_h, - #{level => LogLevel, - config => FileConf(cuttlefish:conf_get("log.file", Conf)), - formatter => Formatter, - filesync_repeat_interval => no_repeat, - filters => Filters - }}]; - true -> [] - end, - - %% For creating additional log files for specific log levels. - AdditionalLogFiles = - lists:foldl( - fun({[_, Level, _] = K, Filename}, Acc) when LogTo =:= file; LogTo =:= both -> - case cuttlefish_variable:is_fuzzy_match(K, ["log", "$level", "file"]) of - true -> [{Level, Filename} | Acc]; - false -> Acc - end; - ({_K, _V}, Acc) -> - Acc - end, [], Conf), - AdditionalHandlers = - [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, - #{level => list_to_atom(Level), - config => FileConf(Filename), - formatter => Formatter, - filesync_repeat_interval => no_repeat}} - || {Level, Filename} <- AdditionalLogFiles], - - DefaultHandler ++ FileHandler ++ AdditionalHandlers -end}. - -%%-------------------------------------------------------------------- -%% Authentication/ACL -%%-------------------------------------------------------------------- - -%% @doc Allow anonymous authentication. -{mapping, "allow_anonymous", "emqx.allow_anonymous", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc ACL nomatch. -{mapping, "acl_nomatch", "emqx.acl_nomatch", [ - {default, deny}, - {datatype, {enum, [allow, deny]}} -]}. - -%% @doc Default ACL file. -{mapping, "acl_file", "emqx.acl_file", [ - {datatype, string}, - hidden -]}. - -%% @doc Enable ACL cache for publish. -{mapping, "enable_acl_cache", "emqx.enable_acl_cache", [ - {default, on}, - {datatype, flag} -]}. - -%% @doc ACL cache time-to-live. -{mapping, "acl_cache_ttl", "emqx.acl_cache_ttl", [ - {default, "1m"}, - {datatype, {duration, ms}} -]}. - -%% @doc ACL cache size. -{mapping, "acl_cache_max_size", "emqx.acl_cache_max_size", [ - {default, 32}, - {datatype, integer}, - {validators, ["range:gt_0"]} -]}. - -%% @doc Action when acl check reject current operation -{mapping, "acl_deny_action", "emqx.acl_deny_action", [ - {default, ignore}, - {datatype, {enum, [ignore, disconnect]}} -]}. - -%% @doc Flapping detect policy -{mapping, "flapping_detect_policy", "emqx.flapping_detect_policy", [ - {datatype, string}, - {default, "30,1m,5m"} -]}. - -{translation, "emqx.flapping_detect_policy", fun(Conf) -> - Policy = cuttlefish:conf_get("flapping_detect_policy", Conf), - [Threshold, Duration, Interval] = string:tokens(Policy, ", "), - ParseDuration = fun(S, Dur) -> - case cuttlefish_duration:parse(S, Dur) of - I when is_integer(I) -> I; - {error, Reason} -> error(Reason) - end - end, - #{threshold => list_to_integer(Threshold), - duration => ParseDuration(Duration, ms), - banned_interval => ParseDuration(Interval, s) - } -end}. - -{validator, "range:gt_0", "must greater than 0", - fun(X) -> X > 0 end -}. - -%%-------------------------------------------------------------------- -%% MQTT Protocol -%%-------------------------------------------------------------------- - -%% @doc Max Packet Size Allowed, 1MB by default. -{mapping, "mqtt.max_packet_size", "emqx.max_packet_size", [ - {default, "1MB"}, - {datatype, bytesize}, - {override_env, "MAX_PACKET_SIZE"} -]}. - -%% @doc Set the Max ClientId Length Allowed. -{mapping, "mqtt.max_clientid_len", "emqx.max_clientid_len", [ - {default, 65535}, - {datatype, integer} -]}. - -%% @doc Set the Maximum topic levels. -{mapping, "mqtt.max_topic_levels", "emqx.max_topic_levels", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Set the Maximum QoS allowed. -{mapping, "mqtt.max_qos_allowed", "emqx.max_qos_allowed", [ - {default, 2}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -%% @doc Set the Maximum Topic Alias. -{mapping, "mqtt.max_topic_alias", "emqx.max_topic_alias", [ - {default, 65535}, - {datatype, integer} -]}. - -%% @doc Whether the server supports MQTT retained messages. -{mapping, "mqtt.retain_available", "emqx.retain_available", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether the Server supports MQTT Wildcard Subscriptions. -{mapping, "mqtt.wildcard_subscription", "emqx.wildcard_subscription", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether the Server supports MQTT Shared Subscriptions. -{mapping, "mqtt.shared_subscription", "emqx.shared_subscription", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether to ignore loop delivery of messages.(for mqtt v3.1.1) -{mapping, "mqtt.ignore_loop_deliver", "emqx.ignore_loop_deliver", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether to parse the MQTT frame in strict mode -{mapping, "mqtt.strict_mode", "emqx.strict_mode", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Specify the response information returned to the client -{mapping, "mqtt.response_information", "emqx.response_information", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Zones -%%-------------------------------------------------------------------- - -%% @doc Idle timeout of the MQTT connection. -{mapping, "zone.$name.idle_timeout", "emqx.zones", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "zone.$name.allow_anonymous", "emqx.zones", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "zone.$name.acl_nomatch", "emqx.zones", [ - {datatype, {enum, [allow, deny]}} -]}. - -%% @doc Enable ACL check. -{mapping, "zone.$name.enable_acl", "emqx.zones", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Action when acl check reject current operation -{mapping, "zone.$name.acl_deny_action", "emqx.zones", [ - {default, ignore}, - {datatype, {enum, [ignore, disconnect]}} -]}. - -%% @doc Enable Ban. -{mapping, "zone.$name.enable_ban", "emqx.zones", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Enable per connection statistics. -{mapping, "zone.$name.enable_stats", "emqx.zones", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Publish limit of the MQTT connections. -{mapping, "zone.$name.publish_limit", "emqx.zones", [ - {datatype, string} -]}. - -%% @doc Max Packet Size Allowed, 64K by default. -{mapping, "zone.$name.max_packet_size", "emqx.zones", [ - {datatype, bytesize} -]}. - -%% @doc Set the Max ClientId Length Allowed. -{mapping, "zone.$name.max_clientid_len", "emqx.zones", [ - {datatype, integer} -]}. - -%% @doc Set the Maximum topic levels. -{mapping, "zone.$name.max_topic_levels", "emqx.zones", [ - {datatype, integer} -]}. - -%% @doc Set the Maximum QoS allowed. -{mapping, "zone.$name.max_qos_allowed", "emqx.zones", [ - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -%% @doc Set the Maximum topic alias. -{mapping, "zone.$name.max_topic_alias", "emqx.zones", [ - {datatype, integer} -]}. - -%% @doc Whether the server supports retained messages. -{mapping, "zone.$name.retain_available", "emqx.zones", [ - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether the Server supports Wildcard Subscriptions. -{mapping, "zone.$name.wildcard_subscription", "emqx.zones", [ - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether the Server supports Shared Subscriptions. -{mapping, "zone.$name.shared_subscription", "emqx.zones", [ - {datatype, {enum, [true, false]}} -]}. - -%% @doc Server Keepalive -{mapping, "zone.$name.server_keepalive", "emqx.zones", [ - {datatype, integer} -]}. - -%% @doc Keepalive backoff -{mapping, "zone.$name.keepalive_backoff", "emqx.zones", [ - {default, 0.75}, - {datatype, float} -]}. - -%% @doc Max Number of Subscriptions Allowed. -{mapping, "zone.$name.max_subscriptions", "emqx.zones", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Upgrade QoS according to subscription? -{mapping, "zone.$name.upgrade_qos", "emqx.zones", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time. -%% 0 is equivalent to maximum allowed -{mapping, "zone.$name.max_inflight", "emqx.zones", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:1-65535"]} -]}. - -%% @doc Retry interval for redelivering QoS1/2 messages. -{mapping, "zone.$name.retry_interval", "emqx.zones", [ - {default, "30s"}, - {datatype, {duration, s}} -]}. - -%% @doc Max Packets that Awaiting PUBREL, 0 means no limit -{mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Awaiting PUBREL timeout -{mapping, "zone.$name.await_rel_timeout", "emqx.zones", [ - {default, "300s"}, - {datatype, {duration, s}} -]}. - -%% @doc Ignore loop delivery of messages -{mapping, "zone.$name.ignore_loop_deliver", "emqx.zones", [ - {datatype, {enum, [true, false]}} -]}. - -%% @doc Session Expiry Interval -{mapping, "zone.$name.session_expiry_interval", "emqx.zones", [ - {default, "2h"}, - {datatype, {duration, s}} -]}. - -%% @doc Max queue length. Enqueued messages when persistent client -%% disconnected, or inflight window is full. 0 means no limit. -{mapping, "zone.$name.max_mqueue_len", "emqx.zones", [ - {default, 1000}, - {datatype, integer} -]}. - -%% @doc Topic Priorities, comma separated topic=priority pairs, -%% where priority should be integer in range 1-255 (inclusive) -%% 1 being the lowest and 255 being the highest. -%% default value `none` to indicate no priority table, hence all -%% messages are treated equal, which means either highest ('infinity'), -%% or lowest (0) depending on mqueue_default_priority config. -{mapping, "zone.$name.mqueue_priorities", "emqx.zones", [ - {default, "none"}, - {datatype, string} -]}. - -%% @doc Default priority for topics not in priority table. -{mapping, "zone.$name.mqueue_default_priority", "emqx.zones", [ - {default, lowest}, - {datatype, {enum, [highest, lowest]}} -]}. - -%% @doc Queue Qos0 messages? -{mapping, "zone.$name.mqueue_store_qos0", "emqx.zones", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "zone.$name.enable_flapping_detect", "emqx.zones", [ - {datatype, flag}, - {default, off} -]}. - -{mapping, "zone.$name.rate_limit.conn_messages_in", "emqx.zones", [ - {datatype, string} -]}. - -{mapping, "zone.$name.rate_limit.conn_bytes_in", "emqx.zones", [ - {datatype, string} -]}. - -{mapping, "zone.$name.conn_congestion.alarm", "emqx.zones", [ - {datatype, flag}, - {default, off} -]}. - -{mapping, "zone.$name.conn_congestion.min_alarm_sustain_duration", "emqx.zones", [ - {default, "1m"}, - {datatype, {duration, ms}} -]}. - -{mapping, "zone.$name.quota.conn_messages_routing", "emqx.zones", [ - {datatype, string} -]}. - -{mapping, "zone.$name.quota.overall_messages_routing", "emqx.zones", [ - {datatype, string} -]}. - -%% @doc Force connection/session process GC after this number of -%% messages | bytes passed through. -%% Numbers delimited by `|'. Zero or negative is to disable. -{mapping, "zone.$name.force_gc_policy", "emqx.zones", [ - {datatype, string} - ]}. - -%% @doc Max message queue length and total heap size to force shutdown -%% connection/session process. -%% Message queue here is the Erlang process mailbox, but not the number -%% of queued MQTT messages of QoS 1 and 2. -%% Zero or negative is to disable. -{mapping, "zone.$name.force_shutdown_policy", "emqx.zones", [ - {default, "default"}, - {datatype, string} -]}. - -{mapping, "zone.$name.mountpoint", "emqx.zones", [ - {datatype, string} -]}. - -%% @doc Use username replace client id -{mapping, "zone.$name.use_username_as_clientid", "emqx.zones", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Whether to parse the MQTT frame in strict mode -{mapping, "zone.$name.strict_mode", "emqx.zones", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Specify the response information returned to the client -{mapping, "zone.$name.response_information", "emqx.zones", [ - {datatype, string} -]}. - -%% @doc Whether to bypass the authentication step -{mapping, "zone.$name.bypass_auth_plugins", "emqx.zones", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqx.zones", fun(Conf) -> - Ratelimit = fun(Val) -> - [L, D] = string:tokens(Val, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason1} -> error(Reason1) - end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason} -> error(Reason) - end, - {Limit, Duration} - end, - Mapping = fun(["publish_limit"], Val) -> - %% XXX: Deprecated at v4.2 - {publish_limit, Ratelimit(Val)}; - (["force_gc_policy"], Val) -> - [Count, Bytes] = string:tokens(Val, "| "), - GcPolicy = case cuttlefish_bytesize:parse(Bytes) of - {error, Reason} -> - error(Reason); - Bytes1 -> - #{bytes => Bytes1, - count => list_to_integer(Count)} - end, - {force_gc_policy, GcPolicy}; - (["force_shutdown_policy"], "default") -> - {DefaultLen, DefaultSize} = - case WordSize = erlang:system_info(wordsize) of - 8 -> % arch_64 - {10000, cuttlefish_bytesize:parse("64MB")}; - 4 -> % arch_32 - {1000, cuttlefish_bytesize:parse("32MB")} - end, - {force_shutdown_policy, #{message_queue_len => DefaultLen, - max_heap_size => DefaultSize div WordSize - }}; - (["force_shutdown_policy"], Val) -> - [Len, Siz] = string:tokens(Val, "| "), - MaxSiz = case WordSize = erlang:system_info(wordsize) of - 8 -> % arch_64 - (1 bsl 59) - 1; - 4 -> % arch_32 - (1 bsl 27) - 1 - end, - ShutdownPolicy = - case cuttlefish_bytesize:parse(Siz) of - {error, Reason} -> - error(Reason); - Siz1 when Siz1 > MaxSiz -> - cuttlefish:invalid(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); - Siz1 -> - #{message_queue_len => list_to_integer(Len), - max_heap_size => Siz1 div WordSize} - end, - {force_shutdown_policy, ShutdownPolicy}; - (["mqueue_priorities"], Val) -> - case Val of - "none" -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE - _ -> - MqueuePriorities = lists:foldl(fun(T, Acc) -> - %% NOTE: space in "= " is intended - [Topic, Prio] = string:tokens(T, "= "), - P = list_to_integer(Prio), - (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), - maps:put(iolist_to_binary(Topic), P, Acc) - end, #{}, string:tokens(Val, ",")), - {mqueue_priorities, MqueuePriorities} - end; - (["mountpoint"], Val) -> - {mountpoint, iolist_to_binary(Val)}; - (["response_information"], Val) -> - {response_information, iolist_to_binary(Val)}; - (["rate_limit", "conn_messages_in"], Val) -> - {ratelimit, {conn_messages_in, Ratelimit(Val)}}; - (["rate_limit", "conn_bytes_in"], Val) -> - {ratelimit, {conn_bytes_in, Ratelimit(Val)}}; - (["conn_congestion", "alarm"], Val) -> - {conn_congestion_alarm_enabled, Val}; - (["conn_congestion", "min_alarm_sustain_duration"], Val) -> - {conn_congestion_min_alarm_sustain_duration, Val}; - (["quota", "conn_messages_routing"], Val) -> - {quota, {conn_messages_routing, Ratelimit(Val)}}; - (["quota", "overall_messages_routing"], Val) -> - {quota, {overall_messages_routing, Ratelimit(Val)}}; - ([Opt], Val) -> - {list_to_atom(Opt), Val} - end, - maps:to_list( - lists:foldl( - fun({["zone", Name | Opt], Val}, Zones) -> - NVal = Mapping(Opt, Val), - maps:update_with(list_to_atom(Name), - fun(Opts) -> - case NVal of - {Key, Rl} when Key == ratelimit; - Key == quota -> - Rls = proplists:get_value(Key, Opts, []), - lists:keystore(Key, 1, Opts, {Key, [Rl|Rls]}); - _ -> - [NVal|Opts] - end - end, [NVal], Zones) - end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("zone.", Conf)))) -end}. - -%%-------------------------------------------------------------------- -%% Listeners -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% TCP Listeners - -{mapping, "listener.tcp.$name", "emqx.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.tcp.$name.acceptors", "emqx.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.max_connections", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.max_conn_rate", "emqx.listeners", [ - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.zone", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.rate_limit", "emqx.listeners", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.access.$id", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.proxy_protocol", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.tcp.$name.proxy_protocol_timeout", "emqx.listeners", [ - {datatype, {duration, ms}} -]}. - -%% The proxy-protocol protocol can get the certificate CN through tcp -{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn]}} -]}. - -%% The proxy-protocol protocol can get the certificate CN through tcp -{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn]}} -]}. - -{mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ - {datatype, integer}, - {default, 1024} -]}. - -{mapping, "listener.tcp.$name.send_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.tcp.$name.send_timeout_close", "emqx.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.tcp.$name.recbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.sndbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.buffer", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.high_watermark", "emqx.listeners", [ - {datatype, bytesize}, - {default, "1MB"} - ]}. - -{mapping, "listener.tcp.$name.tune_buffer", "emqx.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.tcp.$name.nodelay", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.tcp.$name.reuseaddr", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% SSL Listeners - -{mapping, "listener.ssl.$name", "emqx.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.ssl.$name.acceptors", "emqx.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.max_connections", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.max_conn_rate", "emqx.listeners", [ - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.zone", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.rate_limit", "emqx.listeners", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.access.$id", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.proxy_protocol", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.proxy_protocol_timeout", "emqx.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ssl.$name.backlog", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.send_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.ssl.$name.send_timeout_close", "emqx.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.ssl.$name.recbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.sndbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.buffer", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.high_watermark", "emqx.listeners", [ - {datatype, bytesize}, - {default, "1MB"} - ]}. - -{mapping, "listener.ssl.$name.tune_buffer", "emqx.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.ssl.$name.nodelay", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ssl.$name.reuseaddr", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ssl.$name.tls_versions", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.ciphers", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.psk_ciphers", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.handshake_timeout", "emqx.listeners", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ssl.$name.depth", "emqx.listeners", [ - {default, 10}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.key_password", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.dhfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.keyfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.certfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.cacertfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.verify", "emqx.listeners", [ - {datatype, atom} -]}. - -{mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqx.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.ssl.$name.secure_renegotiate", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.reuse_sessions", "emqx.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.honor_cipher_order", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - -{mapping, "listener.ssl.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - -%%-------------------------------------------------------------------- -%% MQTT/WebSocket Listeners - -{mapping, "listener.ws.$name", "emqx.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.ws.$name.mqtt_path", "emqx.listeners", [ - {default, "/mqtt"}, - {datatype, string} -]}. - -{mapping, "listener.ws.$name.acceptors", "emqx.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.max_connections", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.max_conn_rate", "emqx.listeners", [ - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.active_n", "emqx.listeners", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.zone", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.rate_limit", "emqx.listeners", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "listener.ws.$name.access.$id", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.fail_if_no_subprotocol", "emqx.listeners", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.ws.$name.supported_subprotocols", "emqx.listeners", [ - {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, - {datatype, string} -]}. - -{mapping, "listener.ws.$name.proxy_address_header", "emqx.listeners", [ - {default, "X-Forwarded-For"}, - {datatype, string} -]}. - -{mapping, "listener.ws.$name.proxy_port_header", "emqx.listeners", [ - {default, "X-Forwarded-Port"}, - {datatype, string} -]}. - -{mapping, "listener.ws.$name.proxy_protocol", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ws.$name.proxy_protocol_timeout", "emqx.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ws.$name.backlog", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.send_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.ws.$name.send_timeout_close", "emqx.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.ws.$name.recbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.sndbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.buffer", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.tune_buffer", "emqx.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.ws.$name.nodelay", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ws.$name.compress", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.level", "emqx.listeners", [ - {datatype, {enum, [none, default, best_compression, best_speed]}}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.mem_level", "emqx.listeners", [ - {datatype, integer}, - {validators, ["range:1-9"]}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.strategy", "emqx.listeners", [ - {datatype, {enum, [default, filtered, huffman_only, rle]}}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ - {datatype, {enum, [takeover, no_takeover]}}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ - {datatype, {enum, [takeover, no_takeover]}}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ - {datatype, integer}, - hidden -]}. - -{mapping, "listener.ws.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ - {datatype, integer}, - hidden -]}. - -{mapping, "listener.ws.$name.idle_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - hidden -]}. - -{mapping, "listener.ws.$name.max_frame_size", "emqx.listeners", [ - {datatype, integer}, - hidden -]}. - -{mapping, "listener.ws.$name.mqtt_piggyback", "emqx.listeners", [ - {datatype, {enum, [single, multiple]}}, - {default, multiple}, - hidden -]}. - -{mapping, "listener.ws.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn]}} -]}. - -{mapping, "listener.ws.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn]}} -]}. - -{mapping, "listener.ws.$name.check_origin_enable", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - {default, false}, - hidden -]}. - -{mapping, "listener.ws.$name.allow_origin_absence", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - {default, true}, - hidden -]}. - -{mapping, "listener.ws.$name.check_origins", "emqx.listeners", [ - {datatype, string}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% MQTT/WebSocket/SSL Listeners - -{mapping, "listener.wss.$name", "emqx.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.wss.$name.mqtt_path", "emqx.listeners", [ - {default, "/mqtt"}, - {datatype, string} -]}. - -{mapping, "listener.wss.$name.acceptors", "emqx.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.max_connections", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.max_conn_rate", "emqx.listeners", [ - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.active_n", "emqx.listeners", [ - {default, 100}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.zone", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.rate_limit", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.fail_if_no_subprotocol", "emqx.listeners", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.wss.$name.supported_subprotocols", "emqx.listeners", [ - {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, - {datatype, string} -]}. - -{mapping, "listener.wss.$name.access.$id", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.proxy_address_header", "emqx.listeners", [ - {default, "X-Forwarded-For"}, - {datatype, string} -]}. - -{mapping, "listener.wss.$name.proxy_port_header", "emqx.listeners", [ - {default, "X-Forwarded-Port"}, - {datatype, string} -]}. - -{mapping, "listener.wss.$name.proxy_protocol", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.proxy_protocol_timeout", "emqx.listeners", [ - {datatype, {duration, ms}} -]}. - -%%{mapping, "listener.wss.$name.handshake_timeout", "emqx.listeners", [ -%% {default, "15s"}, -%% {datatype, {duration, ms}} -%%]}. - -{mapping, "listener.wss.$name.backlog", "emqx.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.send_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.wss.$name.send_timeout_close", "emqx.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.wss.$name.recbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.sndbuf", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.buffer", "emqx.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.tune_buffer", "emqx.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.wss.$name.nodelay", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.wss.$name.tls_versions", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.ciphers", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.psk_ciphers", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.keyfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.certfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.cacertfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.dhfile", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.depth", "emqx.listeners", [ - {default, 10}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.key_password", "emqx.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.verify", "emqx.listeners", [ - {datatype, atom} -]}. - -{mapping, "listener.wss.$name.fail_if_no_peer_cert", "emqx.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.wss.$name.secure_renegotiate", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.reuse_sessions", "emqx.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.honor_cipher_order", "emqx.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - -{mapping, "listener.wss.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - -{mapping, "listener.wss.$name.compress", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.level", "emqx.listeners", [ - {datatype, {enum, [none, default, best_compression, best_speed]}}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.mem_level", "emqx.listeners", [ - {datatype, integer}, - {validators, ["range:1-9"]}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.strategy", "emqx.listeners", [ - {datatype, {enum, [default, filtered, huffman_only, rle]}}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ - {datatype, {enum, [takeover, no_takeover]}}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ - {datatype, {enum, [takeover, no_takeover]}}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ - {datatype, integer}, - {validators, ["range:8-15"]}, - hidden -]}. - -{mapping, "listener.wss.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ - {datatype, integer}, - {validators, ["range:8-15"]}, - hidden -]}. - -{mapping, "listener.wss.$name.idle_timeout", "emqx.listeners", [ - {datatype, {duration, ms}}, - hidden -]}. - -{mapping, "listener.wss.$name.max_frame_size", "emqx.listeners", [ - {datatype, integer}, - hidden -]}. - -{mapping, "listener.wss.$name.mqtt_piggyback", "emqx.listeners", [ - {datatype, {enum, [single, multiple]}}, - {default, multiple}, - hidden -]}. - -{mapping, "listener.wss.$name.check_origin_enable", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - {default, false}, - hidden -]}. - -{mapping, "listener.wss.$name.allow_origin_absence", "emqx.listeners", [ - {datatype, {enum, [true, false]}}, - {default, true}, - hidden -]}. - -{mapping, "listener.wss.$name.check_origins", "emqx.listeners", [ - {datatype, string}, - hidden -]}. - -{translation, "emqx.listeners", fun(Conf) -> - - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - - Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, - - Access = fun(S) -> - [A, CIDR] = string:tokens(S, " "), - {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} - end, - - AccOpts = fun(Prefix) -> - case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of - [] -> []; - Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}] - end - end, - - RateLimit = fun(undefined) -> - undefined; - (Val) -> - [L, D] = string:tokens(Val, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason} -> error(Reason) - end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason1} -> error(Reason1) - end, - {Limit, Duration} - end, - - CheckOrigin = fun(S) -> - Origins = string:tokens(S, ","), - [ list_to_binary(string:trim(O)) || O <- Origins] - end, - - WsOpts = fun(Prefix) -> - case cuttlefish_variable:filter_by_prefix(Prefix ++ ".check_origins", Conf) of - [] -> undefined; - Rules -> - OriginList = [CheckOrigin(Rule) || {_, Rule} <- Rules], - lists:flatten(OriginList) - end - end, - - LisOpts = fun(Prefix) -> - Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, - {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, - {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, - {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, - {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, - {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, - {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, - {rate_limit, RateLimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, - {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, - {proxy_address_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, "")))}, - {proxy_port_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, "")))}, - {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, - {fail_if_no_subprotocol, cuttlefish:conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf, undefined)}, - {supported_subprotocols, string:tokens(cuttlefish:conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, - {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, - {peer_cert_as_clientid, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_clientid", Conf, undefined)}, - {compress, cuttlefish:conf_get(Prefix ++ ".compress", Conf, undefined)}, - {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)}, - {max_frame_size, cuttlefish:conf_get(Prefix ++ ".max_frame_size", Conf, undefined)}, - {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, - {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, - {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, - {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) - end, - DeflateOpts = fun(Prefix) -> - Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, - {mem_level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.mem_level", Conf, undefined)}, - {strategy, cuttlefish:conf_get(Prefix ++ ".deflate_opts.strategy", Conf, undefined)}, - {server_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf, undefined)}, - {client_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf, undefined)}, - {server_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf, undefined)}, - {client_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf, undefined)}]) - end, - TcpOpts = fun(Prefix) -> - Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, - {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, - {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, - {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, - {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, - {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, - {high_watermark, cuttlefish:conf_get(Prefix ++ ".high_watermark", Conf, undefined)}, - {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, - {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) - end, - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - MapPSKCiphers = fun(PSKCiphers) -> - lists:map( - fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; - ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; - ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; - ("PSK-RC4-SHA") -> {psk, rc4_128, sha} - end, PSKCiphers) - end, - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - TLSCiphers = cuttlefish:conf_get(Prefix++".ciphers", Conf, undefined), - PSKCiphers = cuttlefish:conf_get(Prefix++".psk_ciphers", Conf, undefined), - Ciphers = - case {TLSCiphers, PSKCiphers} of - {undefined, undefined} -> - cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); - {TLSCiphers, undefined} -> - SplitFun(TLSCiphers); - {undefined, PSKCiphers} -> - MapPSKCiphers(SplitFun(PSKCiphers)); - {_TLSCiphers, _PSKCiphers} -> - cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") - end, - UserLookupFun = - case PSKCiphers of - undefined -> undefined; - _ -> {fun emqx_psk:lookup/3, <<>>} - end, - Filter([{versions, Versions}, - {ciphers, Ciphers}, - {user_lookup_fun, UserLookupFun}, - {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, - {depth, cuttlefish:conf_get(Prefix ++ ".depth", Conf, undefined)}, - {password, cuttlefish:conf_get(Prefix ++ ".key_password", Conf, undefined)}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - - Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of - {ok, R} -> {R, Port}; - _ -> {Ip, Port} - end; - (Other) -> Other - end, - - TcpListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - ListenOnN = case cuttlefish:conf_get(Prefix, Conf, undefined) of - undefined -> []; - ListenOn -> Listen_fix(ListenOn) - end, - [#{ proto => Atom(Type) - , name => Name - , listen_on => ListenOnN - , opts => [ {deflate_options, DeflateOpts(Prefix)} - , {tcp_options, TcpOpts(Prefix)} - | LisOpts(Prefix) - ] - } - ] - end, - SslListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - case cuttlefish:conf_get(Prefix, Conf, undefined) of - undefined -> - []; - ListenOn -> - [#{ proto => Atom(Type) - , name => Name - , listen_on => Listen_fix(ListenOn) - , opts => [ {deflate_options, DeflateOpts(Prefix)} - , {tcp_options, TcpOpts(Prefix)} - , {ssl_options, SslOpts(Prefix)} - | LisOpts(Prefix) - ] - } - ] - end - end, - - lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} - <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) - ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] - ++ - [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} - <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) - ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) -end}. - -%%-------------------------------------------------------------------- -%% Modules -%%-------------------------------------------------------------------- - -{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [ - {datatype, string} -]}. - -{mapping, "module.presence.qos", "emqx.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.topic", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.subscription.$id.qos", "emqx.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.nl", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rap", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rh", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.rewrite.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{translation, "emqx.modules", fun(Conf, _, Conf1) -> - Subscriptions = fun() -> - List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), - TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], - [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), - nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), - rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), - rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) - }} || {N, T} <- TopicList] - end, - Rewrites = fun() -> - Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), - PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), - SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), - TotalRules = lists:append( - [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, - [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules - ), - lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> - [Topic, Re, Dest] = string:tokens(Rule, " "), - {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} - end, TotalRules) - end, - lists:append([ - [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], - [{emqx_mod_subscription, Subscriptions()}], - [{emqx_mod_rewrite, Rewrites()}], - [{emqx_mod_topic_metrics, []}], - [{emqx_mod_delayed, []}], - [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] - ]) -end}. - -%%------------------------------------------------------------------- -%% Plugins -%%------------------------------------------------------------------- - -{mapping, "plugins.etc_dir", "emqx.plugins_etc_dir", [ - {datatype, string} -]}. - -{mapping, "plugins.loaded_file", "emqx.plugins_loaded_file", [ - {datatype, string} -]}. - -{mapping, "plugins.expand_plugins_dir", "emqx.expand_plugins_dir", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Broker -%%-------------------------------------------------------------------- - -{mapping, "broker.sys_interval", "emqx.broker_sys_interval", [ - {datatype, {duration, ms}}, - {default, "1m"} -]}. - -{mapping, "broker.sys_heartbeat", "emqx.broker_sys_heartbeat", [ - {datatype, {duration, ms}}, - {default, "30s"} -]}. - -{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ - {default, quorum}, - {datatype, {enum, [local,leader,quorum,all]}} -]}. - -%% @doc Shared Subscription Dispatch Strategy. -{mapping, "broker.shared_subscription_strategy", "emqx.shared_subscription_strategy", [ - {default, round_robin}, - {datatype, - {enum, - [random, %% randomly pick a subscriber - round_robin, %% round robin alive subscribers one message after another - sticky, %% pick a random subscriber and stick to it - hash, %% hash client ID to a group member - hash_clientid, - hash_topic - ]}} -]}. - -%% @doc Enable or disable shared dispatch acknowledgement for QoS1 and QoS2 messages -{mapping, "broker.shared_dispatch_ack_enabled", "emqx.shared_dispatch_ack_enabled", - [ {default, false}, - {datatype, {enum, [true, false]}} - ]}. - -{mapping, "broker.route_batch_clean", "emqx.route_batch_clean", [ - {default, on}, - {datatype, flag} -]}. - -%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. -%% Change this toggle only when there are many wildcard topics. -%% key: mnesia translational updates with per-key locks. recommended for single node setup. -%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. -%% global: global lock protected updates. recommended for larger cluster. -%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster -%% -{mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ - {default, key}, - {datatype, {enum, [key, tab, global]}} -]}. - -%% @doc Enable trie path compaction. -%% Enabling it significantly improves wildcard topic subscribe rate, -%% if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', -%% where ID is unique per subscriber. -%% -%% Topic match performance (when publishing) may degrade if messages -%% are mostly published to topics with large number of levels. -%% -%% NOTE: This is a cluster-wide configuration. -%% It rquires all nodes to be stopped before changing it. -{mapping, "broker.perf.trie_compaction", "emqx.trie_compaction", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%%-------------------------------------------------------------------- -%% System Monitor -%%-------------------------------------------------------------------- - -%% @doc Long GC, don't monitor in production mode for: -%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 -{mapping, "sysmon.long_gc", "emqx.sysmon", [ - {default, 0}, - {datatype, [integer, {duration, ms}]} -]}. - -%% @doc Long Schedule(ms) -{mapping, "sysmon.long_schedule", "emqx.sysmon", [ - {default, 240}, - {datatype, [integer, {duration, ms}]} -]}. - -%% @doc Large Heap -{mapping, "sysmon.large_heap", "emqx.sysmon", [ - {default, "8MB"}, - {datatype, bytesize} -]}. - -%% @doc Monitor Busy Port -{mapping, "sysmon.busy_port", "emqx.sysmon", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Monitor Busy Dist Port -{mapping, "sysmon.busy_dist_port", "emqx.sysmon", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqx.sysmon", fun(Conf) -> - Configs = cuttlefish_variable:filter_by_prefix("sysmon", Conf), - [{list_to_atom(Name), Value} || {[_, Name], Value} <- Configs] -end}. - -%%-------------------------------------------------------------------- -%% Operating System Monitor -%%-------------------------------------------------------------------- - -{mapping, "os_mon.cpu_check_interval", "emqx.os_mon", [ - {default, 60}, - {datatype, {duration, s}} -]}. - -{mapping, "os_mon.cpu_high_watermark", "emqx.os_mon", [ - {default, "80%"}, - {datatype, {percent, float}} -]}. - -{mapping, "os_mon.cpu_low_watermark", "emqx.os_mon", [ - {default, "60%"}, - {datatype, {percent, float}} -]}. - -{mapping, "os_mon.mem_check_interval", "emqx.os_mon", [ - {default, 60}, - {datatype, {duration, s}} -]}. - -{mapping, "os_mon.sysmem_high_watermark", "emqx.os_mon", [ - {default, "70%"}, - {datatype, {percent, float}} -]}. - -{mapping, "os_mon.procmem_high_watermark", "emqx.os_mon", [ - {default, "5%"}, - {datatype, {percent, float}} -]}. - -{translation, "emqx.os_mon", fun(Conf) -> - [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)}, - {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf) * 100}, - {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf) * 100}, - {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)}, - {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf) * 100}, - {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf) * 100}] -end}. - -%%-------------------------------------------------------------------- -%% VM Monitor -%%-------------------------------------------------------------------- -{mapping, "vm_mon.check_interval", "emqx.vm_mon", [ - {default, 30}, - {datatype, {duration, s}} -]}. - -{mapping, "vm_mon.process_high_watermark", "emqx.vm_mon", [ - {default, "80%"}, - {datatype, {percent, float}} -]}. - -{mapping, "vm_mon.process_low_watermark", "emqx.vm_mon", [ - {default, "60%"}, - {datatype, {percent, float}} -]}. - -{translation, "emqx.vm_mon", fun(Conf) -> - [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)}, - {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf) * 100}, - {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf) * 100}] -end}. - -%%-------------------------------------------------------------------- -%% Alarm -%%-------------------------------------------------------------------- -{mapping, "alarm.actions", "emqx.alarm", [ - {default, "log,publish"}, - {datatype, string} -]}. - -{mapping, "alarm.size_limit", "emqx.alarm", [ - {default, 1000}, - {datatype, integer} -]}. - -{mapping, "alarm.validity_period", "emqx.alarm", [ - {default, "24h"}, - {datatype, {duration, s}} -]}. - -{translation, "emqx.alarm", fun(Conf) -> - [{actions, [list_to_atom(Action) || Action <- string:tokens(cuttlefish:conf_get("alarm.actions", Conf), ",")]}, - {size_limit, cuttlefish:conf_get("alarm.size_limit", Conf)}, - {validity_period, cuttlefish:conf_get("alarm.validity_period", Conf)}] -end}. - -%%-------------------------------------------------------------------- -%% Telemetry -%%-------------------------------------------------------------------- -{mapping, "telemetry.enabled", "emqx.telemetry", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "telemetry.url", "emqx.telemetry", [ - {default, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry"}, - {datatype, string} -]}. - -{mapping, "telemetry.report_interval", "emqx.telemetry", [ - {default, "7d"}, - {datatype, {duration, s}} -]}. - -{translation, "emqx.telemetry", fun(Conf) -> - [ {enabled, cuttlefish:conf_get("telemetry.enabled", Conf)} - , {url, cuttlefish:conf_get("telemetry.url", Conf)} - , {report_interval, cuttlefish:conf_get("telemetry.report_interval", Conf)} - ] -end}. diff --git a/rebar.config b/rebar.config index 54a6c7d1e..f123d93b2 100644 --- a/rebar.config +++ b/rebar.config @@ -11,8 +11,6 @@ warn_obsolete_guard,compressed, {d, snk_kind, msg}]}. -{extra_src_dirs, [{"etc", [{recursive,true}]}]}. - {xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, deprecated_function_calls,warnings_as_errors,deprecated_functions]}. @@ -33,8 +31,6 @@ {post_hooks,[]}. -{erl_first_files, ["src/emqx_logger.erl", "src/emqx_rule_actions_trans.erl"]}. - {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.6"}}} @@ -43,9 +39,9 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.9.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.5"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} % TODO: delete when all apps moved to hocon , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} @@ -56,6 +52,8 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.0"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 5901ab706..5cc857398 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -106,7 +106,7 @@ test_plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.9"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "2.0.0"}}} , meck ]. @@ -188,6 +188,9 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} + , {enable_plugin_emqx_resource, true} + , {enable_plugin_emqx_connector, true} + , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -245,6 +248,7 @@ relx_apps(ReleaseType) -> , {ekka, load} , {emqx_plugin_libs, load} , observer_cli + , emqx_http_lib ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -281,6 +285,9 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon + , emqx_resource + , emqx_connector + , emqx_data_bridge , emqx_rule_engine , emqx_sasl ] @@ -323,6 +330,7 @@ relx_overlay(ReleaseType) -> , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} , {template, "data/loaded_modules.tmpl", "data/loaded_modules"} , {template, "data/emqx_vars", "releases/emqx_vars"} + , {template, "data/BUILT_ON", "releases/{{release_version}}/BUILT_ON"} , {copy, "bin/emqx", "bin/emqx"} , {copy, "bin/emqx_ctl", "bin/emqx_ctl"} , {copy, "bin/node_dump", "bin/node_dump"} @@ -334,9 +342,8 @@ relx_overlay(ReleaseType) -> , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} , {copy, "bin/nodetool", "bin/nodetool"} , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"} - , {copy, "priv/emqx.schema", "releases/{{release_version}}/"} + , {copy, "_build/default/lib/hocon/hocon", "bin/hocon"} + , {copy, "_build/default/lib/hocon/hocon", "bin/hocon-{{release_version}}"} ] ++ case is_enterprise() of true -> ee_etc_overlay(ReleaseType); false -> etc_overlay(ReleaseType) @@ -349,7 +356,6 @@ etc_overlay(ReleaseType) -> [community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()], [ {mkdir, "etc/"} , {mkdir, "etc/plugins"} - , {template, "etc/BUILT_ON", "releases/{{release_version}}/BUILT_ON"} , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"} ] ++ lists:map( @@ -366,18 +372,20 @@ extra_overlay(edge) -> []. emqx_etc_overlay(cloud) -> emqx_etc_overlay_common() ++ - [ {"etc/emqx_cloud/vm.args","etc/vm.args"} + [ {"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args","etc/vm.args"} ]; emqx_etc_overlay(edge) -> emqx_etc_overlay_common() ++ - [ {"etc/emqx_edge/vm.args","etc/vm.args"} + [ {"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args","etc/vm.args"} ]. emqx_etc_overlay_common() -> - ["etc/acl.conf", "etc/emqx.conf", "etc/ssl_dist.conf", + [{"{{base_dir}}/lib/emqx/etc/acl.conf", "etc/acl.conf"}, + {"{{base_dir}}/lib/emqx/etc/emqx.conf", "etc/emqx.conf"}, + {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}, %% TODO: check why it has to end with .paho %% and why it is put to etc/plugins dir - {"etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}]. + {"{{base_dir}}/lib/emqx/etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}]. plugin_etc_overlays(App0) -> App = atom_to_list(App0), diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 580fd5ef2..4fc31ccc6 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -5,22 +5,45 @@ latest_release=$(git describe --abbrev=0 --tags) bad_app_count=0 -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true + fi else - app_path="." + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed="$(git diff --name-only "$latest_release"...HEAD \ +} + +while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff --name-only "$latest_release"...HEAD \ -- "$app_path/src" \ -- "$app_path/priv" \ - -- "$app_path/c_src" | wc -l)" - if [ "$changed" -gt 0 ]; then - echo "$src_file needs a vsn bump" + -- "$app_path/c_src")" + if [ "$lines" != '' ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" bad_app_count=$(( bad_app_count + 1)) fi fi diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index fabec239e..1f199269c 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -10,9 +10,6 @@ find_app() { find "${appdir}" -mindepth 1 -maxdepth 1 -type d } -# append emqx application first -echo 'emqx' - find_app 'apps' if [ -f 'EMQX_ENTERPRISE' ]; then find_app 'lib-ee' diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh index 97939d931..da477694d 100755 --- a/scripts/find-suites.sh +++ b/scripts/find-suites.sh @@ -8,8 +8,5 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." -TESTDIR="test" -if [ "$1" != "emqx" ]; then - TESTDIR="$1/test" -fi +TESTDIR="$1/test" find "${TESTDIR}" -name "*_SUITE.erl" 2>/dev/null | xargs | tr ' ' ',' diff --git a/scripts/split-config.escript b/scripts/split-config.escript index 04f94269c..0a26d6fd2 100755 --- a/scripts/split-config.escript +++ b/scripts/split-config.escript @@ -13,7 +13,7 @@ -define(BASE, <<"emqx">>). main(_) -> - {ok, Bin} = file:read_file("etc/emqx.conf"), + {ok, Bin} = file:read_file(conf_file()), Lines = binary:split(Bin, <<"\n">>, [global]), Sections0 = parse_sections(Lines), {value, _, Sections1} = lists:keytake(<<"modules">>, 1, Sections0), @@ -24,6 +24,10 @@ main(_) -> end, IncludeNames), ok = dump_sections([{N, Base ++ Includes}| Sections2]). +etc_dir() -> filename:join(["apps", "emqx", "etc"]). + +conf_file() -> filename:join([etc_dir(), "emqx.conf"]). + parse_sections(Lines) -> {ok, P} = re:compile("#+\s*CONFIG_SECTION_(BGN|END)\s*=\s*([^\s-]+)\s*="), Parser = @@ -57,7 +61,7 @@ parse_sections([Line | Lines], Parse, Section, Sections) -> dump_sections([]) -> ok; dump_sections([{Name, Lines0} | Rest]) -> - Filename = filename:join(["etc", iolist_to_binary([Name, ".conf.seg"])]), + Filename = filename:join([etc_dir(), iolist_to_binary([Name, ".conf.seg"])]), Lines = [[L, "\n"] || L <- Lines0], ok = file:write_file(Filename, Lines), dump_sections(Rest). diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl deleted file mode 100644 index 893c260ee..000000000 --- a/src/emqx_http_lib.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib). - --export([ uri_encode/1 - , uri_decode/1 - , uri_parse/1 - , normalise_headers/1 - ]). - --export_type([uri_map/0]). - --type uri_map() :: #{scheme := http | https, - host := unicode:chardata(), - port := non_neg_integer(), - path => unicode:chardata(), - query => unicode:chardata(), - fragment => unicode:chardata(), - userinfo => unicode:chardata()}. - --type hex_uri() :: string() | binary(). --type maybe_hex_uri() :: string() | binary(). %% A possibly hexadecimal encoded URI. --type uri() :: string() | binary(). - -%% @doc Decode percent-encoded URI. -%% This is copied from http_uri.erl which has been deprecated since OTP-23 -%% The recommended replacement uri_string function is not quite equivalent -%% and not backward compatible. --spec uri_decode(maybe_hex_uri()) -> uri(). -uri_decode(String) when is_list(String) -> - do_uri_decode(String); -uri_decode(String) when is_binary(String) -> - do_uri_decode_binary(String). - -do_uri_decode([$%,Hex1,Hex2|Rest]) -> - [hex2dec(Hex1)*16+hex2dec(Hex2)|do_uri_decode(Rest)]; -do_uri_decode([First|Rest]) -> - [First|do_uri_decode(Rest)]; -do_uri_decode([]) -> - []. - -do_uri_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> - <<(binary_to_integer(Hex, 16)), (do_uri_decode_binary(Rest))/binary>>; -do_uri_decode_binary(<>) -> - <>; -do_uri_decode_binary(<<>>) -> - <<>>. - -%% @doc Encode URI. --spec uri_encode(uri()) -> hex_uri(). -uri_encode(URI) when is_list(URI) -> - lists:append([do_uri_encode(Char) || Char <- URI]); -uri_encode(URI) when is_binary(URI) -> - << <<(do_uri_encode_binary(Char))/binary>> || <> <= URI >>. - -%% @doc Parse URI into a map as uri_string:uri_map(), but with two fields -%% normalised: (1): port number is never 'undefined', default ports are used -%% if missing. (2): scheme is always atom. --spec uri_parse(string() | binary()) -> {ok, uri_map()} | {error, any()}. -uri_parse(URI) -> - try - {ok, do_parse(uri_string:normalize(URI))} - catch - throw : Reason -> - {error, Reason} - end. - -do_parse({error, Reason, Which}) -> throw({Reason, Which}); -do_parse(URI) -> - %% ensure we return string() instead of binary() in uri_map() values. - Map = uri_string:parse(unicode:characters_to_list(URI)), - case maps:is_key(scheme, Map) of - true -> - normalise_parse_result(Map); - false -> - %% missing scheme, add "http://" and try again - Map2 = uri_string:parse(unicode:characters_to_list(["http://", URI])), - normalise_parse_result(Map2) - end. - -%% @doc Return HTTP headers list with keys lower-cased and -%% underscores replaced with hyphens -%% NOTE: assuming the input Headers list is a proplists, -%% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] -%% results in: [{"content-type", "applicaiton/binary"}] -normalise_headers(Headers0) -> - F = fun({K0, V}) -> - K = re:replace(K0, "_", "-", [{return,list}]), - {string:lowercase(K), V} - end, - Headers = lists:map(F, Headers0), - Keys = proplists:get_keys(Headers), - [{K, proplists:get_value(K, Headers)} || K <- Keys]. - -normalise_parse_result(#{host := Host, scheme := Scheme0} = Map) -> - {Scheme, DefaultPort} = atom_scheme_and_default_port(Scheme0), - Port = case maps:get(port, Map, undefined) of - N when is_number(N) -> N; - _ -> DefaultPort - end, - Map#{ scheme := Scheme - , host := emqx_misc:maybe_parse_ip(Host) - , port => Port - }. - -%% NOTE: so far we only support http/coap schemes. -atom_scheme_and_default_port(Scheme) when is_list(Scheme) -> - atom_scheme_and_default_port(list_to_binary(Scheme)); -atom_scheme_and_default_port(<<"http">> ) -> {http, 80}; -atom_scheme_and_default_port(<<"https">>) -> {https, 443}; -atom_scheme_and_default_port(<<"coap">> ) -> {coap, 5683}; -atom_scheme_and_default_port(<<"coaps">>) -> {coaps, 5684}; -atom_scheme_and_default_port(Other) -> throw({unsupported_scheme, Other}). - -do_uri_encode(Char) -> - case reserved(Char) of - true -> - [ $% | integer_to_hexlist(Char)]; - false -> - [Char] - end. - -do_uri_encode_binary(Char) -> - case reserved(Char) of - true -> - << $%, (integer_to_binary(Char, 16))/binary >>; - false -> - <> - end. - -reserved($;) -> true; -reserved($:) -> true; -reserved($@) -> true; -reserved($&) -> true; -reserved($=) -> true; -reserved($+) -> true; -reserved($,) -> true; -reserved($/) -> true; -reserved($?) -> true; -reserved($#) -> true; -reserved($[) -> true; -reserved($]) -> true; -reserved($<) -> true; -reserved($>) -> true; -reserved($\") -> true; -reserved(${) -> true; -reserved($}) -> true; -reserved($|) -> true; -reserved($\\) -> true; -reserved($') -> true; -reserved($^) -> true; -reserved($%) -> true; -reserved($\s) -> true; -reserved(_) -> false. - -integer_to_hexlist(Int) -> - integer_to_list(Int, 16). - -hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; -hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; -hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl deleted file mode 100644 index 7bcb7d056..000000000 --- a/test/emqx_http_lib_tests.erl +++ /dev/null @@ -1,94 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib_tests). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - -uri_encode_decode_test_() -> - Opts = [{numtests, 1000}, {to_file, user}], - {timeout, 10, - fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}. - -prop_run() -> - ?FORALL(Generated, prop_uri(), test_prop_uri(iolist_to_binary(Generated))). - -prop_uri() -> - proper_types:non_empty(proper_types:list(proper_types:union([prop_char(), prop_reserved()]))). - -prop_char() -> proper_types:integer(32, 126). - -prop_reserved() -> - proper_types:oneof([$;, $:, $@, $&, $=, $+, $,, $/, $?, - $#, $[, $], $<, $>, $\", ${, $}, $|, - $\\, $', $^, $%, $ ]). - -test_prop_uri(URI) -> - Encoded = emqx_http_lib:uri_encode(URI), - Decoded1 = emqx_http_lib:uri_decode(Encoded), - ?assertEqual(URI, Decoded1), - Decoded2 = uri_string:percent_decode(Encoded), - ?assertEqual(URI, Decoded2), - true. - -uri_parse_test_() -> - [ {"default port http", - fun() -> ?assertMatch({ok, #{port := 80, scheme := http, host := "localhost"}}, - emqx_http_lib:uri_parse("localhost")) - end - } - , {"default port https", - fun() -> ?assertMatch({ok, #{port := 443, scheme := https}}, - emqx_http_lib:uri_parse("https://localhost")) - end - } - , {"bad url", - fun() -> ?assertMatch({error, {invalid_uri, _}}, - emqx_http_lib:uri_parse("https://localhost:notnumber")) - end - } - , {"normalise", - fun() -> ?assertMatch({ok, #{scheme := https, host := {127, 0, 0, 1}}}, - emqx_http_lib:uri_parse("HTTPS://127.0.0.1")) - end - } - , {"coap default port", - fun() -> ?assertMatch({ok, #{scheme := coap, port := 5683}}, - emqx_http_lib:uri_parse("coap://127.0.0.1")) - end - } - , {"coaps default port", - fun() -> ?assertMatch({ok, #{scheme := coaps, port := 5684}}, - emqx_http_lib:uri_parse("coaps://127.0.0.1")) - end - } - , {"unsupported_scheme", - fun() -> ?assertEqual({error, {unsupported_scheme, <<"wss">>}}, - emqx_http_lib:uri_parse("wss://127.0.0.1")) - end - } - , {"ipv6 host", - fun() -> ?assertMatch({ok, #{scheme := http, host := T}} when size(T) =:= 8, - emqx_http_lib:uri_parse("http://[::1]:80")) - end - } - ]. - -normalise_headers_test() -> - ?assertEqual([{"content-type", "applicaiton/binary"}], - emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"}, - {"content-type", "applicaiton/json"}])).