From 9db86e7c39c29a60541e917232524930769ad837 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 1 Jul 2022 14:45:59 +0200 Subject: [PATCH 001/105] ci: upload deb and rpm packages from slim build jobs too --- .github/workflows/build_slim_packages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index ba27c1b38..96198f1a3 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -76,7 +76,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} - path: _packages/${{ matrix.profile}}/*.tar.gz + path: _packages/${{ matrix.profile}}/* - uses: actions/upload-artifact@v2 with: name: "${{ matrix.profile }}_schema_dump" @@ -120,7 +120,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: windows - path: _packages/${{ matrix.profile}}/*.tar.gz + path: _packages/${{ matrix.profile}}/* mac: strategy: @@ -196,7 +196,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: macos - path: _packages/**/*.tar.gz + path: _packages/**/* spellcheck: needs: linux From bb854a9667fe97d782b62cd7ca93843df2e15209 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Jul 2022 12:12:31 +0200 Subject: [PATCH 002/105] fix: enable_pipelining should not be allowed to be zero --- apps/emqx_connector/i18n/emqx_connector_http.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/i18n/emqx_connector_http.conf b/apps/emqx_connector/i18n/emqx_connector_http.conf index 0d92c269c..0e29a15d3 100644 --- a/apps/emqx_connector/i18n/emqx_connector_http.conf +++ b/apps/emqx_connector/i18n/emqx_connector_http.conf @@ -76,8 +76,8 @@ base URL 只包含host和port。
enable_pipelining { desc { - en: "Whether to send HTTP requests continuously, when set to 0, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." - zh: "是否连续发送 HTTP 请求，当设置为 0 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" + en: "A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." + zh: "正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" } label: { en: "HTTP Pipelineing" From a95474669e7035a5b46c8c93350df8e1d2e73ab4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Jul 2022 17:57:13 +0200 Subject: [PATCH 003/105] chore: remove duplicated haproxy config section the mgmt and dashbaord endpoints are both merged 18083 now no need to declare twice --- .ci/docker-compose-file/haproxy/haproxy.cfg | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg index 89c1d7d5d..1f10c4f9e 100644 --- a/.ci/docker-compose-file/haproxy/haproxy.cfg +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -30,24 +30,12 @@ defaults ##---------------------------------------------------------------- ## API ##---------------------------------------------------------------- -frontend emqx_mgmt - mode tcp - option tcplog - bind *:18083 - default_backend emqx_mgmt_back - frontend emqx_dashboard mode tcp option tcplog bind *:18083 default_backend emqx_dashboard_back -backend emqx_mgmt_back - mode http - # balance static-rr - server emqx-1 node1.emqx.io:18083 - server emqx-2 node2.emqx.io:18083 - backend emqx_dashboard_back mode http # balance static-rr From b7053fa2626fe0e49beb8923051490e852115aa8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Jul 2022 17:58:02 +0200 Subject: [PATCH 004/105] chore: add haproxy to cluster example script --- .../relup-test/start-relup-test-cluster.sh | 2 +- scripts/start-two-nodes-in-docker.sh | 144 ++++++++++++++++-- 2 files changed, 135 insertions(+), 11 deletions(-) diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index 10bb25a60..b66894c95 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -91,7 +91,7 @@ wait_for_webhook() { while ! curl -f -s localhost:7077; do wait_sec=$(( wait_sec + 1 )) if [ $wait_sec -gt "$wait_limit" ]; then - echo "timeout wait for EMQX" + echo "timeout wait for webhook" exit 1 fi echo -n '.' diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/start-two-nodes-in-docker.sh index ea9029752..16f46a53f 100755 --- a/scripts/start-two-nodes-in-docker.sh +++ b/scripts/start-two-nodes-in-docker.sh @@ -18,6 +18,7 @@ NODE2="node2.$NET" COOKIE='this-is-a-secret' ## clean up +docker rm -f haproxy >/dev/null 2>&1 || true docker rm -f "$NODE1" >/dev/null 2>&1 || true docker rm -f "$NODE2" >/dev/null 2>&1 || true docker network rm "$NET" >/dev/null 2>&1 || true @@ -30,7 +31,10 @@ docker run -d -t --restart=always --name "$NODE1" \ -e EMQX_NODE_NAME="emqx@$NODE1" \ -e EMQX_NODE_COOKIE="$COOKIE" \ -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \ - -p 18083:18083 \ + -e EMQX_listeners__ssl__default__enable=false \ + -e EMQX_listeners__wss__default__enable=false \ + -e EMQX_listeners__tcp__default__proxy_protocol=true \ + -e EMQX_listeners__ws__default__proxy_protocol=true \ "$IMAGE" docker run -d -t --restart=always --name "$NODE2" \ @@ -39,19 +43,139 @@ docker run -d -t --restart=always --name "$NODE2" \ -e EMQX_NODE_NAME="emqx@$NODE2" \ -e EMQX_NODE_COOKIE="$COOKIE" \ -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \ - -p 18084:18083 \ + -e EMQX_listeners__ssl__default__enable=false \ + -e EMQX_listeners__wss__default__enable=false \ + -e EMQX_listeners__tcp__default__proxy_protocol=true \ + -e EMQX_listeners__ws__default__proxy_protocol=true \ "$IMAGE" -wait () { - container="$1" - while ! docker exec "$container" emqx_ctl status >/dev/null 2>&1; do - echo -n '.' - sleep 1 - done +mkdir -p tmp +cat < tmp/haproxy.cfg +##---------------------------------------------------------------- +## global 2021/04/05 +##---------------------------------------------------------------- +global + log stdout format raw daemon debug + # Replace 1024000 with deployment connections + maxconn 1000 + nbproc 1 + nbthread 2 + cpu-map auto:1/1-2 0-1 + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP + # Enable the HAProxy Runtime API + # e.g. echo "show table emqx_tcp_back" | sudo socat stdio tcp4-connect:172.100.239.4:9999 + stats socket :9999 level admin expose-fd listeners + +##---------------------------------------------------------------- +## defaults +##---------------------------------------------------------------- +defaults + log global + mode tcp + option tcplog + # Replace 1024000 with deployment connections + maxconn 1000 + timeout connect 30000 + timeout client 600s + timeout server 600s + +##---------------------------------------------------------------- +## API +##---------------------------------------------------------------- +frontend emqx_dashboard + mode tcp + option tcplog + bind *:18083 + default_backend emqx_dashboard_back + +backend emqx_dashboard_back + mode http + # balance static-rr + server emqx-1 $NODE1:18083 + server emqx-2 $NODE2:18083 + +##---------------------------------------------------------------- +## TLS +##---------------------------------------------------------------- +frontend emqx_ssl + mode tcp + option tcplog + bind *:8883 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_ssl_back + +frontend emqx_wss + mode tcp + option tcplog + bind *:8084 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_wss_back + +backend emqx_ssl_back + mode tcp + balance static-rr + server emqx-1 $NODE1:1883 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 $NODE2:1883 check-send-proxy send-proxy-v2-ssl-cn + +backend emqx_wss_back + mode tcp + balance static-rr + server emqx-1 $NODE1:8083 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 $NODE2:8083 check-send-proxy send-proxy-v2-ssl-cn +EOF + + +docker run -d --name haproxy \ + --net "$NET" \ + -v "$(pwd)/tmp/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" \ + -v "$(pwd)/apps/emqx/etc/certs:/usr/local/etc/haproxy/certs" \ + -w /usr/local/etc/haproxy \ + -p 18083:18083 \ + -p 8883:8883 \ + -p 8084:8084 \ + "haproxy:2.4" \ + bash -c 'set -euo pipefail; + cat certs/cert.pem certs/key.pem > /tmp/emqx.pem; + haproxy -f haproxy.cfg' + +wait_limit=60 +wait_for_emqx() { + container="$1" + wait_limit="$2" + wait_sec=0 + while ! docker exec "$container" emqx_ctl status >/dev/null 2>&1; do + wait_sec=$(( wait_sec + 1 )) + if [ $wait_sec -gt "$wait_limit" ]; then + echo "timeout wait for EMQX" + exit 1 + fi + echo -n '.' + sleep 1 + done } -wait $NODE1 -wait $NODE2 +wait_for_haproxy() { + wait_sec=0 + wait_limit="$1" + set +x + while ! openssl s_client \ + -CAfile apps/emqx/etc/certs/cacert.pem \ + -cert apps/emqx/etc/certs/cert.pem \ + -key apps/emqx/etc/certs/key.pem \ + localhost:8084 Date: Fri, 8 Jul 2022 15:30:46 +0200 Subject: [PATCH 005/105] fix: upgrade jq lib to get rid of crash when space in path This commit upgrades the jq library to a version that has fixed an issue that caused a crash when the port program had a space in its path. --- CHANGES-5.0.md | 1 + mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 70d9f7d7b..a94d88d88 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -8,6 +8,7 @@ if it's a HTTPs server, we didn't check if TLS handshake was successful. [commits/6b45d2ea](https://github.com/emqx/emqx/commit/6b45d2ea9fde6d3b4a5b007f7a8c5a1c573d141e) * The `create_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) +* The rule engine's jq function now works even when the path to the EMQX install dir contains spaces # 5.0.3 diff --git a/mix.exs b/mix.exs index 0636b7a07..b907e5329 100644 --- a/mix.exs +++ b/mix.exs @@ -577,7 +577,7 @@ defmodule EMQXUmbrella.MixProject do defp jq_dep() do if enable_jq?(), - do: [{:jq, github: "emqx/jq", tag: "v0.3.4", override: true}], + do: [{:jq, github: "emqx/jq", tag: "v0.3.5", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index a32495844..af5eafc25 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -41,7 +41,7 @@ quicer() -> {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.14"}}}. jq() -> - {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.4"}}}. + {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.5"}}}. deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), From e49686a27624c07832b13f91ba14af90fc7fbe4e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Jul 2022 10:31:43 +0200 Subject: [PATCH 006/105] fix(authz): should apply no rule on superuser --- CHANGES-5.0.md | 48 +++++++++---------- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz.erl | 33 +++++++++++-- .../emqx_authz/test/emqx_authz_file_SUITE.erl | 27 ++++++++++- 4 files changed, 80 insertions(+), 30 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index a94d88d88..f6a26cd29 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -8,20 +8,20 @@ if it's a HTTPs server, we didn't check if TLS handshake was successful. [commits/6b45d2ea](https://github.com/emqx/emqx/commit/6b45d2ea9fde6d3b4a5b007f7a8c5a1c573d141e) * The `create_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) -* The rule engine's jq function now works even when the path to the EMQX install dir contains spaces +* Avoid apply any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) # 5.0.3 ## Bug fixes -* Websocket listener failed to read headers `X-Forwared-For` and `X-Forwarded-Port` [8415](https://github.com/emqx/emqx/pull/8415) -* Deleted `cluster_singleton` from MQTT bridge config document. This config is no longer applicable in 5.0 [8407](https://github.com/emqx/emqx/pull/8407) -* Fix `emqx/emqx:latest` docker image publish to use the Erlang flavor, but not Elixir flavor [8414](https://github.com/emqx/emqx/pull/8414) -* Changed the `exp` field in JWT auth to be optional rather than required to fix backwards compatability with 4.X releases. [8425](https://github.com/emqx/emqx/pull/8425) +* Websocket listener failed to read headers `X-Forwared-For` and `X-Forwarded-Port` [#8415](https://github.com/emqx/emqx/pull/8415) +* Deleted `cluster_singleton` from MQTT bridge config document. This config is no longer applicable in 5.0 [#8407](https://github.com/emqx/emqx/pull/8407) +* Fix `emqx/emqx:latest` docker image publish to use the Erlang flavor, but not Elixir flavor [#8414](https://github.com/emqx/emqx/pull/8414) +* Changed the `exp` field in JWT auth to be optional rather than required to fix backwards compatability with 4.X releases. [#8425](https://github.com/emqx/emqx/pull/8425) ## Enhancements -* Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [8438](https://github.com/emqx/emqx/pull/8438) +* Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) # 5.0.2 @@ -37,8 +37,8 @@ This had been the biggest obstacle for EMQX team to act agile enough in delivery ## Bug fixes -* Fixed a typo in `bin/emqx` which affects MacOs release when trying to enable Erlang distribution over TLS [8398](https://github.com/emqx/emqx/pull/8398) -* Restricted shell was accidentally disabled in 5.0.1, it has been added back. [8396](https://github.com/emqx/emqx/pull/8396) +* Fixed a typo in `bin/emqx` which affects MacOs release when trying to enable Erlang distribution over TLS [#8398](https://github.com/emqx/emqx/pull/8398) +* Restricted shell was accidentally disabled in 5.0.1, it has been added back. [#8396](https://github.com/emqx/emqx/pull/8396) # 5.0.1 @@ -67,25 +67,25 @@ Exceptions: ## Enhancements -* Removed management API auth for prometheus scraping endpoint /api/v5/prometheus/stats [8299](https://github.com/emqx/emqx/pull/8299) -* Added more TCP options for exhook (gRPC) connections. [8317](https://github.com/emqx/emqx/pull/8317) -* HTTP Servers used for authentication and authorization will now indicate the result via the response body. [8374](https://github.com/emqx/emqx/pull/8374) [8377](https://github.com/emqx/emqx/pull/8377) -* Bulk subscribe/unsubscribe APIs [8356](https://github.com/emqx/emqx/pull/8356) -* Added exclusive subscription [8315](https://github.com/emqx/emqx/pull/8315) -* Provide authentication counter metrics [8352](https://github.com/emqx/emqx/pull/8352) [8375](https://github.com/emqx/emqx/pull/8375) -* Do not allow admin user self-deletion [8286](https://github.com/emqx/emqx/pull/8286) -* After restart, ensure to copy `cluster-override.conf` from the clustered node which has the greatest `tnxid`. [8333](https://github.com/emqx/emqx/pull/8333) +* Removed management API auth for prometheus scraping endpoint /api/v5/prometheus/stats [#8299](https://github.com/emqx/emqx/pull/8299) +* Added more TCP options for exhook (gRPC) connections. [#8317](https://github.com/emqx/emqx/pull/8317) +* HTTP Servers used for authentication and authorization will now indicate the result via the response body. [#8374](https://github.com/emqx/emqx/pull/8374) [#8377](https://github.com/emqx/emqx/pull/8377) +* Bulk subscribe/unsubscribe APIs [#8356](https://github.com/emqx/emqx/pull/8356) +* Added exclusive subscription [#8315](https://github.com/emqx/emqx/pull/8315) +* Provide authentication counter metrics [#8352](https://github.com/emqx/emqx/pull/8352) [#8375](https://github.com/emqx/emqx/pull/8375) +* Do not allow admin user self-deletion [#8286](https://github.com/emqx/emqx/pull/8286) +* After restart, ensure to copy `cluster-override.conf` from the clustered node which has the greatest `tnxid`. [#8333](https://github.com/emqx/emqx/pull/8333) ## Bug fixes -* A bug fix ported from 4.x: allow deleting subscriptions from `client.subscribe` hookpoint callback result. [8304](https://github.com/emqx/emqx/pull/8304) [8347](https://github.com/emqx/emqx/pull/8377) -* Fixed Erlang distribution over TLS [8309](https://github.com/emqx/emqx/pull/8309) -* Made possible to override authentication configs from environment variables [8323](https://github.com/emqx/emqx/pull/8309) -* Made authentication passwords in Mnesia database backward compatible to 4.x, so we can support data migration better. [8351](https://github.com/emqx/emqx/pull/8351) -* Fix plugins upload for rpm/deb installations [8379](https://github.com/emqx/emqx/pull/8379) -* Sync data/authz/acl.conf and data/certs from clustered nodes after a new node joins the cluster [8369](https://github.com/emqx/emqx/pull/8369) -* Ensure auto-retry of failed resources [8371](https://github.com/emqx/emqx/pull/8371) -* Fix the issue that the count of `packets.connack.auth_error` is inaccurate when the client uses a protocol version below MQTT v5.0 to access [8178](https://github.com/emqx/emqx/pull/8178) +* A bug fix ported from 4.x: allow deleting subscriptions from `client.subscribe` hookpoint callback result. [#8304](https://github.com/emqx/emqx/pull/8304) [#8347](https://github.com/emqx/emqx/pull/8377) +* Fixed Erlang distribution over TLS [#8309](https://github.com/emqx/emqx/pull/8309) +* Made possible to override authentication configs from environment variables [#8323](https://github.com/emqx/emqx/pull/8309) +* Made authentication passwords in Mnesia database backward compatible to 4.x, so we can support data migration better. [#8351](https://github.com/emqx/emqx/pull/8351) +* Fix plugins upload for rpm/deb installations [#8379](https://github.com/emqx/emqx/pull/8379) +* Sync data/authz/acl.conf and data/certs from clustered nodes after a new node joins the cluster [#8369](https://github.com/emqx/emqx/pull/8369) +* Ensure auto-retry of failed resources [#8371](https://github.com/emqx/emqx/pull/8371) +* Fix the issue that the count of `packets.connack.auth_error` is inaccurate when the client uses a protocol version below MQTT v5.0 to access [#8178](https://github.com/emqx/emqx/pull/8178) ## Others diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index d1d976959..ed19b15a8 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 1880e3898..f7ebcece0 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -53,11 +53,12 @@ -type sources() :: [source()]. +-define(METRIC_SUPERUSER, 'authorization.superuser'). -define(METRIC_ALLOW, 'authorization.matched.allow'). -define(METRIC_DENY, 'authorization.matched.deny'). -define(METRIC_NOMATCH, 'authorization.nomatch'). --define(METRICS, [?METRIC_ALLOW, ?METRIC_DENY, ?METRIC_NOMATCH]). +-define(METRICS, [?METRIC_SUPERUSER, ?METRIC_ALLOW, ?METRIC_DENY, ?METRIC_NOMATCH]). -define(IS_ENABLED(Enable), ((Enable =:= true) or (Enable =:= <<"true">>))). @@ -308,6 +309,30 @@ authorize( Topic, DefaultResult, Sources +) -> + case maps:get(is_superuser, Client, false) of + true -> + log_allowed(#{ + username => Username, + ipaddr => IpAddress, + topic => Topic, + is_superuser => true + }), + emqx_metrics:inc(?METRIC_SUPERUSER), + {stop, allow}; + false -> + authorize_non_superuser(Client, PubSub, Topic, DefaultResult, Sources) + end. + +authorize_non_superuser( + #{ + username := Username, + peerhost := IpAddress + } = Client, + PubSub, + Topic, + DefaultResult, + Sources ) -> case do_authorize(Client, PubSub, Topic, sources_with_defaults(Sources)) of {{matched, allow}, AuthzSource} -> @@ -315,8 +340,7 @@ authorize( 'client.check_authz_complete', [Client, PubSub, Topic, allow, AuthzSource] ), - ?SLOG(info, #{ - msg => "authorization_permission_allowed", + log_allowed(#{ username => Username, ipaddr => IpAddress, topic => Topic, @@ -356,6 +380,9 @@ authorize( {stop, DefaultResult} end. +log_allowed(Meta) -> + ?SLOG(info, Meta#{msg => "authorization_permission_allowed"}). + do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; do_authorize(Client, PubSub, Topic, [#{enable := false} | Rest]) -> diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 9e7767d1e..059a350e2 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -84,8 +84,6 @@ t_ok(_Config) -> <<"rules">> => <<"{allow, {user, \"username\"}, publish, [\"t\"]}.">> }), - io:format("~p", [emqx_authz:acl_conf_file()]), - ?assertEqual( allow, emqx_access_control:authorize(ClientInfo, publish, <<"t">>) @@ -96,6 +94,31 @@ t_ok(_Config) -> emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) ). +t_superuser(_Config) -> + ClientInfo = #{ + clientid => <<"clientid">>, + username => <<"username">>, + is_superuser => true, + peerhost => {127, 0, 0, 1}, + zone => default, + listener => {tcp, default} + }, + + %% no rules apply to superuser + ok = setup_config(?RAW_SOURCE#{ + <<"rules">> => <<"{deny, {user, \"username\"}, publish, [\"t\"]}.">> + }), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + ), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) + ). + t_invalid_file(_Config) -> ?assertMatch( {error, bad_acl_file_content}, From 49d008371f9f6b6ebe844f171b38b13feaa6cd86 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Jul 2022 11:04:06 +0200 Subject: [PATCH 007/105] chore: bump version to 5.0.4 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index e1c065e3e..75d5852c9 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.3"). +-define(EMQX_RELEASE_CE, "5.0.4"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). From 8bf5c20bdf57fc6dcba1dd3d4d0f8237afc3cdcc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 8 Jul 2022 13:16:38 +0200 Subject: [PATCH 008/105] chore: refine CHANGES-5.0.md --- CHANGES-5.0.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index f6a26cd29..22231044a 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -7,14 +7,15 @@ Prior to this change, the webhook only checks the connectivity of the TCP port using `gen_tcp:connect/2`, so if it's a HTTPs server, we didn't check if TLS handshake was successful. [commits/6b45d2ea](https://github.com/emqx/emqx/commit/6b45d2ea9fde6d3b4a5b007f7a8c5a1c573d141e) -* The `create_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) -* Avoid apply any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) +* The `created_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) +* The rule engine's jq function now works even when the path to the EMQX install dir contains spaces [jq#35](https://github.com/emqx/jq/pull/35) [#8455](https://github.com/emqx/emqx/pull/8455) +* Avoid applying any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) # 5.0.3 ## Bug fixes -* Websocket listener failed to read headers `X-Forwared-For` and `X-Forwarded-Port` [#8415](https://github.com/emqx/emqx/pull/8415) +* Websocket listener failed to read headers `X-Forwarded-For` and `X-Forwarded-Port` [#8415](https://github.com/emqx/emqx/pull/8415) * Deleted `cluster_singleton` from MQTT bridge config document. This config is no longer applicable in 5.0 [#8407](https://github.com/emqx/emqx/pull/8407) * Fix `emqx/emqx:latest` docker image publish to use the Erlang flavor, but not Elixir flavor [#8414](https://github.com/emqx/emqx/pull/8414) * Changed the `exp` field in JWT auth to be optional rather than required to fix backwards compatability with 4.X releases. [#8425](https://github.com/emqx/emqx/pull/8425) From 5bd1c4803c6a32168bf3e8481bb2f61a46be56a5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 10 Jul 2022 10:01:01 +0200 Subject: [PATCH 009/105] chore: Update license file --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index ac9372d97..2a081b135 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ Source code in this repository is variously licensed under below licenses. -For EMQX Community Edition: Apache License 2.0, see APL.txt, +For EMQX: Apache License 2.0, see APL.txt, which applies to all source files except for lib-ee sub-directory. -For EMQX Enterprise Edition (since version 5.0): Business Source License 1.1, +For EMQX Enterprise (since version 5.0): Business Source License 1.1, see lib-ee/BSL.txt, which applies to source code in lib-ee sub-directory. From bcc16230ccce8bfca04f50f0b6c9381b76a274a6 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 12 Jul 2022 13:55:36 +0800 Subject: [PATCH 010/105] fix(stats): fix topic name error --- apps/emqx/src/emqx_sys.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index 284fefac2..20421235b 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -333,7 +333,7 @@ publish(brokers, Nodes) -> safe_publish(<<"$SYS/brokers">>, #{retain => true}, Payload); publish(stats, Stats) -> [ - safe_publish(systop(lists:concat(['stats/', Stat])), integer_to_binary(Val)) + safe_publish(systop(stats_topic(Stat)), integer_to_binary(Val)) || {Stat, Val} <- Stats, is_atom(Stat), is_integer(Val) ]; publish(metrics, Metrics) -> @@ -351,7 +351,13 @@ publish(Event, Payload) when safe_publish(Topic, emqx_json:encode(Payload)). metric_topic(Name) -> - lists:concat(["metrics/", string:replace(atom_to_list(Name), ".", "/", all)]). + translate_topic("metrics/", Name). + +stats_topic(Name) -> + translate_topic("stats/", Name). + +translate_topic(Prefix, Name) -> + lists:concat([Prefix, string:replace(atom_to_list(Name), ".", "/", all)]). safe_publish(Topic, Payload) -> safe_publish(Topic, #{}, Payload). From aed4f5fee6fefbc8d6c2c73dc6141dec2e22f703 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 13 Jul 2022 09:26:57 +0800 Subject: [PATCH 011/105] chore: update CHANGES-5.0 --- CHANGES-5.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 22231044a..60651311e 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -10,6 +10,7 @@ * The `created_at` field of rules is missing after emqx restarts. [commits/5fc09e6b](https://github.com/emqx/emqx/commit/5fc09e6b950c340243d7be627a0ce1700691221c) * The rule engine's jq function now works even when the path to the EMQX install dir contains spaces [jq#35](https://github.com/emqx/jq/pull/35) [#8455](https://github.com/emqx/emqx/pull/8455) * Avoid applying any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) +* Fix statistics related system topic name error # 5.0.3 From 59c301243c53822d589854af3d10a1c58f21730c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 11 Jul 2022 11:34:49 +0800 Subject: [PATCH 012/105] fix(connector): redis sentinel field required --- apps/emqx_connector/src/emqx_connector_redis.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index f70093c4e..67310dbac 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -90,6 +90,7 @@ fields(sentinel) -> }}, {sentinel, #{ type => string(), + required => true, desc => ?DESC("sentinel_desc") }} ] ++ From 994a76510d0b56d784d65db1c03d89f4c8626e11 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 11 Jul 2022 16:33:01 +0800 Subject: [PATCH 013/105] fix(authn): jwt ssl opts use emqx_schema:ssl_clients_opts --- .../src/simple_authn/emqx_authn_jwt.erl | 42 ++----------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 9f7e5f0a8..0017754a4 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -75,26 +75,11 @@ fields('jwks') -> {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {refresh_interval, fun refresh_interval/1}, {ssl, #{ - type => hoconsc:union([ - hoconsc:ref(?MODULE, ssl_enable), - hoconsc:ref(?MODULE, ssl_disable) - ]), - desc => ?DESC(ssl), + type => hoconsc:ref(emqx_schema, "ssl_client_opts"), default => #{<<"enable">> => false}, - required => false + desc => ?DESC("ssl") }} - ] ++ common_fields(); -fields(ssl_enable) -> - [ - {enable, #{type => true, desc => ?DESC(enable)}}, - {cacertfile, fun cacertfile/1}, - {certfile, fun certfile/1}, - {keyfile, fun keyfile/1}, - {verify, fun verify/1}, - {server_name_indication, fun server_name_indication/1} - ]; -fields(ssl_disable) -> - [{enable, #{type => false, desc => ?DESC(enable)}}]. + ] ++ common_fields(). desc('hmac-based') -> ?DESC('hmac-based'); @@ -147,27 +132,6 @@ refresh_interval(default) -> 300; refresh_interval(validator) -> [fun(I) -> I > 0 end]; refresh_interval(_) -> undefined. -cacertfile(type) -> string(); -cacertfile(desc) -> ?DESC(?FUNCTION_NAME); -cacertfile(_) -> undefined. - -certfile(type) -> string(); -certfile(desc) -> ?DESC(?FUNCTION_NAME); -certfile(_) -> undefined. - -keyfile(type) -> string(); -keyfile(desc) -> ?DESC(?FUNCTION_NAME); -keyfile(_) -> undefined. - -verify(type) -> hoconsc:enum([verify_peer, verify_none]); -verify(desc) -> ?DESC(?FUNCTION_NAME); -verify(default) -> verify_none; -verify(_) -> undefined. - -server_name_indication(type) -> string(); -server_name_indication(desc) -> ?DESC(?FUNCTION_NAME); -server_name_indication(_) -> undefined. - verify_claims(type) -> list(); verify_claims(desc) -> From ab17fd80e7b0055f2a6b686ed14d7b253844f143 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 12 Jul 2022 14:08:13 +0800 Subject: [PATCH 014/105] ci: common test add redis-sentinel --- .ci/docker-compose-file/Makefile.local | 4 ++++ .../docker-compose-redis-sentinel-tcp.yaml | 4 ++-- .../docker-compose-redis-sentinel-tls.yaml | 4 ++-- .../docker-compose-redis-single-tcp.yaml | 2 +- .ci/docker-compose-file/redis/redis.sh | 6 +++++- .github/workflows/run_test_cases.yaml | 2 ++ 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local index 8b8c6af68..026cc7a1d 100644 --- a/.ci/docker-compose-file/Makefile.local +++ b/.ci/docker-compose-file/Makefile.local @@ -26,6 +26,8 @@ up: -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ up -d --build down: @@ -39,6 +41,8 @@ down: -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ down ct: diff --git a/.ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml b/.ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml index 1cdd28726..07c6cfb0a 100644 --- a/.ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml @@ -1,8 +1,8 @@ version: '3.9' services: - redis_server: - container_name: redis + redis_sentinel_server: + container_name: redis-sentinel image: redis:${REDIS_TAG} volumes: - ./redis/:/data/conf diff --git a/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml b/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml index 045570d5c..b9eaefa9c 100644 --- a/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml @@ -1,8 +1,8 @@ version: '3.9' services: - redis_server: - container_name: redis + redis_sentinel_server_tls: + container_name: redis-sentinel-tls image: redis:${REDIS_TAG} volumes: - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt diff --git a/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml b/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml index 5fa9f0749..6706fe84f 100644 --- a/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml @@ -2,7 +2,7 @@ version: '3.9' services: redis_server: - container_name: redis + container_name: redis image: redis:${REDIS_TAG} ports: - "6379:6379" diff --git a/.ci/docker-compose-file/redis/redis.sh b/.ci/docker-compose-file/redis/redis.sh index 6cc7ce98b..b7cf62a60 100755 --- a/.ci/docker-compose-file/redis/redis.sh +++ b/.ci/docker-compose-file/redis/redis.sh @@ -16,11 +16,15 @@ case $key in shift # past argument shift # past value ;; - -t|--tls-enabled) + -t) tls="$2" shift # past argument shift # past value ;; + --tls-enabled) + tls=1 + shift # past argument + ;; *) shift # past argument ;; diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 44e8bd3c4..e08e3906b 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -118,6 +118,8 @@ jobs: -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build From f42c8ffcfa730e730438d91e7e1f2aaa0103892b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 11 Jul 2022 17:55:39 +0800 Subject: [PATCH 015/105] test(connector): redis sentinel include name --- apps/emqx/test/emqx_common_test_helpers.erl | 13 +++++ .../test/emqx_connector_redis_SUITE.erl | 52 ++++++++++++++----- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index f591d75bd..d0ee99d6d 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -44,6 +44,7 @@ client_ssl_twoway/1, ensure_mnesia_stopped/0, ensure_quic_listener/2, + is_all_tcp_servers_available/1, is_tcp_server_available/2, is_tcp_server_available/3, load_config/2, @@ -432,6 +433,18 @@ load_config(SchemaModule, Config, Opts) -> load_config(SchemaModule, Config) -> load_config(SchemaModule, Config, #{raw_with_default => false}). +-spec is_all_tcp_servers_available(Servers) -> Result when + Servers :: [{Host, Port}], + Host :: inet:socket_address() | inet:hostname(), + Port :: inet:port_number(), + Result :: boolean(). +is_all_tcp_servers_available(Servers) -> + Fun = + fun({Host, Port}) -> + is_tcp_server_available(Host, Port) + end, + lists:all(Fun, Servers). + -spec is_tcp_server_available( Host :: inet:socket_address() | inet:hostname(), Port :: inet:port_number() diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 8c54b7224..4770bbeee 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -23,8 +23,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(REDIS_HOST, "redis"). --define(REDIS_PORT, 6379). +-define(REDIS_SINGLE_HOST, "redis"). +-define(REDIS_SINGLE_PORT, 6379). +-define(REDIS_SENTINEL_HOST, "redis-sentinel"). +-define(REDIS_SENTINEL_PORT, 26379). -define(REDIS_RESOURCE_MOD, emqx_connector_redis). all() -> @@ -34,7 +36,14 @@ groups() -> []. init_per_suite(Config) -> - case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_PORT) of + case + emqx_common_test_helpers:is_all_tcp_servers_available( + [ + {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT}, + {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT} + ] + ) + of true -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_connector]), @@ -141,20 +150,35 @@ redis_config_cluster() -> redis_config_sentinel() -> redis_config_base("sentinel", "servers"). +-define(REDIS_CONFIG_BASE(MaybeSentinel), + "" ++ + "\n" ++ + " auto_reconnect = true\n" ++ + " database = 1\n" ++ + " pool_size = 8\n" ++ + " redis_type = ~s\n" ++ + MaybeSentinel ++ + " password = public\n" ++ + " ~s = \"~s:~b\"\n" ++ + " " ++ + "" +). + redis_config_base(Type, ServerKey) -> + case Type of + "sentinel" -> + Host = ?REDIS_SENTINEL_HOST, + Port = ?REDIS_SENTINEL_PORT, + MaybeSentinel = " sentinel = mymaster\n"; + _ -> + Host = ?REDIS_SINGLE_HOST, + Port = ?REDIS_SINGLE_PORT, + MaybeSentinel = "" + end, RawConfig = list_to_binary( io_lib:format( - "" - "\n" - " auto_reconnect = true\n" - " database = 1\n" - " pool_size = 8\n" - " redis_type = ~s\n" - " password = public\n" - " ~s = \"~s:~b\"\n" - " " - "", - [Type, ServerKey, ?REDIS_HOST, ?REDIS_PORT] + ?REDIS_CONFIG_BASE(MaybeSentinel), + [Type, ServerKey, Host, Port] ) ), From 24f24361da0af485f9dc8196febbee3d3d3c3521 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 12 Jul 2022 18:30:47 +0800 Subject: [PATCH 016/105] chore: update CHANGES.md --- CHANGES-5.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 60651311e..7622ff69f 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -11,6 +11,8 @@ * The rule engine's jq function now works even when the path to the EMQX install dir contains spaces [jq#35](https://github.com/emqx/jq/pull/35) [#8455](https://github.com/emqx/emqx/pull/8455) * Avoid applying any ACL checks on superusers [#8452](https://github.com/emqx/emqx/pull/8452) * Fix statistics related system topic name error +* Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) +* `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) # 5.0.3 From 8f18bdff15b83b8fcb0d96c2943e17510900038d Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:35:33 +0800 Subject: [PATCH 017/105] fix(api): topic metrics check exist before create --- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_modules/src/emqx_topic_metrics_api.erl | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 6565e6dfd..cd107d78c 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {modules, []}, {applications, [kernel, stdlib, emqx]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index 74a37b652..1fb9c3a7b 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -296,11 +296,16 @@ topic_metrics(put, #{body := #{<<"action">> := <<"reset">>}}) -> topic_metrics(post, #{body := #{<<"topic">> := <<>>}}) -> {400, 'BAD_REQUEST', <<"Topic can not be empty">>}; topic_metrics(post, #{body := #{<<"topic">> := Topic}}) -> - case emqx_modules_conf:add_topic_metrics(Topic) of - {ok, Topic} -> - get_cluster_response([Topic]); - {error, Reason} -> - reason2httpresp(Reason) + case lists:member(Topic, emqx_modules_conf:topic_metrics()) of + false -> + case emqx_modules_conf:add_topic_metrics(Topic) of + {ok, Topic} -> + get_cluster_response([Topic]); + {error, Reason} -> + reason2httpresp(Reason) + end; + true -> + reason2httpresp(already_existed) end. operate_topic_metrics(get, #{bindings := #{topic := Topic}}) -> From 941440800bacf06e4fca33739375614ce079b0d5 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 13 Jul 2022 15:22:13 +0800 Subject: [PATCH 018/105] fix(cli): subscriptions with sub options, qos rh rap nl; admins response data format --- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../emqx_dashboard/src/emqx_dashboard_cli.erl | 24 +++++++++++++++---- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 7 +++++- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index c694ab9f8..c3b7b4f13 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl index 677121d93..6f99b761b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl @@ -32,14 +32,22 @@ admins(["add", Username, Password, Desc]) -> {ok, _} -> emqx_ctl:print("ok~n"); {error, Reason} -> - emqx_ctl:print("Error: ~p~n", [Reason]) + print_error(Reason) end; admins(["passwd", Username, Password]) -> - Status = emqx_dashboard_admin:change_password(bin(Username), bin(Password)), - emqx_ctl:print("~p~n", [Status]); + case emqx_dashboard_admin:change_password(bin(Username), bin(Password)) of + {ok, _} -> + emqx_ctl:print("ok~n"); + {error, Reason} -> + print_error(Reason) + end; admins(["del", Username]) -> - Status = emqx_dashboard_admin:remove_user(bin(Username)), - emqx_ctl:print("~p~n", [Status]); + case emqx_dashboard_admin:remove_user(bin(Username)) of + {ok, _} -> + emqx_ctl:print("ok~n"); + {error, Reason} -> + print_error(Reason) + end; admins(_) -> emqx_ctl:usage( [ @@ -53,3 +61,9 @@ unload() -> emqx_ctl:unregister_command(admins). bin(S) -> iolist_to_binary(S). + +print_error(Reason) when is_binary(Reason) -> + emqx_ctl:print("Error: ~s~n", [Reason]). +%% Maybe has more types of error, but there is only binary now. So close it for dialyzer. +% print_error(Reason) -> +% emqx_ctl:print("Error: ~p~n", [Reason]). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 1cc0a477b..e5b769b6f 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 0f60317fa..e6730a18c 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -780,7 +780,12 @@ print({emqx_topic, #route{topic = Topic, dest = {_, Node}}}) -> print({emqx_topic, #route{topic = Topic, dest = Node}}) -> emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]); print({emqx_suboption, {{Pid, Topic}, Options}}) when is_pid(Pid) -> - emqx_ctl:print("~ts -> ~ts~n", [maps:get(subid, Options), Topic]). + SubId = maps:get(subid, Options), + QoS = maps:get(qos, Options, 0), + NL = maps:get(nl, Options, 0), + RH = maps:get(rh, Options, 0), + RAP = maps:get(rap, Options, 0), + emqx_ctl:print("~ts -> topic:~ts qos:~p nl:~p rh:~p rap:~p~n", [SubId, Topic, QoS, NL, RH, RAP]). format(_, undefined) -> undefined; From 421251f4c8484f1120f22357149cf2c76ca20c19 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 14 Jul 2022 09:43:28 +0800 Subject: [PATCH 019/105] chore: improve the dashboard listener startup log --- CHANGES-5.0.md | 5 +++++ apps/emqx/src/emqx_listeners.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 17 +++-------------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 7622ff69f..3ba0add04 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -14,6 +14,11 @@ * Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) * `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) +## Enhancements + +* Improve the dashboard listener startup log, the listener name is no longer spliced with port information, + and the colon(:) is no longer displayed when IP is not specified.[#8480](https://github.com/emqx/emqx/pull/8480) + # 5.0.3 ## Bug fixes diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 61ed6c47d..3c508bacc 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -493,7 +493,7 @@ merge_default(Options) -> end. format_addr(Port) when is_integer(Port) -> - io_lib:format(":~w", [Port]); + io_lib:format("~w", [Port]); %% Print only the port number when bound on all interfaces format_addr({{0, 0, 0, 0}, Port}) -> format_addr(Port); diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 0e106f7a2..042c9d5d8 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -159,7 +159,7 @@ listeners(Listeners) -> maps:get(enable, Conf) andalso begin {Conf1, Bind} = ip_port(Conf), - {true, {listener_name(Protocol, Conf1), Protocol, Bind, ranch_opts(Conf1)}} + {true, {listener_name(Protocol), Protocol, Bind, ranch_opts(Conf1)}} end end, maps:to_list(Listeners) @@ -208,19 +208,8 @@ ranch_opts(Options) -> filter_false(_K, false, S) -> S; filter_false(K, V, S) -> [{K, V} | S]. -listener_name(Protocol, #{port := Port, ip := IP}) -> - Name = - "dashboard:" ++ - atom_to_list(Protocol) ++ ":" ++ - inet:ntoa(IP) ++ ":" ++ - integer_to_list(Port), - list_to_atom(Name); -listener_name(Protocol, #{port := Port}) -> - Name = - "dashboard:" ++ - atom_to_list(Protocol) ++ ":" ++ - integer_to_list(Port), - list_to_atom(Name). +listener_name(Protocol) -> + list_to_atom(atom_to_list(Protocol) ++ ":dashboard"). authorize(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of From 9afad90b9142a31f009945ac14bf4d2a53498c75 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 14 Jul 2022 10:40:03 +0800 Subject: [PATCH 020/105] fix(retainer): set default value of flow control --- apps/emqx_retainer/src/emqx_retainer_schema.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 74f1c6287..526059c9e 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -36,7 +36,8 @@ fields("retainer") -> {flow_control, sc( ?R_REF(flow_control), - flow_control + flow_control, + #{} )}, {max_payload_size, sc( @@ -104,8 +105,8 @@ desc(_) -> %% Internal functions %%-------------------------------------------------------------------- -sc(Type, DescId) -> - hoconsc:mk(Type, #{desc => ?DESC(DescId)}). +%%sc(Type, DescId) -> +%% hoconsc:mk(Type, #{desc => ?DESC(DescId)}). sc(Type, DescId, Default) -> hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId)}). From be6f63d927f5ed0611871457f8ebb275638a9aa9 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 14 Jul 2022 14:03:25 +0800 Subject: [PATCH 021/105] chore(retainer): update app version --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 122895afc..1640f4cc9 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx]}, From 26820ed0c43f592762570de76b0fe8585980b903 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 14 Jul 2022 15:10:50 +0800 Subject: [PATCH 022/105] chore: remove /configs/listeners, we should use /listeners/ API --- CHANGES-5.0.md | 3 ++- apps/emqx_dashboard/src/emqx_dashboard.appup.src | 14 ++++---------- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 3 ++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 3ba0add04..32ab74076 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,7 +17,8 @@ ## Enhancements * Improve the dashboard listener startup log, the listener name is no longer spliced with port information, - and the colon(:) is no longer displayed when IP is not specified.[#8480](https://github.com/emqx/emqx/pull/8480) + and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) +* Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) # 5.0.3 diff --git a/apps/emqx_dashboard/src/emqx_dashboard.appup.src b/apps/emqx_dashboard/src/emqx_dashboard.appup.src index 2b1053380..a1a6f4e79 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.appup.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.appup.src @@ -1,13 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"5.0.0", - [{load_module,emqx_dashboard,brutal_purge,soft_purge,[]}, - {load_module,emqx_dashboard_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_dashboard_token,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"5.0.0", - [{load_module,emqx_dashboard,brutal_purge,soft_purge,[]}, - {load_module,emqx_dashboard_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_dashboard_token,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + %% we should always restart dashboard to make sure api rules/swagger is updated + [{<<".*">>,[{restart_application, emqx_dashboard}]}], + [{<<".*">>,[{restart_application, emqx_dashboard}]}] +}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 3354d50e6..ab18ec488 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -62,7 +62,8 @@ <<"prometheus">>, <<"telemetry">>, <<"sys_topics">>, - <<"limiter">> + <<"limiter">>, + <<"listeners">> ] ++ global_zone_roots() ). From 15ebef446817019ca621911744b0fcb53b27aecf Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 15 Jul 2022 15:23:31 +0800 Subject: [PATCH 023/105] chore: improve metrics interval desc --- .../i18n/emqx_prometheus_schema_i18n.conf | 4 ++-- .../i18n/emqx_statsd_schema_i18n.conf | 24 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf index f49d8d00c..2c7736938 100644 --- a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf +++ b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf @@ -20,8 +20,8 @@ emqx_prometheus_schema { interval { desc { - en: """Data reporting interval, in milliseconds.""" - zh: """数据推送间隔，单位 毫秒""" + en: """Data reporting interval""" + zh: """数据推送间隔""" } } enable { diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf index bc0e1d3b0..4ccad1682 100644 --- a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf +++ b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf @@ -2,40 +2,40 @@ emqx_statsd_schema { statsd { desc { - en: """Settings for reporting metrics to Statsd""" - zh: """Statsd 监控数据推送""" + en: """Settings for reporting metrics to StatsD""" + zh: """StatsD 监控数据推送""" } label { - en: """Statsd""" - zh: """Statsd""" + en: """StatsD""" + zh: """StatsD""" } } server { desc { - en: """URL of Statsd server""" - zh: """Statsd 服务器地址""" + en: """URL of StatsD server""" + zh: """StatsD 服务器地址""" } } sample_interval { desc { - en: """Data collection interval in second.""" - zh: """数据收集间隔，单位 毫秒""" + en: """Data collection interval.""" + zh: """数据收集间隔""" } } flush_interval { desc { - en: """Data reporting interval, in second.""" - zh: """数据推送间隔，单位 毫秒""" + en: """Data reporting interval.""" + zh: """数据推送间隔""" } } enable { desc { - en: """Turn Statsd data pushing on or off""" - zh: """开启或关闭 Statsd 数据推送""" + en: """Turn StatsD data pushing on or off""" + zh: """开启或关闭 StatsD 数据推送""" } } } From 29ea3162910b180dd17e8edde89fe60951f7b594 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 15 Jul 2022 15:24:15 +0800 Subject: [PATCH 024/105] chore: upgrade dashboard to v1.0.3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 09582709e..a18cbee10 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.2 +export EMQX_DASHBOARD_VERSION ?= v1.0.3 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 354c1458975165380f04567782f8b17fab68600d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Lindh=C3=A9?= Date: Mon, 11 Jul 2022 15:37:47 +0200 Subject: [PATCH 025/105] docs(docker): Fix typos in Docker README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capitalize: * EMQX * Linux * MQTT * TCP * Redis 💡 git show --color-words='\w+' Fix typos: * EMQ --> EMQX * Properly mark variables and filenames as code 💡 git show --word-diff-regex='.' Miscellaneous: * Rephrase to fix grammar mistakes. * Remove redundant blank line 💡 git show --color-words='\w+' Fix `markdownlint` warnings: * MD012/no-multiple-blanks Multiple consecutive blank lines * MD025/single-title/single-h1 Multiple top-level headings in the same document * MD034/no-bare-urls Bare URL used --- deploy/docker/README.md | 51 +++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index c07f8fe8b..d2d729b5d 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -1,31 +1,32 @@ -# Quick reference +# EMQX + +## Quick reference + **Where to get help**: - https://emqx.io or https://github.com/emqx/emqx + or + **Where to file issues:** - https://github.com/emqx/emqx/issues + + **Supported architectures** `amd64`, `arm64v8` - + **Supported Docker versions**: [the latest release](https://github.com/docker/docker-ce/releases/latest) -# What is EMQX +## What is EMQX [EMQX MQTT broker](https://emqx.io/products/broker) is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster. -# How to use this image +## How to use this image -### Run emqx +### Run EMQX Execute some command under this docker image @@ -35,7 +36,7 @@ For example ``docker run -d --name emqx -p 18083:18083 -p 1883:1883 emqx/emqx:latest`` -The emqx broker runs as linux user `emqx` in the docker container. +The EMQX broker runs as Linux user `emqx` in the docker container. ### Configuration @@ -43,7 +44,7 @@ Use the environment variable to configure the EMQX docker container. By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files. -You can change the prefix by overriding "HOCON_ENV_OVERRIDE_PREFIX". +You can change the prefix by overriding `HOCON_ENV_OVERRIDE_PREFIX`. Example: @@ -74,26 +75,26 @@ These environment variables will ignore for configuration file. #### EMQX Configuration -> NOTE: All EMQX Configuration in [etc/emqx.conf](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) could config by environment. The following list is just an example, not a complete configuration. +> NOTE: All EMQX Configuration in [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) can be configured via environment variables. The following list is just an example, not a complete configuration. | Options | Default | Mapped | Description | | ---------------------------| ------------------ | ------------------------- | ------------------------------------- | -| EMQX_NAME | container name | none | emqx node short name | -| EMQX_HOST | container IP | none | emqx node host, IP or FQDN | +| `EMQX_NAME` | container name | none | EMQX node short name | +| `EMQX_HOST` | container IP | none | EMQX node host, IP or FQDN | -The list is incomplete and may changed with [etc/emqx.conf](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. +The list is incomplete and may changed with [`etc/emqx.conf`](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE_NAME``, ``EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST``. -For example, set mqtt tcp port to 1883 +For example, set MQTT TCP port to 1883 ``docker run -d --name emqx -e EMQX__LISTENERS__TCP__DEFAULT__BIND=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest`` -#### EMQ Loaded Modules Configuration +#### EMQX Loaded Modules Configuration | Oprtions | Default | Description | | ------------------------ | ------------------ | ------------------------------------- | -| EMQX_LOADED_MODULES | see content below | default modules emqx loaded | +| `EMQX_LOADED_MODULES` | see content below | default EMQX loaded modules | Default environment variable ``EMQX_LOADED_MODULES``, including @@ -116,11 +117,11 @@ EMQX_LOADED_MODULES="emqx_mod_delayed emqx_mod_rewrite" EMQX_LOADED_MODULES="emqx_mod_delayed | emqx_mod_rewrite" ``` -#### EMQ Loaded Plugins Configuration +#### EMQX Loaded Plugins Configuration | Oprtions | Default | Description | | ------------------------ | ------------------ | ------------------------------------- | -| EMQX_LOADED_PLUGINS | see content below | default plugins emqx loaded | +| `EMQX_LOADED_PLUGINS` | see content below | default EMQX loaded plugins | Default environment variable ``EMQX_LOADED_PLUGINS``, including @@ -149,7 +150,7 @@ EMQX_LOADED_PLUGINS="emqx_retainer | emqx_rule_engine" #### EMQX Plugins Configuration -The environment variables which with ``EMQX_`` prefix are mapped to all emqx plugins' configuration file, ``.`` get replaced by ``__``. +The environment variables which with ``EMQX_`` prefix are mapped to all EMQX plugins' configuration file, ``.`` get replaced by ``__``. Example: @@ -158,11 +159,11 @@ EMQX_RETAINER__STORAGE_TYPE <--> retainer.storage_type EMQX_RETAINER__MAX_PAYLOAD_SIZE <--> retainer.max_payload_size ``` -Don't worry about where to find the configuration file of emqx plugins, this docker image will find and config them automatically using some magic. +Don't worry about where to find the configuration file of EMQX plugins, this docker image will find and configure them automatically using some magic. -All plugin of emqx project could config in this way, following the environment variables mapping rule above. +All EMQX plugins can be configured this way, following the environment variables mapping rule above. -Assume you are using redis auth plugin, for example: +Assume you are using Redis auth plugin, for example: ```bash #EMQX_RETAINER__STORAGE_TYPE = "ram" @@ -181,7 +182,7 @@ For numbered configuration options where the number is next to a ``.`` such as: + backend.redis.pool1.server + backend.redis.hook.message.publish.1 -You can configure an arbitrary number of them as long as each has a uniq unber for it's own configuration option: +You can configure an arbitrary number of them as long as each has a unique number for its own configuration option: ```bash docker run -d --name emqx -p 18083:18083 -p 1883:1883 \ @@ -288,9 +289,9 @@ services: ### Kernel Tuning -Under linux host machine, the easiest way is [Tuning guide](https://www.emqx.io/docs/en/latest/tutorial/tune.html). +Under Linux host machine, the easiest way is [Tuning guide](https://www.emqx.io/docs/en/latest/tutorial/tune.html). -If you want tune linux kernel by docker, you must ensure your docker is latest version (>=1.12). +If you want tune Linux kernel by docker, you must ensure your docker is latest version (>=1.12). ```bash From a6f14c255d2255269831095677c977117012276a Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 13 Jul 2022 21:48:44 +0200 Subject: [PATCH 026/105] feat: add emqx_cluster_rescue --- bin/emqx_cluster_rescue | 187 ++++++++++++++++++++++++++++++++++++++++ mix.exs | 8 ++ rebar.config.erl | 1 + 3 files changed, 196 insertions(+) create mode 100755 bin/emqx_cluster_rescue diff --git a/bin/emqx_cluster_rescue b/bin/emqx_cluster_rescue new file mode 100755 index 000000000..c060f2d0e --- /dev/null +++ b/bin/emqx_cluster_rescue @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +set -euo pipefail +# ================================== +# RESCUE THE UNBOOTABLE EMQX CLUSTER +# ================================== + +## Global Vars +# Steal from emqx_ctl +THIS_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")" || true; pwd -P)" + +usage() { + local Script + Script=$(basename "$0") + + echo " + RESCUE THE UNBOOTABLE EMQX CLUSTER + + Use this script only when the entire cluster is stuck at booting & loading. + + This script provides a list of methods to *hack* the DB of EMQX to bring back + the cluster back to service but MAY come with some side effects including: + + - Data loss + - Inconsistent data in the cluster + - Other undefined behaviors + + *DO NOT* use this script unless you understand the consequences. + *DO NOT* use this script when EMQX cluster is partitioned. + + Use Case: + + - Lost one node due to unrecoverable failures (hardware, cloud resource outage) + and this node prevents other nodes in the cluster from starting. + +Usage: + + # For troubleshooting, find out all the tables that are pending at loading + $Script pending-tables + + # For troubleshooting, debug print detailed table info that is pending at loading. + $Script table-details + + # Force load one [Tab] or all pending tables from node local storage to bring this node up + # Use local data as the data source for the pending tables, should bring up the node immediately and + # spread the data to other nodes in the cluster. + # + # * Take effect immediately + # * This is a node local change but the change will be lost after restart. + $Script force-load [Tab] + + # Remove Node from mnesia cluster. + # Most likely will fail if the remote Node is unreachable. + # + # * This is a cluster wide schema change. + $Script remove-node Node + + # Set master node for distributed DB + # The master node will be the data source for pending tables. + # + # * This is a node local change + # * Node could be a remote Erlang node in the cluster or local erlang node + # * Use command: 'unset-master' to rollback + $Script set-master Node + + # Unset master node for distributed DB, this is a node local change + $Script unset-master + + # Cheat the local node that RemoteNode is down so that it will not wait for it to come up. + # Local node will take local data as the data source for pending tables and spread the data + # to the other pending nodes. + # + # * Check EMQX logs to find out which remote node(s) the local node is waiting for + # * To take effect, restart this EMQX node + # * This is a node local setting + + $Script lie-node-down RemoteNode + +Tips: + - Override local node name with envvar: \$EMQX_NODE__NAME + " +} + +# Functions +# +print_pending_tables() { + local erl_cmd='[ io:format("~p :: ~p~n", [T, maps:with([all_nodes, load_order, storage_type, + active_replicas, local_content, load_by_force, + load_node, load_reason, master_nodes] + , maps:from_list(mnesia:table_info(T, all)))]) + || T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node) ], + ok + ' + exec "$THIS_DIR/emqx" eval "$erl_cmd" +} + +print_details_per_table() { + local erl_cmd='[ io:format("~p :: ~p~n", [T, mnesia:table_info(T, all)]) + || T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node) ], + ok + ' + exec "$THIS_DIR/emqx" eval "$erl_cmd" +} + +force-load() { + if [ $# -eq 1 ]; then + local erl_cmd="mnesia:force_load_table(${1})" + else + local erl_cmd='[ {T, mnesia:force_load_table(T)} + || T <- mnesia:system_info(local_tables), + unknown =:= mnesia:table_info(T, load_node) + ] + ' + fi + exec "$THIS_DIR/emqx" eval "$erl_cmd" +} + +remove-node() { + local target_node=$1 + local erl_cmd=" + case [T || T <- mnesia:system_info(local_tables), unknown =:= mnesia:table_info(T, load_node)] of + [] -> + io:format(\"No table need to load\\n\"), + skipped; + TargetTables -> + io:format(\"Going to remove node ${target_node} from schema of the tables:~n~p~n\", [TargetTables]), + case io:read(\"confirm? [yes.] OR Ctrl-D to skip: \") of + {ok, yes} -> + lists:map(fun(T) -> + mnesia:force_load_table(T), + {T, mnesia:del_table_copy(T, '${target_node}') } + end, TargetTables); + eof -> skipped; + R -> {skipped, R} + end + end + " + exec "$THIS_DIR/emqx" eval "$erl_cmd" +} + +set-master-node() { + if [ $# -eq 1 ]; then + local erl_cmd="mnesia:set_master_nodes(['${1}']), mnesia_recover:dump_decision_tab()" + else + local erl_cmd="mnesia:set_master_nodes([]), mnesia_recover:dump_decision_tab()" + fi + + exec "$THIS_DIR/emqx" eval "$erl_cmd" +} + +lie-node-down() { + if [ $# -eq 1 ]; then + local erl_cmd="mnesia_recover:log_mnesia_down('${1}'), mnesia_recover:dump_decision_tab()" + exec "$THIS_DIR/emqx" eval "$erl_cmd" + else + usage + fi +} + + +CMD=${1:-usage} +[ $# -gt 0 ] && shift 1 + +case "$CMD" in + force-load) + force-load "$@" + ;; + remove-node) + remove-node "$@" + ;; + pending-tables) + print_pending_tables + ;; + table-details) + print_details_per_table + ;; + set-master) + set-master-node "$@" + ;; + unset-master) + set-master-node + ;; + lie-node-down) + lie-node-down "$@" + ;; + *) + usage +esac diff --git a/mix.exs b/mix.exs index b907e5329..07362581c 100644 --- a/mix.exs +++ b/mix.exs @@ -408,6 +408,14 @@ defmodule EMQXUmbrella.MixProject do File.chmod!(Path.join(bin, "node_dump"), 0o755) + Mix.Generator.copy_file( + "bin/emqx_cluster_rescue", + Path.join(bin, "emqx_cluster_rescue"), + force: overwrite? + ) + + File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755) + render_template( "rel/BUILD_INFO", assigns, diff --git a/rebar.config.erl b/rebar.config.erl index af5eafc25..8b42d3ce3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -380,6 +380,7 @@ relx_overlay(ReleaseType, Edition) -> {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"}, {copy, "bin/emqx", "bin/emqx"}, {copy, "bin/emqx_ctl", "bin/emqx_ctl"}, + {copy, "bin/emqx_cluster_rescue", "bin/emqx_cluster_rescue"}, {copy, "bin/node_dump", "bin/node_dump"}, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}, %% for relup From 4481ae6ff8e11521a2d0ade963d535a87941bb99 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 15 Jul 2022 16:07:17 +0200 Subject: [PATCH 027/105] feat(quicer): bump to 0.0.16 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 3dfe4f1d7..75f748017 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.14"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/mix.exs b/mix.exs index 07362581c..b681c0960 100644 --- a/mix.exs +++ b/mix.exs @@ -592,7 +592,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.14", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.16", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 8b42d3ce3..d88574a14 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -38,7 +38,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.14"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.5"}}}. From 6f00f5145718776adbe974ce28f4d4e3a9682978 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:30:26 +0200 Subject: [PATCH 028/105] chore(ekka): Bump version --- CHANGES-5.0.md | 1 + apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 32ab74076..6fe656bde 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -32,6 +32,7 @@ ## Enhancements * Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) +* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.2 diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 888ddb15d..0c0a09cc0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 07362581c..55d3529f1 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.13.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, diff --git a/rebar.config b/rebar.config index d7d168f7f..7d2570636 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} From f5d63c1555ef2685972ddb931aa0d0a476044d58 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 18 Jul 2022 15:11:25 +0200 Subject: [PATCH 029/105] fix(conf): Wait for the cluster RPC shard --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 ++ apps/emqx_conf/src/emqx_conf.app.src | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 2478a2540..01d20bb34 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -262,6 +262,8 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> + %% Workaround for https://github.com/emqx/mria/issues/94: + mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index a4946e8cf..1441a4180 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, From 1ef898d126b1076058c5edbcc54eba38012ee4ba Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 18 Jul 2022 17:11:37 -0300 Subject: [PATCH 030/105] ci(rocksdb): conditionally load mnesia_rocksdb in release Otherwise, it might not be included in the release, since it's no longer declared as a dependency on the mria app. --- mix.exs | 72 +++++++++++++++++++++++++++++------------------- rebar.config.erl | 61 ++++++++++++++++++++++------------------ 2 files changed, 78 insertions(+), 55 deletions(-) diff --git a/mix.exs b/mix.exs index 55d3529f1..614fcfdba 100644 --- a/mix.exs +++ b/mix.exs @@ -170,35 +170,40 @@ defmodule EMQXUmbrella.MixProject do hocon: :load, emqx: :load, emqx_conf: :load, - emqx_machine: :permanent, - mria: :load, - mnesia: :load, - ekka: :load, - emqx_plugin_libs: :load, - esasl: :load, - observer_cli: :permanent, - system_monitor: :load, - emqx_http_lib: :permanent, - emqx_resource: :permanent, - emqx_connector: :permanent, - emqx_authn: :permanent, - emqx_authz: :permanent, - emqx_auto_subscribe: :permanent, - emqx_gateway: :permanent, - emqx_exhook: :permanent, - emqx_bridge: :permanent, - emqx_rule_engine: :permanent, - emqx_modules: :permanent, - emqx_management: :permanent, - emqx_dashboard: :permanent, - emqx_retainer: :permanent, - emqx_statsd: :permanent, - emqx_prometheus: :permanent, - emqx_psk: :permanent, - emqx_slow_subs: :permanent, - emqx_plugins: :permanent, - emqx_mix: :none + emqx_machine: :permanent ] ++ + if(enable_rocksdb?(), + do: [mnesia_rocksdb: :load], + else: [] + ) ++ + [ + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + esasl: :load, + observer_cli: :permanent, + system_monitor: :load, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_retainer: :permanent, + emqx_statsd: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent, + emqx_mix: :none + ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ if(enable_jq?(), do: [jq: :permanent], else: []) ++ @@ -616,6 +621,11 @@ defmodule EMQXUmbrella.MixProject do ]) or "1" == System.get_env("BUILD_WITH_QUIC") end + defp enable_rocksdb?() do + not build_without_rocksdb?() or + "1" == System.get_env("BUILD_WITH_QUIC") + end + defp pkg_vsn() do %{edition_type: edition_type} = check_profile!() basedir = Path.dirname(__ENV__.file) @@ -657,6 +667,12 @@ defmodule EMQXUmbrella.MixProject do String.downcase(opt) != "false" end + defp build_without_rocksdb?() do + opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false") + + String.downcase(opt) != "false" + end + defp from_rebar_to_eex_template(str) do # we must not consider surrounding space in the template var name # because some help strings contain informative variables that diff --git a/rebar.config.erl b/rebar.config.erl index 8b42d3ce3..d4adfaabc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -88,6 +88,10 @@ is_quicer_supported() -> is_win32() orelse is_centos_6()) orelse "1" == os:getenv("BUILD_WITH_QUIC"). +is_rocksdb_supported() -> + not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB")) orelse + "1" == os:getenv("BUILD_WITH_ROCKSDB"). + is_macos() -> {unix, darwin} =:= os:type(). @@ -318,34 +322,37 @@ relx_apps(ReleaseType, Edition) -> % started by emqx_machine {emqx, load}, {emqx_conf, load}, - emqx_machine, - {mnesia, load}, - {ekka, load}, - {emqx_plugin_libs, load}, - {esasl, load}, - observer_cli, - % started by emqx_machine - {system_monitor, load}, - emqx_http_lib, - emqx_resource, - emqx_connector, - emqx_authn, - emqx_authz, - emqx_auto_subscribe, - emqx_gateway, - emqx_exhook, - emqx_bridge, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard, - emqx_retainer, - emqx_statsd, - emqx_prometheus, - emqx_psk, - emqx_slow_subs, - emqx_plugins + emqx_machine ] ++ + [{mnesia_rocksdb, load} || is_rocksdb_supported()] ++ + [ + {mnesia, load}, + {ekka, load}, + {emqx_plugin_libs, load}, + {esasl, load}, + observer_cli, + % started by emqx_machine + {system_monitor, load}, + emqx_http_lib, + emqx_resource, + emqx_connector, + emqx_authn, + emqx_authz, + emqx_auto_subscribe, + emqx_gateway, + emqx_exhook, + emqx_bridge, + emqx_rule_engine, + emqx_modules, + emqx_management, + emqx_dashboard, + emqx_retainer, + emqx_statsd, + emqx_prometheus, + emqx_psk, + emqx_slow_subs, + emqx_plugins + ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ [jq || is_jq_supported()] ++ From a05b7bf555d4c6bd6e611cb2cff02dfae825e9a1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 19 Jul 2022 10:07:01 +0800 Subject: [PATCH 031/105] fix: dialyzer warning --- CHANGES-5.0.md | 8 ++++---- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 6fe656bde..fcf2154d5 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -19,6 +19,7 @@ * Improve the dashboard listener startup log, the listener name is no longer spliced with port information, and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) +* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.3 @@ -32,17 +33,16 @@ ## Enhancements * Improve the speed of dashboard's HTTP API routing rule generation, which sometimes causes timeout [#8438](https://github.com/emqx/emqx/pull/8438) -* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.2 -Announcemnet: EMQX team has decided to stop supporting relup for opensouce edition. -Going forward, it will be an enterprise only feature. +Announcement: EMQX team has decided to stop supporting relup for opensource edition. +Going forward, it will be an enterprise-only feature. Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions. For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain. -This had been the biggest obstacle for EMQX team to act agile enough in deliverying enhancements and fixes. +This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes. ## Enhancements diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 01d20bb34..6353a4efa 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -263,7 +263,7 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> %% Workaround for https://github.com/emqx/mria/issues/94: - mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), + _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, From 54367ea76478e62c701aeb36da91b13fa397ef16 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 19 Jul 2022 08:59:22 +0200 Subject: [PATCH 032/105] docs: add a note to Windows.md --- Windows.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Windows.md b/Windows.md index 1e72b1982..a3e5deb11 100644 --- a/Windows.md +++ b/Windows.md @@ -66,6 +66,9 @@ Cygwin is what we tested with. Start (restart) CMD or powershell console and execute `which bash`, it should print out `/usr/bin/bash` +NOTE: Make sure cygwin's bin dir is added before `C:\Windows\system32` in `Path`, +otherwise the build scripts may end up using binaries from wsl instead of cygwin. + ### Other tools Some of the unix world tools are required to build EMQX. Including: From ed9ba0d843c227a4e52f1b5e73097a24be13acbb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Jul 2022 11:52:17 -0300 Subject: [PATCH 033/105] chore: do not include rocksdb on raspbian --- mix.exs | 10 ++++++++-- rebar.config.erl | 11 ++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index f4ed355ed..41ce54972 100644 --- a/mix.exs +++ b/mix.exs @@ -622,8 +622,10 @@ defmodule EMQXUmbrella.MixProject do end defp enable_rocksdb?() do - not build_without_rocksdb?() or - "1" == System.get_env("BUILD_WITH_QUIC") + not Enum.any?([ + build_without_rocksdb?(), + raspbian?() + ]) or "1" == System.get_env("BUILD_WITH_ROCKSDB") end defp pkg_vsn() do @@ -655,6 +657,10 @@ defmodule EMQXUmbrella.MixProject do {:unix, :darwin} == :os.type() end + defp raspbian?() do + os_cmd("./scripts/get-distro.sh", []) =~ "raspbian" + end + defp build_without_jq?() do opt = System.get_env("BUILD_WITHOUT_JQ", "false") diff --git a/rebar.config.erl b/rebar.config.erl index 810c16207..6748f7ce8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -89,7 +89,8 @@ is_quicer_supported() -> "1" == os:getenv("BUILD_WITH_QUIC"). is_rocksdb_supported() -> - not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB")) orelse + not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB") orelse + is_raspbian()) orelse "1" == os:getenv("BUILD_WITH_ROCKSDB"). is_macos() -> @@ -105,6 +106,14 @@ is_centos_6() -> false end. +is_raspbian() -> + case os_cmd("./scripts/get-distro.sh") of + "raspbian" ++ _ -> + true; + _ -> + false + end. + is_win32() -> win32 =:= element(1, os:type()). From d3f965dfe793e279fdf77afe7fe833b5eda1008c Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 15 Jul 2022 17:57:26 +0800 Subject: [PATCH 034/105] refactor(limiter): refactor the user interface --- apps/emqx/i18n/emqx_limiter_i18n.conf | 22 +- apps/emqx/src/emqx_channel.erl | 10 +- apps/emqx/src/emqx_connection.erl | 8 +- .../src/emqx_esockd_htb_limiter.erl | 16 +- .../src/emqx_limiter_container.erl | 44 +-- .../emqx_limiter/src/emqx_limiter_manager.erl | 50 +-- .../emqx_limiter/src/emqx_limiter_schema.erl | 83 ++--- .../emqx_limiter/src/emqx_limiter_server.erl | 204 +++++------ apps/emqx/src/emqx_listeners.erl | 59 +++- apps/emqx/src/emqx_schema.erl | 6 +- apps/emqx/src/emqx_ws_connection.erl | 11 +- apps/emqx/test/emqx_channel_SUITE.erl | 113 ++---- apps/emqx/test/emqx_connection_SUITE.erl | 38 +- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 331 +++++++++--------- apps/emqx/test/emqx_ws_connection_SUITE.erl | 61 +++- apps/emqx_retainer/src/emqx_retainer.erl | 7 +- .../src/emqx_retainer_dispatcher.erl | 8 +- .../src/emqx_retainer_schema.erl | 2 +- .../test/emqx_retainer_SUITE.erl | 66 ++-- 19 files changed, 577 insertions(+), 562 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index 6fbc923b7..b75435225 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -124,20 +124,6 @@ the check/consume will succeed, but it will be forced to wait for a short period } } - batch { - desc { - en: """The batch limiter. -This is used for EMQX internal batch operation -e.g. limit the retainer's deliver rate""" - zh: """批量操作速率控制器。 -这是给 EMQX 内部的批量操作使用的，比如用来控制保留消息的派发速率""" - } - label: { - en: """Batch""" - zh: """批量操作""" - } - } - message_routing { desc { en: """The message routing limiter. @@ -193,4 +179,12 @@ Once the limit is reached, the restricted client will be slow down even be hung zh: """流入字节率""" } } + + internal { + desc { + en: """Limiter for EMQX internal app.""" + zh: """EMQX 内部功能所用限制器。""" + + } + } } diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index d6f2b87ea..cdd4b1a9e 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -252,11 +252,12 @@ init( <<>> -> undefined; MP -> MP end, + ListenerId = emqx_listeners:listener_id(Type, Listener), ClientInfo = set_peercert_infos( Peercert, #{ zone => Zone, - listener => emqx_listeners:listener_id(Type, Listener), + listener => ListenerId, protocol => Protocol, peerhost => PeerHost, sockport => SockPort, @@ -278,7 +279,9 @@ init( outbound => #{} }, auth_cache = #{}, - quota = emqx_limiter_container:get_limiter_by_names([?LIMITER_ROUTING], LimiterCfg), + quota = emqx_limiter_container:get_limiter_by_types( + ListenerId, [?LIMITER_ROUTING], LimiterCfg + ), timers = #{}, conn_state = idle, takeover = false, @@ -1199,9 +1202,6 @@ handle_call( disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> {reply, emqx_authz_cache:list_authz_cache(), Channel}; -handle_call({quota, Bucket}, #channel{quota = Quota} = Channel) -> - Quota2 = emqx_limiter_container:update_by_name(message_routing, Bucket, Quota), - reply(ok, Channel#channel{quota = Quota2}); handle_call( {keepalive, Interval}, Channel = #channel{ diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 59248a0b8..1caf345e6 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -321,7 +321,7 @@ init_state( }, LimiterTypes = [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], - Limiter = emqx_limiter_container:get_limiter_by_names(LimiterTypes, LimiterCfg), + Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg), FrameOpts = #{ strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), @@ -672,12 +672,6 @@ handle_call(_From, info, State) -> {reply, info(State), State}; handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, {ratelimit, Changes}, State = #state{limiter = Limiter}) -> - Fun = fun({Type, Bucket}, Acc) -> - emqx_limiter_container:update_by_name(Type, Bucket, Acc) - end, - Limiter2 = lists:foldl(Fun, Limiter, Changes), - {reply, ok, State#state{limiter = Limiter2}}; handle_call(_From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of {reply, Reply, NChannel} -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl index c39cf2728..16f7b03c8 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl @@ -19,12 +19,13 @@ -behaviour(esockd_generic_limiter). %% API --export([new_create_options/2, create/1, delete/1, consume/2]). +-export([new_create_options/3, create/1, delete/1, consume/2]). -type create_options() :: #{ module := ?MODULE, + id := emqx_limiter_schema:limiter_id(), type := emqx_limiter_schema:limiter_type(), - bucket := emqx_limiter_schema:bucket_name() + bucket := hocons:config() }. %%-------------------------------------------------------------------- @@ -32,15 +33,16 @@ %%-------------------------------------------------------------------- -spec new_create_options( + emqx_limiter_schema:limiter_id(), emqx_limiter_schema:limiter_type(), - emqx_limiter_schema:bucket_name() + hocons:config() ) -> create_options(). -new_create_options(Type, BucketName) -> - #{module => ?MODULE, type => Type, bucket => BucketName}. +new_create_options(Id, Type, BucketCfg) -> + #{module => ?MODULE, id => Id, type => Type, bucket => BucketCfg}. -spec create(create_options()) -> esockd_generic_limiter:limiter(). -create(#{module := ?MODULE, type := Type, bucket := BucketName}) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, BucketName), +create(#{module := ?MODULE, id := Id, type := Type, bucket := BucketCfg}) -> + {ok, Limiter} = emqx_limiter_server:connect(Id, Type, BucketCfg), #{module => ?MODULE, name => Type, limiter => Limiter}. delete(_GLimiter) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index f82a97a5a..74b6c7b87 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -22,10 +22,8 @@ %% API -export([ - new/0, new/1, new/2, - get_limiter_by_names/2, + get_limiter_by_types/3, add_new/3, - update_by_name/3, set_retry_context/2, check/3, retry/2, @@ -48,10 +46,10 @@ }. -type future() :: pos_integer(). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type limiter() :: emqx_htb_limiter:limiter(). -type retry_context() :: emqx_htb_limiter:retry_context(). --type bucket_name() :: emqx_limiter_schema:bucket_name(). -type millisecond() :: non_neg_integer(). -type check_result() :: {ok, container()} @@ -64,46 +62,24 @@ %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- --spec new() -> container(). -new() -> - new([]). - -%% @doc generate default data according to the type of limiter --spec new(list(limiter_type())) -> container(). -new(Types) -> - new(Types, #{}). - --spec new( - list(limiter_type()), - #{limiter_type() => emqx_limiter_schema:bucket_name()} -) -> container(). -new(Types, Names) -> - get_limiter_by_names(Types, Names). - %% @doc generate a container %% according to the type of limiter and the bucket name configuration of the limiter %% @end --spec get_limiter_by_names( +-spec get_limiter_by_types( + limiter_id() | {atom(), atom()}, list(limiter_type()), - #{limiter_type() => emqx_limiter_schema:bucket_name()} + #{limiter_type() => hocons:config()} ) -> container(). -get_limiter_by_names(Types, BucketNames) -> +get_limiter_by_types({Type, Listener}, Types, BucketCfgs) -> + Id = emqx_listeners:listener_id(Type, Listener), + get_limiter_by_types(Id, Types, BucketCfgs); +get_limiter_by_types(Id, Types, BucketCfgs) -> Init = fun(Type, Acc) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, BucketNames), + {ok, Limiter} = emqx_limiter_server:connect(Id, Type, BucketCfgs), add_new(Type, Limiter, Acc) end, lists:foldl(Init, #{retry_ctx => undefined}, Types). -%% @doc add the specified type of limiter to the container --spec update_by_name( - limiter_type(), - bucket_name() | #{limiter_type() => bucket_name()}, - container() -) -> container(). -update_by_name(Type, Buckets, Container) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, Buckets), - add_new(Type, Limiter, Container). - -spec add_new(limiter_type(), limiter(), container()) -> container(). add_new(Type, Limiter, Container) -> Container#{ diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index 89148a12c..aca27a6ff 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -24,11 +24,9 @@ %% API -export([ start_link/0, - find_bucket/1, find_bucket/2, - insert_bucket/2, insert_bucket/3, - make_path/2, + delete_bucket/2, post_config_update/5 ]). @@ -50,20 +48,19 @@ format_status/2 ]). --export_type([path/0]). - --type path() :: list(atom()). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type limiter_type() :: emqx_limiter_schema:limiter_type(). --type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type uid() :: {limiter_id(), limiter_type()}. %% counter record in ets table -record(bucket, { - path :: path(), + uid :: uid(), bucket :: bucket_ref() }). -type bucket_ref() :: emqx_limiter_bucket_ref:bucket_ref(). +-define(UID(Id, Type), {Id, Type}). -define(TAB, emqx_limiter_counters). %%-------------------------------------------------------------------- @@ -85,14 +82,10 @@ restart_server(Type) -> stop_server(Type) -> emqx_limiter_server_sup:stop(Type). --spec find_bucket(limiter_type(), bucket_name()) -> +-spec find_bucket(limiter_id(), limiter_type()) -> {ok, bucket_ref()} | undefined. -find_bucket(Type, BucketName) -> - find_bucket(make_path(Type, BucketName)). - --spec find_bucket(path()) -> {ok, bucket_ref()} | undefined. -find_bucket(Path) -> - case ets:lookup(?TAB, Path) of +find_bucket(Id, Type) -> + case ets:lookup(?TAB, ?UID(Id, Type)) of [#bucket{bucket = Bucket}] -> {ok, Bucket}; _ -> @@ -100,20 +93,19 @@ find_bucket(Path) -> end. -spec insert_bucket( + limiter_id(), limiter_type(), - bucket_name(), bucket_ref() ) -> boolean(). -insert_bucket(Type, BucketName, Bucket) -> - inner_insert_bucket(make_path(Type, BucketName), Bucket). +insert_bucket(Id, Type, Bucket) -> + ets:insert( + ?TAB, + #bucket{uid = ?UID(Id, Type), bucket = Bucket} + ). --spec insert_bucket(path(), bucket_ref()) -> true. -insert_bucket(Path, Bucket) -> - inner_insert_bucket(Path, Bucket). - --spec make_path(limiter_type(), bucket_name()) -> path(). -make_path(Type, BucketName) -> - [Type | BucketName]. +-spec delete_bucket(limiter_id(), limiter_type()) -> true. +delete_bucket(Type, Id) -> + ets:delete(?TAB, ?UID(Id, Type)). post_config_update([limiter, Type], _Config, NewConf, _OldConf, _AppEnvs) -> Config = maps:get(Type, NewConf), @@ -159,7 +151,7 @@ init([]) -> set, public, named_table, - {keypos, #bucket.path}, + {keypos, #bucket.uid}, {write_concurrency, true}, {read_concurrency, true}, {heir, erlang:whereis(emqx_limiter_sup), none} @@ -266,9 +258,3 @@ format_status(_Opt, Status) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- --spec inner_insert_bucket(path(), bucket_ref()) -> true. -inner_insert_bucket(Path, Bucket) -> - ets:insert( - ?TAB, - #bucket{path = Path, bucket = Bucket} - ). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 1e4679ee3..61dc95bc7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -31,7 +31,9 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0 + infinity_value/0, + bucket_opts/0, + bucket_opts_meta/0 ]). -define(KILOBYTE, 1024). @@ -41,8 +43,10 @@ | message_in | connection | message_routing - | batch. + %% internal limiter for unclassified resources + | internal. +-type limiter_id() :: atom(). -type bucket_name() :: atom(). -type rate() :: infinity | float(). -type burst_rate() :: 0 | float(). @@ -76,7 +80,7 @@ bucket_name/0 ]). --export_type([limiter_type/0, bucket_path/0]). +-export_type([limiter_id/0, limiter_type/0, bucket_path/0]). -define(UNIT_TIME_IN_MS, 1000). @@ -87,13 +91,13 @@ roots() -> [limiter]. fields(limiter) -> [ {Type, - ?HOCON(?R_REF(limiter_opts), #{ + ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), default => make_limiter_default(Type) })} || Type <- types() ]; -fields(limiter_opts) -> +fields(node_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {burst, @@ -101,38 +105,16 @@ fields(limiter_opts) -> desc => ?DESC(burst), default => 0 })}, - {bucket, - ?HOCON( - ?MAP("bucket_name", ?R_REF(bucket_opts)), - #{ - desc => ?DESC(bucket_cfg), - default => #{<<"default">> => #{}}, - example => #{ - <<"mybucket-name">> => #{ - <<"rate">> => <<"infinity">>, - <<"capcity">> => <<"infinity">>, - <<"initial">> => <<"100">>, - <<"per_client">> => #{<<"rate">> => <<"infinity">>} - } - } - } - )} + {client, ?HOCON(?R_REF(client_opts), #{default => #{}})} ]; fields(bucket_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, - {per_client, - ?HOCON( - ?R_REF(client_bucket), - #{ - default => #{}, - desc => ?DESC(per_client) - } - )} + {client, ?HOCON(?R_REF(client_opts), #{required => false})} ]; -fields(client_bucket) -> +fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => "infinity", desc => ?DESC(rate)})}, {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, @@ -181,15 +163,35 @@ fields(client_bucket) -> desc(limiter) -> "Settings for the rate limiter."; -desc(limiter_opts) -> - "Settings for the limiter."; +desc(node_opts) -> + "Settings for the limiter of the node level."; +desc(node_client_opts) -> + "Settings for the client in the node level."; desc(bucket_opts) -> "Settings for the bucket."; -desc(client_bucket) -> - "Settings for the client bucket."; +desc(client_opts) -> + "Settings for the client in bucket level."; desc(_) -> undefined. +bucket_opts() -> + ?HOCON( + ?MAP("bucket_name", ?R_REF(bucket_opts)), + bucket_opts_meta() + ). + +bucket_opts_meta() -> + #{ + default => #{}, + example => + #{ + <<"rate">> => <<"infinity">>, + <<"capcity">> => <<"infinity">>, + <<"initial">> => <<"100">>, + <<"client">> => #{<<"rate">> => <<"infinity">>} + } + }. + %% default period is 100ms default_period() -> 100. @@ -202,7 +204,7 @@ get_bucket_cfg_path(Type, BucketName) -> [limiter, Type, bucket, BucketName]. types() -> - [bytes_in, message_in, connection, message_routing, batch]. + [bytes_in, message_in, connection, message_routing, internal]. %%-------------------------------------------------------------------- %% Internal functions @@ -323,15 +325,6 @@ apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). make_limiter_default(connection) -> - #{ - <<"rate">> => <<"1000/s">>, - <<"bucket">> => #{ - <<"default">> => - #{ - <<"rate">> => <<"1000/s">>, - <<"capacity">> => 1000 - } - } - }; + #{<<"rate">> => <<"1000/s">>}; make_limiter_default(_) -> #{}. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 519b32eca..939824e02 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -42,11 +42,13 @@ -export([ start_link/2, - connect/2, + connect/3, + add_bucket/3, + del_bucket/2, + get_initial_val/1, whereis/1, info/1, name/1, - get_initial_val/1, restart/1, update_config/2 ]). @@ -83,6 +85,7 @@ -type buckets() :: #{bucket_name() => bucket()}. -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type rate() :: decimal(). -type flow() :: decimal(). -type capacity() :: decimal(). @@ -94,7 +97,7 @@ %% minimum coefficient for overloaded limiter -define(OVERLOAD_MIN_ALLOC, 0.3). --define(CURRYING(X, F2), fun(Y) -> F2(X, Y) end). +-define(COUNTER_SIZE, 8). -export_type([index/0]). -import(emqx_limiter_decimal, [add/2, sub/2, mul/2, put_to_counter/3]). @@ -105,39 +108,53 @@ %% API %%-------------------------------------------------------------------- -spec connect( + limiter_id(), limiter_type(), bucket_name() | #{limiter_type() => bucket_name() | undefined} ) -> {ok, emqx_htb_limiter:limiter()} | {error, _}. %% If no bucket path is set in config, there will be no limit -connect(_Type, undefined) -> +connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; -connect(Type, BucketName) when is_atom(BucketName) -> - case get_bucket_cfg(Type, BucketName) of - undefined -> - ?SLOG(error, #{msg => "bucket_config_not_found", type => Type, bucket => BucketName}), - {error, config_not_found}; - #{ - rate := BucketRate, - capacity := BucketSize, - per_client := #{rate := CliRate, capacity := CliSize} = Cfg - } -> - case emqx_limiter_manager:find_bucket(Type, BucketName) of - {ok, Bucket} -> +connect( + Id, + Type, + #{ + rate := BucketRate, + capacity := BucketSize + } = BucketCfg +) -> + case emqx_limiter_manager:find_bucket(Id, Type) of + {ok, Bucket} -> + case find_client_cfg(Type, BucketCfg) of + #{rate := CliRate, capacity := CliSize} = ClientCfg -> {ok, if CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(Cfg, Bucket); + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); true -> - emqx_htb_limiter:make_ref_limiter(Cfg, Bucket) + emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) end}; - undefined -> - ?SLOG(error, #{msg => "bucket_not_found", type => Type, bucket => BucketName}), - {error, invalid_bucket} - end + {error, invalid_node_cfg} = Error -> + ?SLOG(error, #{msg => "invalid_node_cfg", type => Type, id => Id}), + Error + end; + undefined -> + ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), + {error, invalid_bucket} end; -connect(Type, Paths) -> - connect(Type, maps:get(Type, Paths, undefined)). +connect(Id, Type, Paths) -> + connect(Id, Type, maps:get(Type, Paths, undefined)). + +-spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. +add_bucket(_Id, _Type, undefine) -> + ok; +add_bucket(Id, Type, Cfg) -> + ?CALL(Type, {add_bucket, Id, Cfg}). + +-spec del_bucket(limiter_id(), limiter_type()) -> ok. +del_bucket(Id, Type) -> + ?CALL(Type, {del_bucket, Id}). -spec info(limiter_type()) -> state() | {error, _}. info(Type) -> @@ -213,6 +230,12 @@ handle_call(restart, _From, #{type := Type}) -> handle_call({update_config, Type, Config}, _From, #{type := Type}) -> NewState = init_tree(Type, Config), {reply, ok, NewState}; +handle_call({add_bucket, Id, Cfg}, _From, State) -> + NewState = do_add_bucket(Id, Cfg, State), + {reply, ok, NewState}; +handle_call({del_bucket, Id}, _From, State) -> + NewState = do_del_bucket(Id, State), + {reply, ok, NewState}; handle_call(Req, _From, State) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. @@ -456,24 +479,14 @@ init_tree(Type) when is_atom(Type) -> Cfg = emqx:get_config([limiter, Type]), init_tree(Type, Cfg). -init_tree(Type, #{bucket := Buckets} = Cfg) -> - State = #{ +init_tree(Type, Cfg) -> + #{ type => Type, - root => undefined, - counter => undefined, - index => 1, + root => make_root(Cfg), + counter => counters:new(?COUNTER_SIZE, [write_concurrency]), + index => 0, buckets => #{} - }, - - Root = make_root(Cfg), - {CounterNum, DelayBuckets} = make_bucket(maps:to_list(Buckets), Type, Cfg, 1, []), - - State2 = State#{ - root := Root, - counter := counters:new(CounterNum, [write_concurrency]) - }, - - lists:foldl(fun(F, Acc) -> F(Acc) end, State2, DelayBuckets). + }. -spec make_root(hocons:confg()) -> root(). make_root(#{rate := Rate, burst := Burst}) -> @@ -484,79 +497,50 @@ make_root(#{rate := Rate, burst := Burst}) -> produced => 0.0 }. -make_bucket([{Name, Conf} | T], Type, GlobalCfg, CounterNum, DelayBuckets) -> - Path = emqx_limiter_manager:make_path(Type, Name), - Rate = get_counter_rate(Conf, GlobalCfg), - #{capacity := Capacity} = Conf, - Initial = get_initial_val(Conf), - CounterNum2 = CounterNum + 1, - InitFun = fun(#{name := BucketName} = Bucket, #{buckets := Buckets} = State) -> - {Counter, Idx, State2} = alloc_counter(Path, Rate, Initial, State), - Bucket2 = Bucket#{counter := Counter, index := Idx}, - State2#{buckets := Buckets#{BucketName => Bucket2}} - end, +do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buckets} = State) -> + case maps:get(Id, Buckets, undefined) of + undefined -> + make_bucket(Id, Cfg, State); + Bucket -> + Bucket2 = Bucket#{rate := Rate, capacity := Capacity}, + State#{buckets := Buckets#{Id := Bucket2}} + end. +make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> + add_bucket(Id, Cfg, State#{ + counter => counters:new(?COUNTER_SIZE, [write_concurrency]), + index => 0 + }); +make_bucket( + Id, + #{rate := Rate, capacity := Capacity} = Cfg, + #{type := Type, counter := Counter, index := Index, buckets := Buckets} = State +) -> + NewIndex = Index + 1, + Initial = get_initial_val(Cfg), Bucket = #{ - name => Name, + name => Id, rate => Rate, obtained => Initial, correction => 0, capacity => Capacity, - counter => undefined, - index => undefined + counter => Counter, + index => NewIndex }, + _ = put_to_counter(Counter, NewIndex, Initial), + Ref = emqx_limiter_bucket_ref:new(Counter, NewIndex, Rate), + emqx_limiter_manager:insert_bucket(Id, Type, Ref), + State#{buckets := Buckets#{Id => Bucket}}. - DelayInit = ?CURRYING(Bucket, InitFun), - - make_bucket( - T, - Type, - GlobalCfg, - CounterNum2, - [DelayInit | DelayBuckets] - ); -make_bucket([], _Type, _Global, CounterNum, DelayBuckets) -> - {CounterNum, DelayBuckets}. - --spec alloc_counter(emqx_limiter_manager:path(), rate(), capacity(), state()) -> - {counters:counters_ref(), pos_integer(), state()}. -alloc_counter( - Path, - Rate, - Initial, - #{counter := Counter, index := Index} = State -) -> - case emqx_limiter_manager:find_bucket(Path) of - {ok, #{ - counter := ECounter, - index := EIndex - }} when ECounter =/= undefined -> - init_counter(Path, ECounter, EIndex, Rate, Initial, State); +do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> + case maps:get(Id, Buckets, undefined) of + undefined -> + State; _ -> - init_counter( - Path, - Counter, - Index, - Rate, - Initial, - State#{index := Index + 1} - ) + emqx_limiter_manager:delete_bucket(Id, Type), + State#{buckets := maps:remove(Id, Buckets)} end. -init_counter(Path, Counter, Index, Rate, Initial, State) -> - _ = put_to_counter(Counter, Index, Initial), - Ref = emqx_limiter_bucket_ref:new(Counter, Index, Rate), - emqx_limiter_manager:insert_bucket(Path, Ref), - {Counter, Index, State}. - -%% @doc find first limited node -get_counter_rate(#{rate := Rate}, _GlobalCfg) when Rate =/= infinity -> - Rate; -get_counter_rate(_Cfg, #{rate := Rate}) when Rate =/= infinity -> - Rate; -get_counter_rate(_Cfg, _GlobalCfg) -> - emqx_limiter_schema:infinity_value(). - -spec get_initial_val(hocons:config()) -> decimal(). get_initial_val( #{ @@ -587,8 +571,14 @@ call(Type, Msg) -> gen_server:call(Pid, Msg) end. --spec get_bucket_cfg(limiter_type(), bucket_name()) -> - undefined | limiter_not_started | hocons:config(). -get_bucket_cfg(Type, Bucket) -> - Path = emqx_limiter_schema:get_bucket_cfg_path(Type, Bucket), - emqx:get_config(Path, undefined). +find_client_cfg(Type, Cfg) -> + NodeCfg = emqx:get_config([limiter, Type, client], undefined), + BucketCfg = maps:get(client, Cfg, undefined), + merge_client_cfg(NodeCfg, BucketCfg). + +merge_client_cfg(undefined, BucketCfg) -> + BucketCfg; +merge_client_cfg(NodeCfg, undefined) -> + NodeCfg; +merge_client_cfg(NodeCfg, BucketCfg) -> + maps:merge(NodeCfg, BucketCfg). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 3c508bacc..dcb35adbb 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -279,12 +279,19 @@ stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> end. -spec do_stop_listener(atom(), atom(), map()) -> ok | {error, term()}. -do_stop_listener(Type, ListenerName, #{bind := ListenOn}) when Type == tcp; Type == ssl -> - esockd:close(listener_id(Type, ListenerName), ListenOn); -do_stop_listener(Type, ListenerName, _Conf) when Type == ws; Type == wss -> - cowboy:stop_listener(listener_id(Type, ListenerName)); -do_stop_listener(quic, ListenerName, _Conf) -> - quicer:stop_listener(listener_id(quic, ListenerName)). + +do_stop_listener(Type, ListenerName, #{bind := ListenOn} = Conf) when Type == tcp; Type == ssl -> + Id = listener_id(Type, ListenerName), + del_limiter_bucket(Id, Conf), + esockd:close(Id, ListenOn); +do_stop_listener(Type, ListenerName, Conf) when Type == ws; Type == wss -> + Id = listener_id(Type, ListenerName), + del_limiter_bucket(Id, Conf), + cowboy:stop_listener(Id); +do_stop_listener(quic, ListenerName, Conf) -> + Id = listener_id(quic, ListenerName), + del_limiter_bucket(Id, Conf), + quicer:stop_listener(Id). -ifndef(TEST). console_print(Fmt, Args) -> ?ULOG(Fmt, Args). @@ -300,10 +307,12 @@ do_start_listener(_Type, _ListenerName, #{enabled := false}) -> do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == tcp; Type == ssl -> + Id = listener_id(Type, ListenerName), + add_limiter_bucket(Id, Opts), esockd:open( - listener_id(Type, ListenerName), + Id, ListenOn, - merge_default(esockd_opts(Type, Opts)), + merge_default(esockd_opts(Id, Type, Opts)), {emqx_connection, start_link, [ #{ listener => {Type, ListenerName}, @@ -318,6 +327,7 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == ws; Type == wss -> Id = listener_id(Type, ListenerName), + add_limiter_bucket(Id, Opts), RanchOpts = ranch_opts(Type, ListenOn, Opts), WsOpts = ws_opts(Type, ListenerName, Opts), case Type of @@ -352,8 +362,10 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> limiter => limiter(Opts) }, StreamOpts = [{stream_callback, emqx_quic_stream}], + Id = listener_id(quic, ListenerName), + add_limiter_bucket(Id, Opts), quicer:start_listener( - listener_id(quic, ListenerName), + Id, port(ListenOn), {ListenOpts, ConnectionOpts, StreamOpts} ); @@ -410,16 +422,18 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) -> ok. -esockd_opts(Type, Opts0) -> +esockd_opts(ListenerId, Type, Opts0) -> Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0), Limiter = limiter(Opts0), Opts2 = case maps:get(connection, Limiter, undefined) of undefined -> Opts1; - BucketName -> + BucketCfg -> Opts1#{ - limiter => emqx_esockd_htb_limiter:new_create_options(connection, BucketName) + limiter => emqx_esockd_htb_limiter:new_create_options( + ListenerId, connection, BucketCfg + ) } end, Opts3 = Opts2#{ @@ -524,6 +538,27 @@ zone(Opts) -> limiter(Opts) -> maps:get(limiter, Opts, #{}). +add_limiter_bucket(Id, #{limiter := Limiters}) -> + maps:fold( + fun(Type, Cfg, _) -> + emqx_limiter_server:add_bucket(Id, Type, Cfg) + end, + ok, + Limiters + ); +add_limiter_bucket(_Id, _cfg) -> + ok. + +del_limiter_bucket(Id, #{limiter := Limiters}) -> + lists:foreach( + fun(Type) -> + emqx_limiter_server:del_bucket(Id, Type) + end, + maps:keys(Limiters) + ); +del_limiter_bucket(_Id, _cfg) -> + ok. + enable_authn(Opts) -> maps:get(enable_authn, Opts, true). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5652b37f7..0c8a073a0 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1619,10 +1619,10 @@ base_listener(Bind) -> )}, {"limiter", sc( - map("ratelimit_name", emqx_limiter_schema:bucket_name()), + map("ratelimit_name", ?R_REF(emqx_limiter_schema, bucket_opts)), #{ - desc => ?DESC(base_listener_limiter), - default => #{<<"connection">> => <<"default">>} + desc => ?DESC(base_listener_limiter) + %% TODO default => #{<<"connection">> => <<"default">>} } )}, {"enable_authn", diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 0134810c1..c7c31a2d8 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -273,7 +273,7 @@ check_origin_header(Req, #{listener := {Type, Listener}} = Opts) -> end. websocket_init([Req, Opts]) -> - #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener}} = Opts, + #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener} = ListenerCfg} = Opts, case check_max_connection(Type, Listener) of allow -> {Peername, PeerCert} = get_peer_info(Type, Listener, Req, Opts), @@ -287,8 +287,10 @@ websocket_init([Req, Opts]) -> ws_cookie => WsCookie, conn_mod => ?MODULE }, - Limiter = emqx_limiter_container:get_limiter_by_names( - [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], LimiterCfg + Limiter = emqx_limiter_container:get_limiter_by_types( + ListenerCfg, + [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], + LimiterCfg ), MQTTPiggyback = get_ws_opts(Type, Listener, mqtt_piggyback), FrameOpts = #{ @@ -487,9 +489,6 @@ handle_call(From, info, State) -> handle_call(From, stats, State) -> gen_server:reply(From, stats(State)), return(State); -handle_call(_From, {ratelimit, Type, Bucket}, State = #state{limiter = Limiter}) -> - Limiter2 = emqx_limiter_container:update_by_name(Type, Bucket, Limiter), - {reply, ok, State#state{limiter = Limiter2}}; handle_call(From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of {reply, Reply, NChannel} -> diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 40bf6ff45..03be7448c 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -33,18 +33,6 @@ force_gc_conf() -> force_shutdown_conf() -> #{enable => true, max_heap_size => 4194304, max_message_queue_len => 1000}. -rate_limit_conf() -> - #{ - conn_bytes_in => ["100KB", "10s"], - conn_messages_in => ["100", "10s"], - max_conn_rate => 1000, - quota => - #{ - conn_messages_routing => infinity, - overall_messages_routing => infinity - } - }. - rpc_conf() -> #{ async_batch_size => 256, @@ -173,27 +161,9 @@ listeners_conf() -> limiter_conf() -> Make = fun() -> #{ - bucket => - #{ - default => - #{ - capacity => infinity, - initial => 0, - rate => infinity, - per_client => - #{ - capacity => infinity, - divisible => false, - failure_strategy => force, - initial => 0, - low_watermark => 0, - max_retry_time => 5000, - rate => infinity - } - } - }, burst => 0, - rate => infinity + rate => infinity, + capacity => infinity } end, @@ -202,7 +172,7 @@ limiter_conf() -> Acc#{Name => Make()} end, #{}, - [bytes_in, message_in, message_routing, connection, batch] + [bytes_in, message_in, message_routing, connection, internal] ). stats_conf() -> @@ -213,7 +183,6 @@ zone_conf() -> basic_conf() -> #{ - rate_limit => rate_limit_conf(), force_gc => force_gc_conf(), force_shutdown => force_shutdown_conf(), mqtt => mqtt_conf(), @@ -274,10 +243,9 @@ end_per_suite(_Config) -> emqx_banned ]). -init_per_testcase(TestCase, Config) -> +init_per_testcase(_TestCase, Config) -> OldConf = set_test_listener_confs(), emqx_common_test_helpers:start_apps([]), - check_modify_limiter(TestCase), [{config, OldConf} | Config]. end_per_testcase(_TestCase, Config) -> @@ -285,41 +253,6 @@ end_per_testcase(_TestCase, Config) -> emqx_common_test_helpers:stop_apps([]), Config. -check_modify_limiter(TestCase) -> - Checks = [t_quota_qos0, t_quota_qos1, t_quota_qos2], - case lists:member(TestCase, Checks) of - true -> - modify_limiter(); - _ -> - ok - end. - -%% per_client 5/1s,5 -%% aggregated 10/1s,10 -modify_limiter() -> - Limiter = emqx_config:get([limiter]), - #{message_routing := #{bucket := Bucket} = Routing} = Limiter, - #{default := #{per_client := Client} = Default} = Bucket, - Client2 = Client#{ - rate := 5, - initial := 0, - capacity := 5, - low_watermark := 1 - }, - Default2 = Default#{ - per_client := Client2, - rate => 10, - initial => 0, - capacity => 10 - }, - Bucket2 = Bucket#{default := Default2}, - Routing2 = Routing#{bucket := Bucket2}, - - emqx_config:put([limiter], Limiter#{message_routing := Routing2}), - emqx_limiter_manager:restart_server(message_routing), - timer:sleep(100), - ok. - %%-------------------------------------------------------------------- %% Test cases for channel info/stats/caps %%-------------------------------------------------------------------- @@ -729,6 +662,7 @@ t_process_unsubscribe(_) -> t_quota_qos0(_) -> esockd_limiter:start_link(), + add_bucket(), Cnter = counters:new(1, []), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), ok = meck:expect( @@ -755,10 +689,12 @@ t_quota_qos0(_) -> ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), + del_bucket(), esockd_limiter:stop(). t_quota_qos1(_) -> esockd_limiter:start_link(), + add_bucket(), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), Chann = channel(#{conn_state => connected, quota => quota()}), Pub = ?PUBLISH_PACKET(?QOS_1, <<"topic">>, 1, <<"payload">>), @@ -769,10 +705,12 @@ t_quota_qos1(_) -> {ok, ?PUBACK_PACKET(1, ?RC_SUCCESS), Chann4} = emqx_channel:handle_in(Pub, Chann3), %% Quota in overall {ok, ?PUBACK_PACKET(1, ?RC_QUOTA_EXCEEDED), _} = emqx_channel:handle_in(Pub, Chann4), + del_bucket(), esockd_limiter:stop(). t_quota_qos2(_) -> esockd_limiter:start_link(), + add_bucket(), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), Chann = channel(#{conn_state => connected, quota => quota()}), Pub1 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 1, <<"payload">>), @@ -786,6 +724,7 @@ t_quota_qos2(_) -> {ok, ?PUBREC_PACKET(3, ?RC_SUCCESS), Chann4} = emqx_channel:handle_in(Pub3, Chann3), %% Quota in overall {ok, ?PUBREC_PACKET(4, ?RC_QUOTA_EXCEEDED), _} = emqx_channel:handle_in(Pub4, Chann4), + del_bucket(), esockd_limiter:stop(). %%-------------------------------------------------------------------- @@ -952,12 +891,6 @@ t_handle_call_takeover_end(_) -> {shutdown, takenover, [], _, _Chan} = emqx_channel:handle_call({takeover, 'end'}, channel()). -t_handle_call_quota(_) -> - {reply, ok, _Chan} = emqx_channel:handle_call( - {quota, default}, - channel() - ). - t_handle_call_unexpected(_) -> {reply, ignored, _Chan} = emqx_channel:handle_call(unexpected_req, channel()). @@ -1176,7 +1109,7 @@ t_ws_cookie_init(_) -> ConnInfo, #{ zone => default, - limiter => limiter_cfg(), + limiter => undefined, listener => {tcp, default} } ), @@ -1210,7 +1143,7 @@ channel(InitFields) -> ConnInfo, #{ zone => default, - limiter => limiter_cfg(), + limiter => undefined, listener => {tcp, default} } ), @@ -1270,9 +1203,27 @@ session(InitFields) when is_map(InitFields) -> %% conn: 5/s; overall: 10/s quota() -> - emqx_limiter_container:get_limiter_by_names([message_routing], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?MODULE, [message_routing], limiter_cfg()). -limiter_cfg() -> #{message_routing => default}. +limiter_cfg() -> #{message_routing => make_limiter_cfg()}. + +make_limiter_cfg() -> + Client = #{ + rate => 5, + initial => 0, + capacity => 5, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => 10, initial => 0, capacity => 10}. + +add_bucket() -> + emqx_limiter_server:add_bucket(?MODULE, message_routing, make_limiter_cfg()). + +del_bucket() -> + emqx_limiter_server:del_bucket(?MODULE, message_routing). v4(Channel) -> ConnInfo = emqx_channel:info(conninfo, Channel), diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index b199565c2..141dbdad6 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -78,6 +78,7 @@ end_per_suite(_Config) -> init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected -> + add_bucket(), ok = meck:expect(emqx_transport, wait, fun(Sock) -> {ok, Sock} end), ok = meck:expect(emqx_transport, type, fun(_Sock) -> tcp end), ok = meck:expect( @@ -104,9 +105,11 @@ init_per_testcase(TestCase, Config) when _ -> Config end; init_per_testcase(_, Config) -> + add_bucket(), Config. end_per_testcase(TestCase, Config) -> + del_bucket(), case erlang:function_exported(?MODULE, TestCase, 2) of true -> ?MODULE:TestCase('end', Config); false -> ok @@ -291,11 +294,6 @@ t_handle_call(_) -> ?assertMatch({ok, _St}, handle_msg({event, undefined}, St)), ?assertMatch({reply, _Info, _NSt}, handle_call(self(), info, St)), ?assertMatch({reply, _Stats, _NSt}, handle_call(self(), stats, St)), - ?assertMatch({reply, ok, _NSt}, handle_call(self(), {ratelimit, []}, St)), - ?assertMatch( - {reply, ok, _NSt}, - handle_call(self(), {ratelimit, [{bytes_in, default}]}, St) - ), ?assertEqual({reply, ignored, St}, handle_call(self(), for_testing, St)), ?assertMatch( {stop, {shutdown, kicked}, ok, _NSt}, @@ -704,7 +702,33 @@ handle_msg(Msg, St) -> emqx_connection:handle_msg(Msg, St). handle_call(Pid, Call, St) -> emqx_connection:handle_call(Pid, Call, St). -limiter_cfg() -> #{}. +-define(LIMITER_ID, 'tcp:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). + +limiter_cfg() -> + Cfg = make_limiter_cfg(), + #{bytes_in => Cfg, message_in => Cfg}. + +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Infinity, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket() -> + Cfg = make_limiter_cfg(), + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + +del_bucket() -> + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), + emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 1251278f2..e31a220e9 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -24,48 +24,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(BASE_CONF, << - "" - "\n" - "limiter {\n" - " bytes_in {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " message_in {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " connection {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " message_routing {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " batch {\n" - " bucket.retainer {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "}\n" - "\n" - "" ->>). +-define(BASE_CONF, <<"">>). -record(client, { counter :: counters:counter_ref(), @@ -97,6 +56,9 @@ end_per_suite(_Config) -> init_per_testcase(_TestCase, Config) -> Config. +end_per_testcase(_TestCase, Config) -> + Config. + load_conf() -> emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). @@ -116,12 +78,12 @@ t_consume(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), {ok, L2} = emqx_htb_limiter:consume(50, Client), {ok, _L3} = emqx_htb_limiter:consume(150, L2) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_retry(_) -> Cfg = fun(Cfg) -> @@ -133,15 +95,15 @@ t_retry(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), - {ok, Client} = emqx_htb_limiter:retry(Client), - {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), + {ok, Client2} = emqx_htb_limiter:retry(Client), + {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client2), L3 = emqx_htb_limiter:set_retry(Retry, L2), timer:sleep(500), {ok, _L4} = emqx_htb_limiter:retry(L3) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_restore(_) -> Cfg = fun(Cfg) -> @@ -153,15 +115,15 @@ t_restore(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), timer:sleep(200), {ok, L3} = emqx_htb_limiter:check(Retry, L2), Avaiable = emqx_htb_limiter:available(L3), ?assert(Avaiable >= 50) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_max_retry_time(_) -> Cfg = fun(Cfg) -> @@ -172,15 +134,15 @@ t_max_retry_time(_) -> failure_strategy := drop } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Begin = ?NOW, Result = emqx_htb_limiter:consume(101, Client), ?assertMatch({drop, _}, Result), Time = ?NOW - Begin, ?assert(Time >= 500 andalso Time < 550) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_divisible(_) -> Cfg = fun(Cfg) -> @@ -191,8 +153,8 @@ t_divisible(_) -> capacity := 600 } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Result = emqx_htb_limiter:check(1000, Client), ?assertMatch( {partial, 400, @@ -206,7 +168,7 @@ t_divisible(_) -> Result ) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_low_watermark(_) -> Cfg = fun(Cfg) -> @@ -217,8 +179,8 @@ t_low_watermark(_) -> capacity := 1000 } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Result = emqx_htb_limiter:check(500, Client), ?assertMatch({ok, _}, Result), {_, Client2} = Result, @@ -233,28 +195,21 @@ t_low_watermark(_) -> Result2 ) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_infinity_client(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> - Bucket2 = Bucket#{ - rate := infinity, - capacity := infinity - }, - Cli2 = Cli#{rate := infinity, capacity := infinity}, - Bucket2#{per_client := Cli2} - end, - Case = fun() -> - Client = connect(default), + Fun = fun(Cfg) -> Cfg end, + Case = fun(Cfg) -> + Client = connect(Cfg), InfVal = emqx_limiter_schema:infinity_value(), ?assertMatch(#{bucket := #{rate := InfVal}}, Client), Result = emqx_htb_limiter:check(100000, Client), ?assertEqual({ok, Client}, Result) end, - with_bucket(default, Fun, Case). + with_per_client(Fun, Case). t_try_restore_agg(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := 1, capacity := 200, @@ -267,20 +222,20 @@ t_try_restore_agg(_) -> max_retry_time := 100, failure_strategy := force }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), timer:sleep(200), {ok, L3} = emqx_htb_limiter:check(Retry, L2), Avaiable = emqx_htb_limiter:available(L3), ?assert(Avaiable >= 50) end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_short_board(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/1s"), initial := 0, @@ -291,18 +246,18 @@ t_short_board(_) -> capacity := 600, initial := 600 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> + Case = fun(Cfg) -> Counter = counters:new(1, []), - start_client(default, ?NOW + 2000, Counter, 20), + start_client(Cfg, ?NOW + 2000, Counter, 20), timer:sleep(2100), check_average_rate(Counter, 2, 100) end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_rate(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, @@ -313,10 +268,10 @@ t_rate(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), Ts1 = erlang:system_time(millisecond), C1 = emqx_htb_limiter:available(Client), timer:sleep(1000), @@ -326,11 +281,11 @@ t_rate(_) -> Inc = C2 - C1, ?assert(in_range(Inc, ShouldInc - 100, ShouldInc + 100), "test bucket rate") end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_capacity(_) -> Capacity = 600, - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, @@ -341,15 +296,15 @@ t_capacity(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), timer:sleep(1000), C1 = emqx_htb_limiter:available(Client), ?assertEqual(Capacity, C1, "test bucket capacity") end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). %%-------------------------------------------------------------------- %% Test Cases Global Level @@ -359,7 +314,7 @@ t_collaborative_alloc(_) -> Cfg#{rate := ?RATE("600/1s")} end, - Bucket1 = fun(#{per_client := Cli} = Bucket) -> + Bucket1 = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("400/1s"), initial := 0, @@ -370,7 +325,7 @@ t_collaborative_alloc(_) -> capacity := 100, initial := 100 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Bucket2 = fun(Bucket) -> @@ -381,8 +336,8 @@ t_collaborative_alloc(_) -> Case = fun() -> C1 = counters:new(1, []), C2 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), - start_client(b2, ?NOW + 2000, C2, 30), + start_client({b1, Bucket1}, ?NOW + 2000, C1, 20), + start_client({b2, Bucket2}, ?NOW + 2000, C2, 30), timer:sleep(2100), check_average_rate(C1, 2, 300), check_average_rate(C2, 2, 300) @@ -402,7 +357,7 @@ t_burst(_) -> } end, - Bucket = fun(#{per_client := Cli} = Bucket) -> + Bucket = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("200/1s"), initial := 0, @@ -413,16 +368,16 @@ t_burst(_) -> capacity := 200, divisible := true }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Case = fun() -> C1 = counters:new(1, []), C2 = counters:new(1, []), C3 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), - start_client(b2, ?NOW + 2000, C2, 30), - start_client(b3, ?NOW + 2000, C3, 30), + start_client({b1, Bucket}, ?NOW + 2000, C1, 20), + start_client({b2, Bucket}, ?NOW + 2000, C2, 30), + start_client({b3, Bucket}, ?NOW + 2000, C3, 30), timer:sleep(2100), Total = lists:sum([counters:get(X, 1) || X <- [C1, C2, C3]]), @@ -440,7 +395,7 @@ t_limit_global_with_unlimit_other(_) -> Cfg#{rate := ?RATE("600/1s")} end, - Bucket = fun(#{per_client := Cli} = Bucket) -> + Bucket = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := infinity, initial := 0, @@ -451,12 +406,12 @@ t_limit_global_with_unlimit_other(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Case = fun() -> C1 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), + start_client({b1, Bucket}, ?NOW + 2000, C1, 20), timer:sleep(2100), check_average_rate(C1, 2, 600) end, @@ -470,28 +425,6 @@ t_limit_global_with_unlimit_other(_) -> %%-------------------------------------------------------------------- %% Test Cases container %%-------------------------------------------------------------------- -t_new_container(_) -> - C1 = emqx_limiter_container:new(), - C2 = emqx_limiter_container:new([message_routing]), - C3 = emqx_limiter_container:update_by_name(message_routing, default, C1), - ?assertMatch( - #{ - message_routing := _, - retry_ctx := undefined, - {retry, message_routing} := _ - }, - C2 - ), - ?assertMatch( - #{ - message_routing := _, - retry_ctx := undefined, - {retry, message_routing} := _ - }, - C3 - ), - ok. - t_check_container(_) -> Cfg = fun(Cfg) -> Cfg#{ @@ -500,10 +433,11 @@ t_check_container(_) -> capacity := 1000 } end, - Case = fun() -> - C1 = emqx_limiter_container:new( + Case = fun(BucketCfg) -> + C1 = emqx_limiter_container:get_limiter_by_types( + ?MODULE, [message_routing], - #{message_routing => default} + #{message_routing => BucketCfg} ), {ok, C2} = emqx_limiter_container:check(1000, message_routing, C1), {pause, Pause, C3} = emqx_limiter_container:check(1000, message_routing, C2), @@ -514,7 +448,39 @@ t_check_container(_) -> RetryData = emqx_limiter_container:get_retry_context(C5), ?assertEqual(Context, RetryData) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). + +%%-------------------------------------------------------------------- +%% Test Override +%%-------------------------------------------------------------------- +t_bucket_no_client(_) -> + Rate = ?RATE("1/s"), + GlobalMod = fun(#{client := Client} = Cfg) -> + Cfg#{client := Client#{rate := Rate}} + end, + BucketMod = fun(Bucket) -> + maps:remove(client, Bucket) + end, + Case = fun() -> + Limiter = connect(BucketMod(make_limiter_cfg())), + ?assertMatch(#{rate := Rate}, Limiter) + end, + with_global(GlobalMod, [BucketMod], Case). + +t_bucket_client(_) -> + GlobalRate = ?RATE("1/s"), + BucketRate = ?RATE("10/s"), + GlobalMod = fun(#{client := Client} = Cfg) -> + Cfg#{client := Client#{rate := GlobalRate}} + end, + BucketMod = fun(#{client := Client} = Bucket) -> + Bucket#{client := Client#{rate := BucketRate}} + end, + Case = fun() -> + Limiter = connect(BucketMod(make_limiter_cfg())), + ?assertMatch(#{rate := BucketRate}, Limiter) + end, + with_global(GlobalMod, [BucketMod], Case). %%-------------------------------------------------------------------- %% Test Cases misc @@ -607,19 +573,23 @@ t_schema_unit(_) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -start_client(Name, EndTime, Counter, Number) -> +start_client(Cfg, EndTime, Counter, Number) -> lists:foreach( fun(_) -> spawn(fun() -> - start_client(Name, EndTime, Counter) + do_start_client(Cfg, EndTime, Counter) end) end, lists:seq(1, Number) ). -start_client(Name, EndTime, Counter) -> - #{per_client := PerClient} = - emqx_config:get([limiter, message_routing, bucket, Name]), +do_start_client({Name, CfgFun}, EndTime, Counter) -> + do_start_client(Name, CfgFun(make_limiter_cfg()), EndTime, Counter); +do_start_client(Cfg, EndTime, Counter) -> + do_start_client(?MODULE, Cfg, EndTime, Counter). + +do_start_client(Name, Cfg, EndTime, Counter) -> + #{client := PerClient} = Cfg, #{rate := Rate} = PerClient, Client = #client{ start = ?NOW, @@ -627,7 +597,7 @@ start_client(Name, EndTime, Counter) -> counter = Counter, obtained = 0, rate = Rate, - client = connect(Name) + client = connect(Name, Cfg) }, client_loop(Client). @@ -711,35 +681,50 @@ to_rate(Str) -> {ok, Rate} = emqx_limiter_schema:to_rate(Str), Rate. -with_global(Modifier, BuckeTemps, Case) -> - Fun = fun(Cfg) -> - #{bucket := #{default := BucketCfg}} = Cfg2 = Modifier(Cfg), - Fun = fun({Name, BMod}, Acc) -> - Acc#{Name => BMod(BucketCfg)} - end, - Buckets = lists:foldl(Fun, #{}, BuckeTemps), - Cfg2#{bucket := Buckets} - end, +with_global(Modifier, Buckets, Case) -> + with_config([limiter, message_routing], Modifier, Buckets, Case). - with_config([limiter, message_routing], Fun, Case). +with_bucket(Modifier, Case) -> + Cfg = Modifier(make_limiter_cfg()), + add_bucket(Cfg), + Case(Cfg), + del_bucket(). -with_bucket(Bucket, Modifier, Case) -> - Path = [limiter, message_routing, bucket, Bucket], - with_config(Path, Modifier, Case). +with_per_client(Modifier, Case) -> + #{client := Client} = Cfg = make_limiter_cfg(), + Cfg2 = Cfg#{client := Modifier(Client)}, + add_bucket(Cfg2), + Case(Cfg2), + del_bucket(). -with_per_client(Bucket, Modifier, Case) -> - Path = [limiter, message_routing, bucket, Bucket, per_client], - with_config(Path, Modifier, Case). - -with_config(Path, Modifier, Case) -> +with_config(Path, Modifier, Buckets, Case) -> Cfg = emqx_config:get(Path), NewCfg = Modifier(Cfg), - ct:pal("test with config:~p~n", [NewCfg]), emqx_config:put(Path, NewCfg), emqx_limiter_server:restart(message_routing), timer:sleep(500), + BucketCfg = make_limiter_cfg(), + lists:foreach( + fun + ({Name, BucketFun}) -> + add_bucket(Name, BucketFun(BucketCfg)); + (BucketFun) -> + add_bucket(BucketFun(BucketCfg)) + end, + Buckets + ), DelayReturn = delay_return(Case), + lists:foreach( + fun + ({Name, _Cfg}) -> + del_bucket(Name); + (_Cfg) -> + del_bucket() + end, + Buckets + ), emqx_config:put(Path, Cfg), + emqx_limiter_server:restart(message_routing), DelayReturn(). delay_return(Case) -> @@ -751,10 +736,40 @@ delay_return(Case) -> fun() -> erlang:raise(Type, Reason, Trace) end end. -connect(Name) -> - {ok, Limiter} = emqx_limiter_server:connect(message_routing, Name), +connect({Name, CfgFun}) -> + connect(Name, CfgFun(make_limiter_cfg())); +connect(Cfg) -> + connect(?MODULE, Cfg). + +connect(Name, Cfg) -> + {ok, Limiter} = emqx_limiter_server:connect(Name, message_routing, Cfg), Limiter. +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Infinity, + initial => 0, + capacity => Infinity, + low_watermark => 0, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket(Cfg) -> + add_bucket(?MODULE, Cfg). + +add_bucket(Name, Cfg) -> + emqx_limiter_server:add_bucket(Name, message_routing, Cfg). + +del_bucket() -> + del_bucket(?MODULE). + +del_bucket(Name) -> + emqx_limiter_server:del_bucket(Name, message_routing). + check_average_rate(Counter, Second, Rate) -> Cost = counters:get(Counter, 1), PerSec = Cost / Second, diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 89d892c67..47591bf64 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -59,6 +59,7 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + add_bucket(), %% Meck Cm ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), @@ -96,6 +97,7 @@ init_per_testcase(TestCase, Config) when | Config ]; init_per_testcase(t_ws_non_check_origin, Config) -> + add_bucket(), ok = emqx_common_test_helpers:start_apps([]), PrevConfig = emqx_config:get_listener_conf(ws, default, [websocket]), emqx_config:put_listener_conf(ws, default, [websocket, check_origin_enable], false), @@ -105,6 +107,7 @@ init_per_testcase(t_ws_non_check_origin, Config) -> | Config ]; init_per_testcase(_, Config) -> + add_bucket(), PrevConfig = emqx_config:get_listener_conf(ws, default, [websocket]), ok = emqx_common_test_helpers:start_apps([]), [ @@ -119,6 +122,7 @@ end_per_testcase(TestCase, _Config) when TestCase =/= t_ws_non_check_origin, TestCase =/= t_ws_pingreq_before_connected -> + del_bucket(), lists:foreach( fun meck:unload/1, [ @@ -131,11 +135,13 @@ end_per_testcase(TestCase, _Config) when ] ); end_per_testcase(t_ws_non_check_origin, Config) -> + del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), emqx_common_test_helpers:stop_apps([]), ok; end_per_testcase(_, Config) -> + del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), emqx_common_test_helpers:stop_apps([]), @@ -501,15 +507,11 @@ t_handle_timeout_emit_stats(_) -> ?assertEqual(undefined, ?ws_conn:info(stats_timer, St)). t_ensure_rate_limit(_) -> - %% XXX In the future, limiter should provide API for config update - Path = [limiter, bytes_in, bucket, default, per_client], - PerClient = emqx_config:get(Path), {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), - emqx_config:put(Path, PerClient#{rate := Rate}), - emqx_limiter_server:restart(bytes_in), - timer:sleep(100), - - Limiter = init_limiter(), + Limiter = init_limiter(#{ + bytes_in => make_limiter_cfg(Rate), + message_in => make_limiter_cfg() + }), St = st(#{limiter => Limiter}), %% must bigger than value in emqx_ratelimit_SUITE @@ -522,11 +524,7 @@ t_ensure_rate_limit(_) -> St ), ?assertEqual(blocked, ?ws_conn:info(sockstate, St1)), - ?assertEqual([{active, false}], ?ws_conn:info(postponed, St1)), - - emqx_config:put(Path, PerClient), - emqx_limiter_server:restart(bytes_in), - timer:sleep(100). + ?assertEqual([{active, false}], ?ws_conn:info(postponed, St1)). t_parse_incoming(_) -> {Packets, St} = ?ws_conn:parse_incoming(<<48, 3>>, [], st()), @@ -691,7 +689,40 @@ ws_client(State) -> ct:fail(ws_timeout) end. -limiter_cfg() -> #{bytes_in => default, message_in => default}. +-define(LIMITER_ID, 'ws:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). + init_limiter(limiter_cfg()). + +init_limiter(LimiterCfg) -> + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). + +limiter_cfg() -> + Cfg = make_limiter_cfg(), + #{bytes_in => Cfg, message_in => Cfg}. + +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + make_limiter_cfg(Infinity). + +make_limiter_cfg(ClientRate) -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => ClientRate, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket() -> + Cfg = make_limiter_cfg(), + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + +del_bucket() -> + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), + emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d911b5f4..f5a3ad403 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -348,12 +348,16 @@ enable_retainer( #{context_id := ContextId} = State, #{ msg_clear_interval := ClearInterval, - backend := BackendCfg + backend := BackendCfg, + flow_control := FlowControl } ) -> NewContextId = ContextId + 1, Context = create_resource(new_context(NewContextId), BackendCfg), load(Context), + emqx_limiter_server:add_bucket( + ?APP, internal, maps:get(batch_deliver_limiter, FlowControl, undefined) + ), State#{ enable := true, context_id := NewContextId, @@ -369,6 +373,7 @@ disable_retainer( } = State ) -> unload(), + emqx_limiter_server:del_bucket(?APP, internal), ok = close_resource(Context), State#{ enable := false, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 29818481d..f52fd982c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -115,8 +115,8 @@ start_link(Pool, Id) -> init([Pool, Id]) -> erlang:process_flag(trap_exit, true), true = gproc_pool:connect_worker(Pool, {Pool, Id}), - BucketName = emqx:get_config([retainer, flow_control, batch_deliver_limiter], undefined), - {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), + BucketCfg = emqx:get_config([retainer, flow_control, batch_deliver_limiter], undefined), + {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {ok, #{pool => Pool, id => Id, limiter => Limiter}}. %%-------------------------------------------------------------------- @@ -155,8 +155,8 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketName = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), - {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), + BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 526059c9e..22083ba2c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - emqx_limiter_schema:bucket_name(), + ?R_REF(emqx_limiter_schema, bucket_opts), batch_deliver_limiter, undefined )} diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index ed49f6f5c..d7ddc2424 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -368,27 +368,16 @@ t_stop_publish_clear_msg(_) -> ok = emqtt:disconnect(C1). t_flow_control(_) -> - #{per_client := PerClient} = RetainerCfg = emqx_config:get([limiter, batch, bucket, retainer]), - RetainerCfg2 = RetainerCfg#{ - per_client := - PerClient#{ - rate := emqx_ratelimiter_SUITE:to_rate("1/1s"), - capacity := 1 - } - }, - emqx_config:put([limiter, batch, bucket, retainer], RetainerCfg2), - emqx_limiter_manager:restart_server(batch), - timer:sleep(500), - - emqx_retainer_dispatcher:refresh_limiter(), - timer:sleep(500), - + Rate = emqx_ratelimiter_SUITE:to_rate("1/1s"), + LimiterCfg = make_limiter_cfg(Rate), + JsonCfg = make_limiter_json(<<"1/1s">>), + emqx_limiter_server:add_bucket(emqx_retainer, internal, LimiterCfg), emqx_retainer:update_config(#{ <<"flow_control">> => #{ <<"batch_read_number">> => 1, <<"batch_deliver_number">> => 1, - <<"batch_deliver_limiter">> => retainer + <<"batch_deliver_limiter">> => JsonCfg } }), {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), @@ -424,13 +413,14 @@ t_flow_control(_) -> ok = emqtt:disconnect(C1), - %% recover the limiter - emqx_config:put([limiter, batch, bucket, retainer], RetainerCfg), - emqx_limiter_manager:restart_server(batch), - timer:sleep(500), - - emqx_retainer_dispatcher:refresh_limiter(), - timer:sleep(500), + emqx_limiter_server:del_bucket(emqx_retainer, internal), + emqx_retainer:update_config(#{ + <<"flow_control">> => + #{ + <<"batch_read_number">> => 1, + <<"batch_deliver_number">> => 1 + } + }), ok. t_clear_expired(_) -> @@ -684,3 +674,33 @@ with_conf(ConfMod, Case) -> emqx_retainer:update_config(Conf), erlang:raise(Type, Error, Strace) end. + +make_limiter_cfg(Rate) -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Rate, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +make_limiter_json(Rate) -> + Client = #{ + <<"rate">> => Rate, + <<"initial">> => 0, + <<"capacity">> => <<"infinity">>, + <<"low_watermark">> => 0, + <<"divisible">> => <<"false">>, + <<"max_retry_time">> => <<"5s">>, + <<"failure_strategy">> => <<"force">> + }, + #{ + <<"client">> => Client, + <<"rate">> => <<"infinity">>, + <<"initial">> => 0, + <<"capacity">> => <<"infinity">> + }. From dbab1bc96aeeb8dab40ab31e86bc7ad1c97fd92e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 20 Jul 2022 15:00:12 +0800 Subject: [PATCH 035/105] fix(limiter): fix elvis && dialyzer error --- .../emqx_limiter/src/emqx_limiter_server.erl | 8 ++--- apps/emqx/src/emqx_listeners.erl | 4 +-- apps/emqx_retainer/src/emqx_retainer_api.erl | 36 ++----------------- 3 files changed, 8 insertions(+), 40 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 939824e02..997f6b788 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -75,11 +75,11 @@ -type state() :: #{ type := limiter_type(), - root := undefined | root(), + root := root(), buckets := buckets(), %% current counter to alloc - counter := undefined | counters:counters_ref(), - index := index() + counter := counters:counters_ref(), + index := 0 | index() }. -type buckets() :: #{bucket_name() => bucket()}. @@ -507,7 +507,7 @@ do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buck end. make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> - add_bucket(Id, Cfg, State#{ + make_bucket(Id, Cfg, State#{ counter => counters:new(?COUNTER_SIZE, [write_concurrency]), index => 0 }); diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index dcb35adbb..fdbd5350e 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -546,7 +546,7 @@ add_limiter_bucket(Id, #{limiter := Limiters}) -> ok, Limiters ); -add_limiter_bucket(_Id, _cfg) -> +add_limiter_bucket(_Id, _Cfg) -> ok. del_limiter_bucket(Id, #{limiter := Limiters}) -> @@ -556,7 +556,7 @@ del_limiter_bucket(Id, #{limiter := Limiters}) -> end, maps:keys(Limiters) ); -del_limiter_bucket(_Id, _cfg) -> +del_limiter_bucket(_Id, _Cfg) -> ok. enable_authn(Opts) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 7d085b422..2c0bd725c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -151,13 +151,8 @@ config(get, _) -> {200, emqx:get_raw_config([retainer])}; config(put, #{body := Body}) -> try - check_bucket_exists( - Body, - fun(Conf) -> - {ok, _} = emqx_retainer:update_config(Conf), - {200, emqx:get_raw_config([retainer])} - end - ) + {ok, _} = emqx_retainer:update_config(Body), + {200, emqx:get_raw_config([retainer])} catch _:Reason:_ -> {400, #{ @@ -237,30 +232,3 @@ check_backend(Type, Params, Cont) -> _ -> {400, 'BAD_REQUEST', <<"This API only support built in database">>} end. - -check_bucket_exists( - #{ - <<"flow_control">> := - #{<<"batch_deliver_limiter">> := Name} = Flow - } = Conf, - Cont -) -> - case erlang:binary_to_atom(Name) of - '' -> - %% workaround, empty string means set the value to undefined, - %% but now, we can't store `undefined` in the config file correct, - %% but, we can delete this field - Cont(Conf#{ - <<"flow_control">> := maps:remove(<<"batch_deliver_limiter">>, Flow) - }); - Bucket -> - Path = emqx_limiter_schema:get_bucket_cfg_path(batch, Bucket), - case emqx:get_config(Path, undefined) of - undefined -> - {400, 'BAD_REQUEST', <<"The limiter bucket not exists">>}; - _ -> - Cont(Conf) - end - end; -check_bucket_exists(Conf, Cont) -> - Cont(Conf). From a12478225afd6c92a8d48127b3681711eae6d184 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 20 Jul 2022 16:56:07 +0800 Subject: [PATCH 036/105] fix: fix bad swagger format --- .../src/emqx_authn_user_import_api.erl | 20 +--- .../emqx_authz/src/emqx_authz_api_sources.erl | 108 +++++++++--------- .../emqx_dashboard/src/emqx_dashboard_api.erl | 1 - .../src/emqx_dashboard_swagger.erl | 15 +++ .../emqx_gateway_api_authn_user_import.erl | 20 +--- 5 files changed, 76 insertions(+), 88 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 30417acf7..48e623438 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -66,15 +66,7 @@ schema("/authentication/:id/import_users") -> tags => ?API_TAGS_GLOBAL, description => ?DESC(authentication_id_import_users_post), parameters => [emqx_authn_api:param_auth_id()], - 'requestBody' => #{ - content => #{ - 'multipart/form-data' => #{ - schema => #{ - filename => file - } - } - } - }, + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => #{ 204 => <<"Users imported">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -89,15 +81,7 @@ schema("/listeners/:listener_id/authentication/:id/import_users") -> tags => ?API_TAGS_SINGLE, description => ?DESC(listeners_listener_id_authentication_id_import_users_post), parameters => [emqx_authn_api:param_listener_id(), emqx_authn_api:param_auth_id()], - 'requestBody' => #{ - content => #{ - 'multipart/form-data' => #{ - schema => #{ - filename => file - } - } - } - }, + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => #{ 204 => <<"Users imported">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 2dfc6da33..5f3873998 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -565,58 +565,64 @@ bin(Term) -> erlang:iolist_to_binary(io_lib:format("~p", [Term])). status_metrics_example() -> #{ - resource_metrics => #{ - matched => 0, - success => 0, - failed => 0, - rate => 0.0, - rate_last5m => 0.0, - rate_max => 0.0 - }, - node_resource_metrics => [ - #{ - node => node(), - metrics => #{ - matched => 0, - success => 0, - failed => 0, - rate => 0.0, - rate_last5m => 0.0, - rate_max => 0.0 - } - } - ], - metrics => #{ - total => 0, - allow => 0, - deny => 0, - nomatch => 0, - rate => 0.0, - rate_last5m => 0.0, - rate_max => 0.0 - }, - node_metrics => [ - #{ - node => node(), - metrics => #{ - total => 0, - allow => 0, - deny => 0, - nomatch => 0, - rate => 0.0, - rate_last5m => 0.0, - rate_max => 0.0 - } - } - ], + 'metrics_example' => #{ + summary => <<"Showing a typical metrics example">>, + value => + #{ + resource_metrics => #{ + matched => 0, + success => 0, + failed => 0, + rate => 0.0, + rate_last5m => 0.0, + rate_max => 0.0 + }, + node_resource_metrics => [ + #{ + node => node(), + metrics => #{ + matched => 0, + success => 0, + failed => 0, + rate => 0.0, + rate_last5m => 0.0, + rate_max => 0.0 + } + } + ], + metrics => #{ + total => 0, + allow => 0, + deny => 0, + nomatch => 0, + rate => 0.0, + rate_last5m => 0.0, + rate_max => 0.0 + }, + node_metrics => [ + #{ + node => node(), + metrics => #{ + total => 0, + allow => 0, + deny => 0, + nomatch => 0, + rate => 0.0, + rate_last5m => 0.0, + rate_max => 0.0 + } + } + ], - status => connected, - node_status => [ - #{ - node => node(), - status => connected - } - ] + status => connected, + node_status => [ + #{ + node => node(), + status => connected + } + ] + } + } }. create_authz_file(Body) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index b6231e7c6..a8fa5d187 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -180,7 +180,6 @@ field(username_in_path) -> {username, mk(binary(), #{ desc => ?DESC(username), - 'maxLength' => 100, example => <<"admin">>, in => path, required => true diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 80b4e9624..3d65e569c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -24,6 +24,7 @@ -export([namespace/0, namespace/1, fields/1]). -export([schema_with_example/2, schema_with_examples/2]). -export([error_codes/1, error_codes/2]). +-export([file_schema/1]). -export([filter_check_request/2, filter_check_request_and_translate_body/2]). @@ -165,6 +166,20 @@ error_codes(Codes = [_ | _], MsgDesc) -> })} ]. +file_schema(FileName) -> + #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + type => object, + properties => #{ + FileName => #{type => string, format => binary} + } + } + } + } + }. + %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index b86dc4510..c324262ee 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -124,15 +124,7 @@ schema("/gateway/:name/authentication/import_users") -> #{ desc => ?DESC(emqx_gateway_api_authn, import_users), parameters => params_gateway_name_in_path(), - 'requestBody' => #{ - content => #{ - 'multipart/form-data' => #{ - schema => #{ - filename => file - } - } - } - }, + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => ?STANDARD_RESP(#{204 => <<"Imported">>}) } @@ -145,15 +137,7 @@ schema("/gateway/:name/listeners/:id/authentication/import_users") -> desc => ?DESC(emqx_gateway_api_listeners, import_users), parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), - 'requestBody' => #{ - content => #{ - 'multipart/form-data' => #{ - schema => #{ - filename => file - } - } - } - }, + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => ?STANDARD_RESP(#{204 => <<"Imported">>}) } From a468f4ee9d66c66fc845b82e871197d79b8b9ca4 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 20 Jul 2022 17:03:04 +0800 Subject: [PATCH 037/105] chore: bump emqx_gateway to 0.1.2 --- CHANGES-5.0.md | 1 + apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index fcf2154d5..8f390df9d 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -13,6 +13,7 @@ * Fix statistics related system topic name error * Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) * `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) +* Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) ## Enhancements diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index dbe966751..963cacc2f 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, From c638e607cc9b317f7ecc098cccfc39ef0f89abc8 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 20 Jul 2022 18:16:28 +0800 Subject: [PATCH 038/105] chore(relup): download relup base version packages from s3 --- scripts/relup-build/download-base-packages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/relup-build/download-base-packages.sh b/scripts/relup-build/download-base-packages.sh index 6d4095edb..1a03f7ef8 100755 --- a/scripts/relup-build/download-base-packages.sh +++ b/scripts/relup-build/download-base-packages.sh @@ -14,7 +14,7 @@ export PROFILE case $PROFILE in "emqx-enterprise") - DIR='enterprise' + DIR='emqx-ee' EDITION='enterprise' ;; "emqx") @@ -51,7 +51,7 @@ mkdir -p _upgrade_base pushd _upgrade_base >/dev/null for tag in ${BASE_VERSIONS}; do filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz" - url="https://www.emqx.com/downloads/$DIR/$tag/$filename" + url="https://packages.emqx.io/$DIR/$tag/$filename" echo "downloading ${filename} ..." ## if the file does not exist (not downloaded yet) ## and there is such a package to downlaod From 2566f50c4eef995e093848c2194d917798792e5d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 21 Jul 2022 11:01:18 +0800 Subject: [PATCH 039/105] fix(exhook): ensuring that exhook dispatches the client events are sequential porting by: https://github.com/emqx/emqx/pull/8311 --- CHANGES-5.0.md | 1 + apps/emqx_exhook/rebar.config | 3 +-- apps/emqx_exhook/src/emqx_exhook_server.erl | 15 ++++++++++++++- apps/emqx_gateway/rebar.config | 3 +-- rebar.config | 1 + 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 8f390df9d..3a6a2f953 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -14,6 +14,7 @@ * Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) * `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) * Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) +* Ensuring that exhook dispatches the client events are sequential. ## Enhancements diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 235d4be1b..fad539ed1 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,8 +5,7 @@ ]}. {deps, [ - {emqx, {path, "../emqx"}}, - {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} + {emqx, {path, "../emqx"}} ]}. {grpc, [ diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 286d062dd..832915e3f 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -369,8 +369,11 @@ match_topic_filter(TopicName, TopicFilter) -> -spec do_call(binary(), atom(), atom(), map(), map()) -> {ok, map()} | {error, term()}. do_call(ChannName, Hookpoint, Fun, Req, ReqOpts) -> - Options = ReqOpts#{channel => ChannName}, NReq = Req#{meta => emqx_exhook_handler:request_meta()}, + Options = ReqOpts#{ + channel => ChannName, + key_dispatch => key_dispatch(NReq) + }, ?SLOG(debug, #{ msg => "do_call", module => ?PB_CLIENT_MOD, @@ -481,3 +484,13 @@ available_hooks() -> 'session.terminated' | message_hooks() ]. + +%% @doc Get dispatch_key for each request +key_dispatch(_Req = #{clientinfo := #{clientid := ClientId}}) -> + ClientId; +key_dispatch(_Req = #{conninfo := #{clientid := ClientId}}) -> + ClientId; +key_dispatch(_Req = #{message := #{from := From}}) -> + From; +key_dispatch(_Req) -> + self(). diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 49279e35e..272783758 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -2,8 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {emqx, {path, "../emqx"}}, - {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} + {emqx, {path, "../emqx"}} ]}. {plugins, [ diff --git a/rebar.config b/rebar.config index 7d2570636..37dce676d 100644 --- a/rebar.config +++ b/rebar.config @@ -56,6 +56,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.4"} From 116c664df14cbc2811843b6cbe2fa7f3c8e53d69 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 21 Jul 2022 12:08:32 +0800 Subject: [PATCH 040/105] feat: add sysmon config handler to reload conf --- apps/emqx/src/emqx_config.erl | 2 + apps/emqx/src/emqx_os_mon.erl | 24 ++++++++--- apps/emqx/src/emqx_sys_mon.erl | 42 ++++++++++++++++--- apps/emqx/src/emqx_vm_mon.erl | 2 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 7 ++++ 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 3f89e2082..3d602349d 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -556,10 +556,12 @@ save_to_override_conf(RawConf, Opts) -> add_handlers() -> ok = emqx_config_logger:add_handler(), + emqx_sys_mon:add_handler(), ok. remove_handlers() -> ok = emqx_config_logger:remove_handler(), + emqx_sys_mon:remove_handler(), ok. load_hocon_file(FileName, LoadType) -> diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 74acfff93..462b80589 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -35,6 +35,8 @@ current_sysmem_percent/0 ]). +-export([update/1]). + %% gen_server callbacks -export([ init/1, @@ -52,6 +54,9 @@ start_link() -> gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []). +update(OS) -> + erlang:send(?MODULE, {monitor_conf_update, OS}). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -87,18 +92,24 @@ current_sysmem_percent() -> init([]) -> %% memsup is not reliable, ignore memsup:set_sysmem_high_watermark(1.0), + SysHW = init_os_monitor(), + _ = start_mem_check_timer(), + _ = start_cpu_check_timer(), + {ok, #{sysmem_high_watermark => SysHW}}. + +init_os_monitor() -> + init_os_monitor(emqx:get_config([sysmon, os])). + +init_os_monitor(OS) -> #{ sysmem_high_watermark := SysHW, procmem_high_watermark := PHW, mem_check_interval := MCI - } = emqx:get_config([sysmon, os]), - + } = OS, set_procmem_high_watermark(PHW), set_mem_check_interval(MCI), ok = update_mem_alarm_status(SysHW), - _ = start_mem_check_timer(), - _ = start_cpu_check_timer(), - {ok, #{sysmem_high_watermark => SysHW}}. + SysHW. handle_call(get_sysmem_high_watermark, _From, #{sysmem_high_watermark := HWM} = State) -> {reply, HWM, State}; @@ -147,6 +158,9 @@ handle_info({timeout, _Timer, cpu_check}, State) -> end, ok = start_cpu_check_timer(), {noreply, State}; +handle_info({monitor_conf_update, OS}, _State) -> + SysHW = init_os_monitor(OS), + {noreply, #{sysmem_high_watermark => SysHW}}; handle_info(Info, State) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 78da63057..732cc2321 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -35,32 +35,52 @@ terminate/2, code_change/3 ]). +-export([add_handler/0, remove_handler/0, post_config_update/5]). +-export([update/1]). -define(SYSMON, ?MODULE). +-define(SYSMON_CONF_ROOT, [sysmon]). %% @doc Start the system monitor. -spec start_link() -> startlink_ret(). start_link() -> gen_server:start_link({local, ?SYSMON}, ?MODULE, [], []). +add_handler() -> + ok = emqx_config_handler:add_handler(?SYSMON_CONF_ROOT, ?MODULE), + ok. + +remove_handler() -> + ok = emqx_config_handler:remove_handler(?SYSMON_CONF_ROOT), + ok. + +post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> + #{os := OS1, vm := VM1} = OldConf, + #{os := OS2, vm := VM2} = NewConf, + VM1 =/= VM2 andalso ?MODULE:update(VM2), + OS1 =/= OS2 andalso emqx_os_mon:update(OS2), + ok. + +update(VM) -> + erlang:send(?MODULE, {monitor_conf_update, VM}). + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- init([]) -> - _ = erlang:system_monitor(self(), sysm_opts()), emqx_logger:set_proc_metadata(#{sysmon => true}), + init_system_monitor(), %% Monitor cluster partition event ekka:monitor(partition, fun handle_partition_event/1), - {ok, start_timer(#{timer => undefined, events => []})}. start_timer(State) -> State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. -sysm_opts() -> - sysm_opts(maps:to_list(emqx:get_config([sysmon, vm])), []). +sysm_opts(VM) -> + sysm_opts(maps:to_list(VM), []). sysm_opts([], Acc) -> Acc; sysm_opts([{_, disabled} | Opts], Acc) -> @@ -176,12 +196,16 @@ handle_info({monitor, SusPid, busy_dist_port, Port}, State) -> ); handle_info({timeout, _Ref, reset}, State) -> {noreply, State#{events := []}, hibernate}; +handle_info({monitor_conf_update, VM}, State) -> + init_system_monitor(VM), + {noreply, State#{events := []}, hibernate}; handle_info(Info, State) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_misc:cancel_timer(TRef), + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -237,3 +261,11 @@ safe_publish(Event, WarnMsg) -> sysmon_msg(Topic, Payload) -> Msg = emqx_message:make(?SYSMON, Topic, Payload), emqx_message:set_flag(sys, Msg). + +init_system_monitor() -> + VM = emqx:get_config([sysmon, vm]), + init_system_monitor(VM). + +init_system_monitor(VM) -> + _ = erlang:system_monitor(self(), sysm_opts(VM)), + ok. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 299c20c28..498503495 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -86,7 +86,7 @@ handle_info({timeout, _Timer, check}, State) -> }, Message ); - _Precent -> + _Percent -> ok end, _ = start_check_timer(), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index e2e73fe1f..97939bbaf 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -50,6 +50,7 @@ t_update(_Config) -> {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, ?assertEqual(BusyPort, not BusyPort1), + assert_busy_port(BusyPort1), %% update failed ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), @@ -64,6 +65,7 @@ t_update(_Config) -> ok = reset_config(<<"sysmon">>, "conf_path=vm.busy_port"), {ok, SysMon3} = get_config(<<"sysmon">>), ?assertMatch(#{<<"vm">> := #{<<"busy_port">> := true}}, SysMon3), + assert_busy_port(true), %% reset no_default_value config NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), @@ -73,6 +75,11 @@ t_update(_Config) -> ?assertMatch(#{<<"vm">> := #{<<"busy_port">> := false}}, SysMon4), ok. +assert_busy_port(BusyPort) -> + {_Pid, Monitors} = erlang:system_monitor(), + RealBusyPort = proplists:get_value(busy_port, Monitors, false), + ?assertEqual(BusyPort, RealBusyPort). + t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", From f080715ce93bc0d56b72857b9523ad58ece9bc0f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 21 Jul 2022 15:44:54 +0800 Subject: [PATCH 041/105] fix: ensure /listeners API return all node's status --- apps/emqx_management/src/emqx_mgmt_api_listeners.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 35fcd3bca..842ac4bfb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -549,7 +549,7 @@ format_status(Key, Node, Listener, Acc) -> NodeStatus = case maps:find(Node, NodeStatus0) of error -> - #{ + NodeStatus0#{ Node => #{ max_connections => MaxConnections, current_connections => CurrentConnections From dbeb28520b2751d5ae1cb5c542d62a75492c15c3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 21 Jul 2022 13:20:49 +0800 Subject: [PATCH 042/105] chore: bump exhook app vsn --- CHANGES-5.0.md | 2 +- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- mix.exs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 3a6a2f953..85487d234 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -14,7 +14,7 @@ * Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) * `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) * Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) -* Ensuring that exhook dispatches the client events are sequential. +* Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) ## Enhancements diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 43829c7be..9c518f8e0 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/mix.exs b/mix.exs index f4ed355ed..cb1efd341 100644 --- a/mix.exs +++ b/mix.exs @@ -54,6 +54,7 @@ defmodule EMQXUmbrella.MixProject do {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, + {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.4", override: true}, From e45422b2201814d9c95924447db6efb0ec95bae6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 21 Jul 2022 11:59:03 -0300 Subject: [PATCH 043/105] ci(mac): drop macos 10.15 Github is dropping support for macos 10.15 in its CI at 2022-08-30. --- .github/workflows/build_packages.yaml | 1 - .github/workflows/build_slim_packages.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cce8a8d2e..6877b25d7 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -143,7 +143,6 @@ jobs: - 24.2.1-1 os: - macos-11 - - macos-10.15 runs-on: ${{ matrix.os }} steps: - uses: actions/download-artifact@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index ba27c1b38..69eab7804 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -133,7 +133,6 @@ jobs: - 24.2.1-1 macos: - macos-11 - - macos-10.15 runs-on: ${{ matrix.macos }} From be9efcae2e9c0be364942953eaf8a345d22cd92b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 21 Jul 2022 15:40:29 -0300 Subject: [PATCH 044/105] fix: dashboard monitor crash after badrpc Accumulator becomes unusable after a `badrpc` error. ``` 2022-07-21T15:22:17.825688-03:00 [warning] exception: error, line: 116, mfa: minirest_handler:apply_callback/3, path: /monitor_current, reason: {badmap,{badrpc,{'emqx@emqx-17.int.thales',{'emqx@emqx-17.int.thales',timeout}}}}, stacktrace: [{maps,get,[connections,{badrpc,{'emqx@emqx-17.int.thales',{'emqx@emqx-17.int.thales',timeout}}},0],[{file,"maps.erl"},{line,517},{error_info,#{module => erl_stdlib_errors}}]},{emqx_dashboard_monitor,'-merge_cluster_rate/2-fun-0-',3,[{file,"/emqx/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl"},{line,259}]},{maps,fold_1,3,[{file,"maps.erl"},{line,410}]},{lists,foldl,3,[{file,"lists.erl"},{line,1267}]},{emqx_dashboard_monitor,current_rate,0,[{file,"/emqx/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl"},{line,126}]},{emqx_dashboard_monitor_api,monitor_current,2,[{file,"/emqx/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl"},{line,135}]},{minirest_handler,apply_callback,3,[{file,"/emqx/deps/minirest/src/minirest_handler.erl"},{line,111}]},{minirest_handler,handle,2,[{file,"/emqx/deps/minirest/src/minirest_handler.erl"},{line,44}]},{minirest_handler,init,2,[{file,"/emqx/deps/minirest/src/minirest_handler.erl"},{line,27}]},{cowboy_handler,execute,2,[{file,"/emqx/deps/cowboy/src/cowboy_handler.erl"},{line,41}]},{cowboy_stream_h,execute,3,[{file,"/emqx/deps/cowboy/src/cowboy_stream_h.erl"},{line,318}]},{cowboy_stream_h,request_process,3,[{file,"/emqx/deps/cowboy/src/cowboy_stream_h.erl"},{line,302}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}] ``` --- .../src/emqx_dashboard_monitor.erl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl index 445e2d2b7..910e7f1dd 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl @@ -115,13 +115,16 @@ granularity_adapter(List) -> %% Get the current rate. Not the current sampler data. current_rate() -> Fun = - fun(Node, Cluster) -> - case current_rate(Node) of - {ok, CurrentRate} -> - merge_cluster_rate(CurrentRate, Cluster); - {badrpc, Reason} -> - {badrpc, {Node, Reason}} - end + fun + (Node, Cluster) when is_map(Cluster) -> + case current_rate(Node) of + {ok, CurrentRate} -> + merge_cluster_rate(CurrentRate, Cluster); + {badrpc, Reason} -> + {badrpc, {Node, Reason}} + end; + (_Node, Error) -> + Error end, case lists:foldl(Fun, #{}, mria_mnesia:cluster_nodes(running)) of {badrpc, Reason} -> From 4d5dfe265b1ecc3766ad10da3107016e01e32f9d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 19 Jul 2022 10:02:29 +0800 Subject: [PATCH 045/105] fix: logger json formatter's config not working --- CHANGES-5.0.md | 1 + apps/emqx/src/emqx_logger_jsonfmt.erl | 5 +++-- apps/emqx_conf/i18n/emqx_conf_schema.conf | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 8f390df9d..831eb3ce2 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -14,6 +14,7 @@ * Fix AuthN JWKS SSL schema. Using schema in `emqx_schema`. [#8458](https://github.com/emqx/emqx/pull/8458) * `sentinel` field should be required when AuthN/AuthZ Redis using sentinel mode. [#8458](https://github.com/emqx/emqx/pull/8458) * Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) +* Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) ## Enhancements diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index b9f1d4fa8..03adeed04 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -69,9 +69,10 @@ best_effort_json(Input, Opts) -> jsx:encode(JsonReady, Opts). -spec format(logger:log_event(), config()) -> iodata(). -format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) -> +format(#{level := Level, msg := Msg, meta := Meta} = Event, Config0) when is_map(Config0) -> Config = add_default_config(Config0), - [format(Msg, Meta#{level => Level}, Config), "\n"]. + MsgBin = format(Msg, Meta#{level => Level}, Config), + logger_formatter:format(Event#{msg => {string, MsgBin}}, Config). format(Msg, Meta, Config) -> Data0 = diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 2951e3370..40882cbea 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -1039,12 +1039,18 @@ Defaults to: system. common_handler_chars_limit { desc { - en: """Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated.""" - zh: """设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。""" + en: """ +Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated. +NOTE: Restrict char limiter if formatter is json , it will get a truncated incomplete json data, which is not recommended. +""" + zh: """ +设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。 +注意：如果日志格式为 json，限制字符长度可能会导致截断不完整的 json 数据。 +""" } label { en: "Single Log Max Length" - zh: "单个日志最大长度" + zh: "单条日志长度限制" } } From 7b270db8d9de83bd3bc84340caa527c9e59d7e0c Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 21 Jul 2022 21:19:59 +0800 Subject: [PATCH 046/105] chore: update docker entry point, support dns cluster in k8s --- .github/workflows/run_fvt_tests.yaml | 6 +++++- deploy/charts/emqx/Chart.yaml | 2 +- deploy/charts/emqx/values.yaml | 14 +++++++------- deploy/docker/docker-entrypoint.sh | 29 ++++++++++++++++++---------- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8d9d65d2e..9e56dd6b3 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -157,6 +157,10 @@ jobs: if: matrix.discovery == 'k8s' run: | helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ + --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ + --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="emqx-headless" \ + --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ --set emqxAclConfig="" \ @@ -173,8 +177,8 @@ jobs: run: | helm install emqx \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ - --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ --set emqxAclConfig="" \ diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index caf453e7e..e18bac2b8 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,7 +14,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.4.1 +version: 5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 07ef543e0..7ed4f4995 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -91,13 +91,13 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: - EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s" - # EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" - # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" - # EMQX_CLUSTER__DNS__RECORD_TYPE: "srv" - EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" - EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" - EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" + EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" + EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + EMQX_CLUSTER__DNS__RECORD_TYPE: "srv" + # EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s" + # EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + # EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + # EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note：Hostname is only supported after v4.0-rc.2 diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 1638ebd7c..1c18ef829 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -16,22 +16,31 @@ shopt -s nullglob LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1) -if [[ -z "$EMQX_NODE_NAME" ]]; then - EMQX_NAME="${EMQX_NAME:-emqx}" - if [[ -z "$EMQX_HOST" ]]; then - if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then +export EMQX_NAME="${EMQX_NAME:-emqx}" + +if [[ -z "$EMQX_HOST" ]]; then + if [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "dns" ]] && \ + [[ "$EMQX_CLUSTER__DNS__RECORD_TYPE" == "srv" ]] && \ + grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then + EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME" + elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" - elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") - else - EMQX_HOST="$LOCAL_IP" - fi + else + EMQX_HOST="$LOCAL_IP" fi + export EMQX_HOST +fi + +if [[ -z "$EMQX_NODE_NAME" ]]; then export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST" - unset EMQX_NAME - unset EMQX_HOST fi # The default rpc port discovery 'stateless' is mostly for clusters From 15c8110af219582949304f62436a16eac17545b4 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 22 Jul 2022 17:14:24 +0800 Subject: [PATCH 047/105] fix(limiter): lift the level of the `client` field --- apps/emqx/i18n/emqx_limiter_i18n.conf | 6 +- .../emqx_limiter/src/emqx_limiter_schema.erl | 93 +++++++++++++------ .../emqx_limiter/src/emqx_limiter_server.erl | 58 ++++++------ apps/emqx/src/emqx_schema.erl | 9 +- .../src/emqx_dashboard_swagger.erl | 2 + .../src/emqx_retainer_schema.erl | 2 +- 6 files changed, 107 insertions(+), 63 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index b75435225..99ecc9e1e 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -89,10 +89,10 @@ the check/consume will succeed, but it will be forced to wait for a short period } } - per_client { + client { desc { - en: """The rate limit for each user of the bucket, this field is not required""" - zh: """对桶的每个使用者的速率控制设置，这个不是必须的""" + en: """The rate limit for each user of the bucket""" + zh: """对桶的每个使用者的速率控制设置""" } label: { en: """Per Client""" diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 61dc95bc7..f5e90a7e8 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -32,8 +32,7 @@ desc/1, types/0, infinity_value/0, - bucket_opts/0, - bucket_opts_meta/0 + bucket_fields/1 ]). -define(KILOBYTE, 1024). @@ -96,7 +95,17 @@ fields(limiter) -> default => make_limiter_default(Type) })} || Type <- types() - ]; + ] ++ + [ + {client, + ?HOCON( + ?R_REF(client_fields), + #{ + desc => ?DESC(client), + default => #{} + } + )} + ]; fields(node_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, @@ -104,15 +113,22 @@ fields(node_opts) -> ?HOCON(burst_rate(), #{ desc => ?DESC(burst), default => 0 - })}, - {client, ?HOCON(?R_REF(client_opts), #{default => #{}})} + })} + ]; +fields(client_fields) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + default => #{} + })} + || Type <- types() ]; fields(bucket_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, - {client, ?HOCON(?R_REF(client_opts), #{required => false})} + {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})} ]; fields(client_opts) -> [ @@ -159,14 +175,23 @@ fields(client_opts) -> default => force } )} - ]. + ]; +fields({client_fields, Types}) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + required => false + })} + || Type <- Types + ]; +fields({bucket_fields, Types}) -> + bucket_fields(Types). desc(limiter) -> "Settings for the rate limiter."; desc(node_opts) -> "Settings for the limiter of the node level."; -desc(node_client_opts) -> - "Settings for the client in the node level."; desc(bucket_opts) -> "Settings for the bucket."; desc(client_opts) -> @@ -174,23 +199,37 @@ desc(client_opts) -> desc(_) -> undefined. -bucket_opts() -> - ?HOCON( - ?MAP("bucket_name", ?R_REF(bucket_opts)), - bucket_opts_meta() - ). - -bucket_opts_meta() -> - #{ - default => #{}, - example => - #{ - <<"rate">> => <<"infinity">>, - <<"capcity">> => <<"infinity">>, - <<"initial">> => <<"100">>, - <<"client">> => #{<<"rate">> => <<"infinity">>} - } - }. +bucket_fields(Type) when is_atom(Type) -> + fields(bucket_opts) ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, client_opts), + #{ + desc => ?DESC(client), + required => false + } + )} + ]; +bucket_fields(Types) -> + [ + {Type, + ?HOCON(?R_REF(?MODULE, bucket_opts), #{ + desc => ?DESC(?MODULE, Type), + required => false + })} + || Type <- Types + ] ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, {client_fields, Types}), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. %% default period is 100ms default_period() -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 997f6b788..66cafa7dc 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -116,35 +116,28 @@ %% If no bucket path is set in config, there will be no limit connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; -connect( - Id, - Type, - #{ - rate := BucketRate, - capacity := BucketSize - } = BucketCfg -) -> - case emqx_limiter_manager:find_bucket(Id, Type) of - {ok, Bucket} -> - case find_client_cfg(Type, BucketCfg) of - #{rate := CliRate, capacity := CliSize} = ClientCfg -> - {ok, - if - CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); - true -> - emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) - end}; - {error, invalid_node_cfg} = Error -> - ?SLOG(error, #{msg => "invalid_node_cfg", type => Type, id => Id}), - Error - end; +connect(Id, Type, Cfg) -> + case find_limiter_cfg(Type, Cfg) of + {undefined, _} -> + {ok, emqx_htb_limiter:make_infinity_limiter()}; + { + #{ + rate := BucketRate, + capacity := BucketSize + } = BucketCfg, + #{rate := CliRate, capacity := CliSize} = ClientCfg + } -> + {ok, + if + CliRate < BucketRate orelse CliSize < BucketSize -> + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, BucketCfg); + true -> + emqx_htb_limiter:make_ref_limiter(ClientCfg, BucketCfg) + end}; undefined -> ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), {error, invalid_bucket} - end; -connect(Id, Type, Paths) -> - connect(Id, Type, maps:get(Type, Paths, undefined)). + end. -spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. add_bucket(_Id, _Type, undefine) -> @@ -571,9 +564,16 @@ call(Type, Msg) -> gen_server:call(Pid, Msg) end. -find_client_cfg(Type, Cfg) -> - NodeCfg = emqx:get_config([limiter, Type, client], undefined), - BucketCfg = maps:get(client, Cfg, undefined), +find_limiter_cfg(Type, #{rate := _} = Cfg) -> + {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; +find_limiter_cfg(Type, Cfg) -> + { + maps:get(Type, Cfg), + find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + }. + +find_client_cfg(Type, BucketCfg) -> + NodeCfg = emqx:get_config([limiter, client, Type], undefined), merge_client_cfg(NodeCfg, BucketCfg). merge_client_cfg(undefined, BucketCfg) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0c8a073a0..81f5c922a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1619,10 +1619,13 @@ base_listener(Bind) -> )}, {"limiter", sc( - map("ratelimit_name", ?R_REF(emqx_limiter_schema, bucket_opts)), + ?R_REF( + emqx_limiter_schema, + {bucket_fields, [bytes_in, message_in, connection, message_routing]} + ), #{ - desc => ?DESC(base_listener_limiter) - %% TODO default => #{<<"connection">> => <<"default">>} + desc => ?DESC(base_listener_limiter), + default => #{} } )}, {"enable_authn", diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 80b4e9624..b9c1e5c61 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -755,6 +755,8 @@ to_bin(List) when is_list(List) -> end; to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_bin({Type, Args}) -> + unicode:characters_to_binary(io_lib:format("~p(~p)", [Type, Args])); to_bin(X) -> X. diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 22083ba2c..986eb4105 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - ?R_REF(emqx_limiter_schema, bucket_opts), + ?R_REF(emqx_limiter_schema, {bucket_fields, internal}), batch_deliver_limiter, undefined )} From ce46cb9216efe56f875db5b7ef1a2498e06ee771 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 22 Jul 2022 18:59:02 +0800 Subject: [PATCH 048/105] fix(limiter): fix test case error --- .../emqx_limiter/src/emqx_limiter_schema.erl | 5 +++- .../emqx_limiter/src/emqx_limiter_server.erl | 29 ++++++++++-------- apps/emqx/src/emqx_listeners.erl | 4 +-- apps/emqx/test/emqx_channel_SUITE.erl | 14 +++++---- apps/emqx/test/emqx_connection_SUITE.erl | 13 ++++---- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 30 ++++++++++--------- apps/emqx/test/emqx_ws_connection_SUITE.erl | 29 ++++++++++-------- 7 files changed, 71 insertions(+), 53 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index f5e90a7e8..23d4d4e4c 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -102,7 +102,10 @@ fields(limiter) -> ?R_REF(client_fields), #{ desc => ?DESC(client), - default => #{} + default => maps:from_list([ + {erlang:atom_to_binary(Type), #{}} + || Type <- types() + ]) } )} ]; diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 66cafa7dc..c5e919296 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -124,19 +124,22 @@ connect(Id, Type, Cfg) -> #{ rate := BucketRate, capacity := BucketSize - } = BucketCfg, + }, #{rate := CliRate, capacity := CliSize} = ClientCfg } -> - {ok, - if - CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, BucketCfg); - true -> - emqx_htb_limiter:make_ref_limiter(ClientCfg, BucketCfg) - end}; - undefined -> - ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), - {error, invalid_bucket} + case emqx_limiter_manager:find_bucket(Id, Type) of + {ok, Bucket} -> + {ok, + if + CliRate < BucketRate orelse CliSize < BucketSize -> + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); + true -> + emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) + end}; + undefined -> + ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), + {error, invalid_bucket} + end end. -spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. @@ -523,7 +526,7 @@ make_bucket( _ = put_to_counter(Counter, NewIndex, Initial), Ref = emqx_limiter_bucket_ref:new(Counter, NewIndex, Rate), emqx_limiter_manager:insert_bucket(Id, Type, Ref), - State#{buckets := Buckets#{Id => Bucket}}. + State#{buckets := Buckets#{Id => Bucket}, index := NewIndex}. do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> case maps:get(Id, Buckets, undefined) of @@ -568,7 +571,7 @@ find_limiter_cfg(Type, #{rate := _} = Cfg) -> {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; find_limiter_cfg(Type, Cfg) -> { - maps:get(Type, Cfg), + maps:get(Type, Cfg, undefined), find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) }. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index fdbd5350e..9ccc5f2df 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -538,13 +538,13 @@ zone(Opts) -> limiter(Opts) -> maps:get(limiter, Opts, #{}). -add_limiter_bucket(Id, #{limiter := Limiters}) -> +add_limiter_bucket(Id, #{limiter := Limiter}) -> maps:fold( fun(Type, Cfg, _) -> emqx_limiter_server:add_bucket(Id, Type, Cfg) end, ok, - Limiters + maps:without([client], Limiter) ); add_limiter_bucket(_Id, _Cfg) -> ok. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 03be7448c..df1720772 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1205,9 +1205,7 @@ session(InitFields) when is_map(InitFields) -> quota() -> emqx_limiter_container:get_limiter_by_types(?MODULE, [message_routing], limiter_cfg()). -limiter_cfg() -> #{message_routing => make_limiter_cfg()}. - -make_limiter_cfg() -> +limiter_cfg() -> Client = #{ rate => 5, initial => 0, @@ -1217,10 +1215,16 @@ make_limiter_cfg() -> max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => 10, initial => 0, capacity => 10}. + #{ + message_routing => bucket_cfg(), + client => #{message_routing => Client} + }. + +bucket_cfg() -> + #{rate => 10, initial => 0, capacity => 10}. add_bucket() -> - emqx_limiter_server:add_bucket(?MODULE, message_routing, make_limiter_cfg()). + emqx_limiter_server:add_bucket(?MODULE, message_routing, bucket_cfg()). del_bucket() -> emqx_limiter_server:del_bucket(?MODULE, message_routing). diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 141dbdad6..c5dfdf34a 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -708,11 +708,8 @@ init_limiter() -> emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). limiter_cfg() -> - Cfg = make_limiter_cfg(), - #{bytes_in => Cfg, message_in => Cfg}. - -make_limiter_cfg() -> Infinity = emqx_limiter_schema:infinity_value(), + Cfg = bucket_cfg(), Client = #{ rate => Infinity, initial => 0, @@ -722,10 +719,14 @@ make_limiter_cfg() -> max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + +bucket_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + #{rate => Infinity, initial => 0, capacity => Infinity}. add_bucket() -> - Cfg = make_limiter_cfg(), + Cfg = bucket_cfg(), emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index e31a220e9..7efcbaa18 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -310,8 +310,8 @@ t_capacity(_) -> %% Test Cases Global Level %%-------------------------------------------------------------------- t_collaborative_alloc(_) -> - GlobalMod = fun(Cfg) -> - Cfg#{rate := ?RATE("600/1s")} + GlobalMod = fun(#{message_routing := MR} = Cfg) -> + Cfg#{message_routing := MR#{rate := ?RATE("600/1s")}} end, Bucket1 = fun(#{client := Cli} = Bucket) -> @@ -350,10 +350,12 @@ t_collaborative_alloc(_) -> ). t_burst(_) -> - GlobalMod = fun(Cfg) -> + GlobalMod = fun(#{message_routing := MR} = Cfg) -> Cfg#{ - rate := ?RATE("200/1s"), - burst := ?RATE("400/1s") + message_routing := MR#{ + rate := ?RATE("200/1s"), + burst := ?RATE("400/1s") + } } end, @@ -391,8 +393,8 @@ t_burst(_) -> ). t_limit_global_with_unlimit_other(_) -> - GlobalMod = fun(Cfg) -> - Cfg#{rate := ?RATE("600/1s")} + GlobalMod = fun(#{message_routing := MR} = Cfg) -> + Cfg#{message_routing := MR#{rate := ?RATE("600/1s")}} end, Bucket = fun(#{client := Cli} = Bucket) -> @@ -433,11 +435,11 @@ t_check_container(_) -> capacity := 1000 } end, - Case = fun(BucketCfg) -> + Case = fun(#{client := Client} = BucketCfg) -> C1 = emqx_limiter_container:get_limiter_by_types( ?MODULE, [message_routing], - #{message_routing => BucketCfg} + #{message_routing => BucketCfg, client => #{message_routing => Client}} ), {ok, C2} = emqx_limiter_container:check(1000, message_routing, C1), {pause, Pause, C3} = emqx_limiter_container:check(1000, message_routing, C2), @@ -455,8 +457,8 @@ t_check_container(_) -> %%-------------------------------------------------------------------- t_bucket_no_client(_) -> Rate = ?RATE("1/s"), - GlobalMod = fun(#{client := Client} = Cfg) -> - Cfg#{client := Client#{rate := Rate}} + GlobalMod = fun(#{client := #{message_routing := MR} = Client} = Cfg) -> + Cfg#{client := Client#{message_routing := MR#{rate := Rate}}} end, BucketMod = fun(Bucket) -> maps:remove(client, Bucket) @@ -470,8 +472,8 @@ t_bucket_no_client(_) -> t_bucket_client(_) -> GlobalRate = ?RATE("1/s"), BucketRate = ?RATE("10/s"), - GlobalMod = fun(#{client := Client} = Cfg) -> - Cfg#{client := Client#{rate := GlobalRate}} + GlobalMod = fun(#{client := #{message_routing := MR} = Client} = Cfg) -> + Cfg#{client := Client#{message_routing := MR#{rate := GlobalRate}}} end, BucketMod = fun(#{client := Client} = Bucket) -> Bucket#{client := Client#{rate := BucketRate}} @@ -682,7 +684,7 @@ to_rate(Str) -> Rate. with_global(Modifier, Buckets, Case) -> - with_config([limiter, message_routing], Modifier, Buckets, Case). + with_config([limiter], Modifier, Buckets, Case). with_bucket(Modifier, Case) -> Cfg = Modifier(make_limiter_cfg()), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 47591bf64..47efc1829 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -509,8 +509,9 @@ t_handle_timeout_emit_stats(_) -> t_ensure_rate_limit(_) -> {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), Limiter = init_limiter(#{ - bytes_in => make_limiter_cfg(Rate), - message_in => make_limiter_cfg() + bytes_in => bucket_cfg(), + message_in => bucket_cfg(), + client => #{bytes_in => client_cfg(Rate)} }), St = st(#{limiter => Limiter}), @@ -698,28 +699,32 @@ init_limiter(LimiterCfg) -> emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). limiter_cfg() -> - Cfg = make_limiter_cfg(), - #{bytes_in => Cfg, message_in => Cfg}. + Cfg = bucket_cfg(), + Client = client_cfg(), + #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. -make_limiter_cfg() -> +client_cfg() -> Infinity = emqx_limiter_schema:infinity_value(), - make_limiter_cfg(Infinity). + client_cfg(Infinity). -make_limiter_cfg(ClientRate) -> +client_cfg(Rate) -> Infinity = emqx_limiter_schema:infinity_value(), - Client = #{ - rate => ClientRate, + #{ + rate => Rate, initial => 0, capacity => Infinity, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force - }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + }. + +bucket_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + #{rate => Infinity, initial => 0, capacity => Infinity}. add_bucket() -> - Cfg = make_limiter_cfg(), + Cfg = bucket_cfg(), emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). From a6e45be2181ed7e281cdc90a6677ad3a28599f4e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 22 Jul 2022 09:18:33 -0300 Subject: [PATCH 049/105] chore(mix): add prefix to pre-release version to please elixir version parser Since upstream refuses to fix this issue: https://github.com/elixir-lang/elixir/issues/12000 On rare occasions, our pre-release version, which is the prefix of the git hash, might consist only of digits. Even more rarely, it might start with a `0`. When that happens, Elixir will refuse to parse that as a valid pre-release version (it wants it to be an integer without a leading 0). To prevent that, we will always prepend a "g" to the pre-release version, ensuring it's never a valid number. --- pkg-vsn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 7e5c44d27..7f50f03e2 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -101,7 +101,7 @@ if [ "$GIT_EXACT_VSN" != '' ]; then fi SUFFIX='' else - SUFFIX="-$(git rev-parse HEAD | cut -b1-8)" + SUFFIX="-g$(git rev-parse HEAD | cut -b1-8)" fi PKG_VSN="${PKG_VSN:-${RELEASE}${SUFFIX}}" From 343a78b08a3344d46c44f65ba1baa39cb18290cd Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 20 Jul 2022 15:06:24 -0300 Subject: [PATCH 050/105] feat(persistent_sessions): don't use rocksdb when unavailable --- CHANGES-5.0.md | 1 + .../emqx_persistent_session_backend_builtin.erl | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index a30aea08d..e278c9502 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -16,6 +16,7 @@ * Fix bad swagger format. [#8517](https://github.com/emqx/emqx/pull/8517) * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) +* Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) ## Enhancements diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl index f6dde5af4..59657470b 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl @@ -131,16 +131,23 @@ storage_properties(_, Backend) when ?IS_ETS(Backend) -> storage_properties(_, _) -> []. +%% Dialyzer sees the compiled literal in +%% `mria:rocksdb_backend_available/0' and complains about the +%% complementar match arm... +-dialyzer({no_match, table_type/1}). -spec table_type(atom()) -> mria_table_type(). table_type(Table) -> DiscPersistence = emqx_config:get([?cfg_root, on_disc]), RamCache = get_overlayed(Table, ram_cache), - case {DiscPersistence, RamCache} of - {true, true} -> + RocksDBAvailable = mria:rocksdb_backend_available(), + case {DiscPersistence, RamCache, RocksDBAvailable} of + {true, true, _} -> disc_copies; - {true, false} -> + {true, false, true} -> rocksdb_copies; - {false, _} -> + {true, false, false} -> + disc_copies; + {false, _, _} -> ram_copies end. From f8f4ad3e5b9143563c5192f096e60a8358f8eea0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 21 Jul 2022 11:43:58 +0800 Subject: [PATCH 051/105] fix(authn): fix cert_subject and cert_common_name placeholder --- apps/emqx_authn/src/emqx_authn_utils.erl | 11 +++-- .../test/emqx_authn_redis_SUITE.erl | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 8d3d45b1b..ac14d9dff 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -117,21 +117,21 @@ parse_sql(Template, ReplaceWith) -> render_deep(Template, Credential) -> emqx_placeholder:proc_tmpl_deep( Template, - Credential, + mapping_credential(Credential), #{return => full_binary, var_trans => fun handle_var/2} ). render_str(Template, Credential) -> emqx_placeholder:proc_tmpl( Template, - Credential, + mapping_credential(Credential), #{return => full_binary, var_trans => fun handle_var/2} ). render_sql_params(ParamList, Credential) -> emqx_placeholder:proc_tmpl( ParamList, - Credential, + mapping_credential(Credential), #{return => rawlist, var_trans => fun handle_sql_var/2} ). @@ -216,3 +216,8 @@ handle_sql_var({var, <<"peerhost">>}, PeerHost) -> emqx_placeholder:bin(inet:ntoa(PeerHost)); handle_sql_var(_, Value) -> emqx_placeholder:sql_data(Value). + +mapping_credential(C = #{cn := CN, dn := DN}) -> + C#{cert_common_name => CN, cert_subject => DN}; +mapping_credential(C) -> + C. diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 3423879f6..c91c8817f 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -475,6 +475,52 @@ user_seeds() -> } }, result => {ok, #{is_superuser => true}} + }, + + #{ + data => #{ + password => + <<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>, + is_superuser => <<"1">> + }, + credentials => #{ + clientid => <<"sha256_no_salt">>, + cn => <<"cert_common_name">>, + dn => <<"cert_subject_name">>, + password => <<"sha256_no_salt">> + }, + key => <<"mqtt_user:cert_common_name">>, + config_params => #{ + <<"cmd">> => <<"HMGET mqtt_user:${cert_common_name} password_hash is_superuser">>, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"sha256">>, + <<"salt_position">> => <<"disable">> + } + }, + result => {ok, #{is_superuser => true}} + }, + + #{ + data => #{ + password => + <<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>, + is_superuser => <<"1">> + }, + credentials => #{ + clientid => <<"sha256_no_salt">>, + cn => <<"cert_common_name">>, + dn => <<"cert_subject_name">>, + password => <<"sha256_no_salt">> + }, + key => <<"mqtt_user:cert_subject_name">>, + config_params => #{ + <<"cmd">> => <<"HMGET mqtt_user:${cert_subject} password_hash is_superuser">>, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"sha256">>, + <<"salt_position">> => <<"disable">> + } + }, + result => {ok, #{is_superuser => true}} } ]. From ba1347513e5ef2afe490565ed5326cf6bc4368fe Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 21 Jul 2022 20:03:16 +0800 Subject: [PATCH 052/105] test(authn): test cert_subject/cert_common_name placeholder --- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 12 +++++-- .../test/emqx_authn_mongo_SUITE.erl | 27 ++++++++++++++++ .../test/emqx_authn_mysql_SUITE.erl | 31 +++++++++++++++++++ .../test/emqx_authn_pgsql_SUITE.erl | 31 +++++++++++++++++++ .../test/emqx_authn_redis_SUITE.erl | 4 +-- 5 files changed, 100 insertions(+), 5 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 5384bcf6e..44ef43903 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -34,7 +34,9 @@ password => <<"plain">>, peerhost => {127, 0, 0, 1}, listener => 'tcp:default', - protocol => mqtt + protocol => mqtt, + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">> }). -define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)). @@ -517,7 +519,9 @@ samples() -> <<"username">> := <<"plain">>, <<"password">> := <<"plain">>, <<"clientid">> := <<"clienta">>, - <<"peerhost">> := <<"127.0.0.1">> + <<"peerhost">> := <<"127.0.0.1">>, + <<"cert_subject">> := <<"cert_subject_data">>, + <<"cert_common_name">> := <<"cert_common_name_data">> } = jiffy:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, @@ -534,7 +538,9 @@ samples() -> <<"clientid">> => ?PH_CLIENTID, <<"username">> => ?PH_USERNAME, <<"password">> => ?PH_PASSWORD, - <<"peerhost">> => ?PH_PEERHOST + <<"peerhost">> => ?PH_PEERHOST, + <<"cert_subject">> => ?PH_CERT_SUBJECT, + <<"cert_common_name">> => ?PH_CERT_CN_NAME } }, result => {ok, #{is_superuser => false, user_property => #{}}} diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index f68f9a528..2f7dd2391 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -345,6 +345,33 @@ user_seeds() -> result => {ok, #{is_superuser => true}} }, + #{ + data => #{ + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">>, + password_hash => + <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>, + salt => <<"salt">>, + is_superuser => 1 + }, + credentials => #{ + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">>, + password => <<"sha256">> + }, + config_params => #{ + <<"filter">> => #{ + <<"cert_subject">> => <<"${cert_subject}">>, + <<"cert_common_name">> => <<"${cert_common_name}">> + }, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"sha256">>, + <<"salt_position">> => <<"prefix">> + } + }, + result => {ok, #{is_superuser => true}} + }, + #{ data => #{ username => <<"bcrypt">>, diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index abf36c167..f7679f22b 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -318,6 +318,35 @@ user_seeds() -> result => {ok, #{is_superuser => true}} }, + #{ + data => #{ + username => "sha256", + password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf", + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">>, + salt => "salt", + is_superuser_int => 1 + }, + credentials => #{ + clientid => <<"sha256">>, + password => <<"sha256">>, + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">> + }, + config_params => #{ + <<"query">> => + << + "SELECT password_hash, salt, is_superuser_int as is_superuser\n" + " FROM users where cert_subject = ${cert_subject} AND cert_common_name = ${cert_common_name} LIMIT 1" + >>, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"sha256">>, + <<"salt_position">> => <<"prefix">> + } + }, + result => {ok, #{is_superuser => true}} + }, + #{ data => #{ username => <<"bcrypt">>, @@ -433,6 +462,8 @@ init_seeds() -> " username VARCHAR(255),\n" " password_hash VARCHAR(255),\n" " salt VARCHAR(255),\n" + " cert_subject VARCHAR(255),\n" + " cert_common_name VARCHAR(255),\n" " is_superuser_str VARCHAR(255),\n" " is_superuser_int TINYINT)" ), diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 2d9607c41..1d309c88d 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -380,6 +380,35 @@ user_seeds() -> result => {ok, #{is_superuser => true}} }, + #{ + data => #{ + username => "sha256", + password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf", + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">>, + salt => "salt", + is_superuser_int => 1 + }, + credentials => #{ + clientid => <<"sha256">>, + password => <<"sha256">>, + cert_subject => <<"cert_subject_data">>, + cert_common_name => <<"cert_common_name_data">> + }, + config_params => #{ + <<"query">> => + << + "SELECT password_hash, salt, is_superuser_int as is_superuser\n" + " FROM users where cert_subject = ${cert_subject} AND cert_common_name = ${cert_common_name} LIMIT 1" + >>, + <<"password_hash_algorithm">> => #{ + <<"name">> => <<"sha256">>, + <<"salt_position">> => <<"prefix">> + } + }, + result => {ok, #{is_superuser => true}} + }, + #{ data => #{ username => <<"bcrypt">>, @@ -474,6 +503,8 @@ init_seeds() -> " username varchar(255),\n" " password_hash varchar(255),\n" " salt varchar(255),\n" + " cert_subject varchar(255),\n" + " cert_common_name varchar(255),\n" " is_superuser_str varchar(255),\n" " is_superuser_int smallint,\n" " is_superuser_bool boolean)" diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index c91c8817f..dde5f8188 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -479,7 +479,7 @@ user_seeds() -> #{ data => #{ - password => + password_hash => <<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>, is_superuser => <<"1">> }, @@ -502,7 +502,7 @@ user_seeds() -> #{ data => #{ - password => + password_hash => <<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>, is_superuser => <<"1">> }, From 78449d77d12f972bcfb7e70b50367c03ef4dbb2f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 21 Jul 2022 20:08:00 +0800 Subject: [PATCH 053/105] chore: update changes --- CHANGES-5.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e278c9502..57fa98036 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,6 +17,7 @@ * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) +* Fix AuthN `cert_subject` and `cert_common_name` placeholder rendering failure. [#8531](https://github.com/emqx/emqx/pull/8531) ## Enhancements From e0b33dc2585f230554da7f5bd165cd0c40b72783 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 22 Jul 2022 10:31:59 +0800 Subject: [PATCH 054/105] fix(listener): support listen on ipv6 address --- apps/emqx/src/emqx_schema.erl | 27 ++++++++++++++++++++-- apps/emqx_management/src/emqx_mgmt_cli.erl | 14 ++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5652b37f7..a6cdb4772 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2113,9 +2113,13 @@ to_comma_separated_atoms(Str) -> to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. +%% @doc support the following format: +%% - 127.0.0.1:1883 +%% - ::1:1883 +%% - [::1]:1883 to_ip_port(Str) -> - case string:tokens(Str, ": ") of - [Ip, Port] -> + case split_ip_port(Str) of + {Ip, Port} -> PortVal = list_to_integer(Port), case inet:parse_address(Ip) of {ok, R} -> @@ -2133,6 +2137,25 @@ to_ip_port(Str) -> {error, Str} end. +split_ip_port(Str) -> + case lists:split(string:rchr(Str, $:), Str) of + %% no port + {[], Str} -> + error; + {IpPlusColon, PortString} -> + IpStr0 = lists:droplast(IpPlusColon), + case IpStr0 of + %% dropp head/tail brackets + [$[ | S] -> + case lists:last(S) of + $] -> {lists:droplast(S), PortString}; + _ -> error + end; + _ -> + {IpStr0, PortString} + end + end. + to_erl_cipher_suite(Str) -> case ssl:str_to_suite(Str) of {error, Reason} -> error({invalid_cipher, Reason}); diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index e6730a18c..c846c9d58 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -566,23 +566,23 @@ trace_type(_, _) -> error. listeners([]) -> lists:foreach( fun({ID, Conf}) -> - {Host, Port} = maps:get(bind, Conf), + Bind = maps:get(bind, Conf), Acceptors = maps:get(acceptors, Conf), ProxyProtocol = maps:get(proxy_protocol, Conf, undefined), Running = maps:get(running, Conf), CurrentConns = - case emqx_listeners:current_conns(ID, {Host, Port}) of + case emqx_listeners:current_conns(ID, Bind) of {error, _} -> []; CC -> [{current_conn, CC}] end, MaxConn = - case emqx_listeners:max_conns(ID, {Host, Port}) of + case emqx_listeners:max_conns(ID, Bind) of {error, _} -> []; MC -> [{max_conns, MC}] end, Info = [ - {listen_on, {string, format_listen_on(Port)}}, + {listen_on, {string, format_listen_on(Bind)}}, {acceptors, Acceptors}, {proxy_protocol, ProxyProtocol}, {running, Running} @@ -806,8 +806,10 @@ format_listen_on(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); format_listen_on({Addr, Port}) when is_list(Addr) -> io_lib:format("~ts:~w", [Addr, Port]); -format_listen_on({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). +format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 4 -> + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]); +format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 8 -> + io_lib:format("[~ts]:~w", [inet:ntoa(Addr), Port]). name(Filter) -> iolist_to_binary(["CLI-", Filter]). From 18a7ed95d08e6e5e1b7adc8df0fc70da8fe19e2f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 22 Jul 2022 10:33:37 +0800 Subject: [PATCH 055/105] chore: update changes --- CHANGES-5.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e278c9502..4a03a9c34 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,6 +17,7 @@ * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) +* Support listen on an IPv6 address. [#8547](https://github.com/emqx/emqx/pull/8547) ## Enhancements From 74c5fd411b04db805e96aae3568e174165402d81 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 25 Jul 2022 09:33:47 +0800 Subject: [PATCH 056/105] chore(schema): trim space in ip_port --- apps/emqx/src/emqx_schema.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index a6cdb4772..54b57b21d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2137,7 +2137,8 @@ to_ip_port(Str) -> {error, Str} end. -split_ip_port(Str) -> +split_ip_port(Str0) -> + Str = re:replace(Str0, " ", "", [{return, list}, global]), case lists:split(string:rchr(Str, $:), Str) of %% no port {[], Str} -> From 56417a3130eae6d51cd1243578ba47d4b973d97e Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 13 Jul 2022 16:26:46 +0800 Subject: [PATCH 057/105] feat: list rules support for pagination and fuzzy filtering --- CHANGES-5.0.md | 4 + Makefile | 2 +- .../i18n/emqx_rule_engine_api.conf | 40 +++++++ .../src/emqx_rule_engine_api.erl | 104 +++++++++++++++++- .../test/emqx_rule_engine_api_SUITE.erl | 77 ++++++++++++- 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e278c9502..f4baef697 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,6 +17,10 @@ * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) +* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472) + **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become + `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, + which will carry the paging meta information. ## Enhancements diff --git a/Makefile b/Makefile index a18cbee10..bde3e6d76 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.3 +export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf b/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf index c77c66d7c..96f999d41 100644 --- a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf +++ b/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf @@ -10,6 +10,46 @@ emqx_rule_engine_api { zh: "列出所有规则" } } + api1_enable { + desc { + en: "Filter enable/disable rules" + zh: "根据规则是否开启条件过滤" + } + } + + api1_from { + desc { + en: "Filter rules by from(topic), exact match" + zh: "根据规则来源 Topic 过滤, 需要完全匹配" + } + } + + api1_like_id { + desc { + en: "Filter rules by id, Substring matching" + zh: "根据规则 id 过滤, 使用子串模糊匹配" + } + } + + api1_like_from { + desc { + en: "Filter rules by from(topic), Substring matching" + zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配" + } + } + + api1_like_description { + desc { + en: "Filter rules by description, Substring matching" + zh: "根据规则描述过滤, 使用子串模糊匹配" + } + } + api1_match_from { + desc { + en: "Filter rules by from(topic), Mqtt topic matching" + zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" + } + } api2 { desc { diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 658781636..597ee838f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -33,6 +33,9 @@ %% API callbacks -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]). +%% query callback +-export([query/4]). + -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -109,6 +112,15 @@ end). } ). +-define(RULE_QS_SCHEMA, [ + {<<"enable">>, atom}, + {<<"from">>, binary}, + {<<"like_id">>, binary}, + {<<"like_from">>, binary}, + {<<"match_from">>, binary}, + {<<"like_description">>, binary} +]). + namespace() -> "rule". api_spec() -> @@ -134,9 +146,31 @@ schema("/rules") -> get => #{ tags => [<<"rules">>], description => ?DESC("api1"), + parameters => [ + {enable, + mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})}, + {from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})}, + {like_id, + mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})}, + {like_from, + mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})}, + {like_description, + mk(binary(), #{ + desc => ?DESC("api1_like_description"), in => query, required => false + })}, + {match_from, + mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})}, + ref(emqx_dashboard_swagger, page), + ref(emqx_dashboard_swagger, limit) + ], summary => <<"List Rules">>, responses => #{ - 200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")}) + 200 => + [ + {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})}, + {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} + ], + 400 => error_schema('BAD_REQUEST', "Invalid Parameters") } }, post => #{ @@ -236,9 +270,21 @@ param_path_id() -> '/rule_events'(get, _Params) -> {200, emqx_rule_events:event_info()}. -'/rules'(get, _Params) -> - Records = emqx_rule_engine:get_rules_ordered_by_ts(), - {200, format_rule_resp(Records)}; +'/rules'(get, #{query_string := QueryString}) -> + case + emqx_mgmt_api:node_query( + node(), + QueryString, + ?RULE_TAB, + ?RULE_QS_SCHEMA, + {?MODULE, query} + ) + of + {error, page_limit_invalid} -> + {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + Result -> + {200, Result} + end; '/rules'(post, #{body := Params0}) -> case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of <<>> -> @@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). format_rule_resp(Rules) when is_list(Rules) -> [format_rule_resp(R) || R <- Rules]; +format_rule_resp({Id, Rule}) -> + format_rule_resp(Rule#{id => Id}); format_rule_resp(#{ id := Id, name := Name, @@ -503,3 +551,51 @@ filter_out_request_body(Conf) -> <<"node">> ], maps:without(ExtraConfs, Conf). + +query(Tab, {Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(), + FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy), + emqx_mgmt_api:select_table_with_count( + Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1 + ). + +%% rule is not a record, so everything is fuzzy filter. +qs2ms() -> + [{'_', [], ['$_']}]. + +fuzzy_match_fun(Qs, Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter( + fun(E) -> + run_qs_match(E, Qs) andalso + run_fuzzy_match(E, Fuzzy) + end, + Ls + ) + end. + +run_qs_match(_, []) -> + true; +run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) -> + Enable =:= Pattern andalso run_qs_match(E, Qs); +run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) -> + lists:member(Pattern, From) andalso run_qs_match(E, Qs); +run_qs_match(E, [_ | Qs]) -> + run_qs_match(E, Qs). + +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) -> + binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) -> + binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) -> + lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) -> + lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E, [_ | Fuzzy]) -> + run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 13db82aa9..da4e299f9 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -45,7 +45,7 @@ t_crud_rule_api(_Config) -> ), ?assertEqual(RuleID, maps:get(id, Rule)), - {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}), + {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), @@ -91,6 +91,81 @@ t_crud_rule_api(_Config) -> ), ok. +t_list_rule_api(_Config) -> + AddIds = + lists:map( + fun(Seq0) -> + Seq = integer_to_binary(Seq0), + Params = #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => <<"test_rule", Seq/binary>> + }, + {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + Id + end, + lists:seq(1, 20) + ), + + {200, #{data := Rules, meta := #{count := Count}}} = + emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), + ?assertEqual(20, length(AddIds)), + ?assertEqual(20, length(Rules)), + ?assertEqual(20, Count), + + [RuleID | _] = AddIds, + UpdateParams = #{ + <<"description">> => <<"中文的描述也能搜索"/utf8>>, + <<"enable">> => false, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1/+\"">>, + <<"name">> => <<"test_rule_update1">> + }, + {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ + bindings => #{id => RuleID}, + body => UpdateParams + }), + QueryStr1 = #{query_string => #{<<"enable">> => false}}, + {200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1), + ?assertEqual(1, Count1), + + QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}}, + {200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2), + ?assertEqual(Result1, Result2), + + QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}}, + {200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3), + ?assertEqual(19, Count3), + + QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}}, + {200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4), + ?assertEqual(Result1, Result4), + + QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}}, + {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5), + ?assertEqual(Result1, Result5), + + QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}}, + {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6), + ?assertEqual(Result1, Result6), + + %% clean up + lists:foreach( + fun(Id) -> + ?assertMatch( + {204}, + emqx_rule_engine_api:'/rules/:id'( + delete, + #{bindings => #{id => Id}} + ) + ) + end, + AddIds + ), + ok. + test_rule_params() -> #{ body => #{ From 1c8defeedac4e4cd5107bfd3ee1c146b04df331c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 22 Jul 2022 10:50:54 +0800 Subject: [PATCH 058/105] chore: make elvis happy --- .../emqx_authn/test/emqx_authn_mysql_SUITE.erl | 17 +++++++++++++---- .../emqx_authn/test/emqx_authn_pgsql_SUITE.erl | 18 ++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index f7679f22b..0fdba0b31 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -337,7 +337,8 @@ user_seeds() -> <<"query">> => << "SELECT password_hash, salt, is_superuser_int as is_superuser\n" - " FROM users where cert_subject = ${cert_subject} AND cert_common_name = ${cert_common_name} LIMIT 1" + " FROM users where cert_subject = ${cert_subject} AND \n" + " cert_common_name = ${cert_common_name} LIMIT 1" >>, <<"password_hash_algorithm">> => #{ <<"name">> => <<"sha256">>, @@ -468,10 +469,18 @@ init_seeds() -> " is_superuser_int TINYINT)" ), - Fields = [username, password_hash, salt, is_superuser_str, is_superuser_int], + Fields = [ + username, + password_hash, + salt, + cert_subject, + cert_common_name, + is_superuser_str, + is_superuser_int + ], InsertQuery = - "INSERT INTO users(username, password_hash, salt, " - " is_superuser_str, is_superuser_int) VALUES(?, ?, ?, ?, ?)", + "INSERT INTO users(username, password_hash, salt, cert_subject, cert_common_name," + " is_superuser_str, is_superuser_int) VALUES(?, ?, ?, ?, ?, ?, ?)", lists:foreach( fun(#{data := Values}) -> diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 1d309c88d..ff017a79e 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -399,7 +399,8 @@ user_seeds() -> <<"query">> => << "SELECT password_hash, salt, is_superuser_int as is_superuser\n" - " FROM users where cert_subject = ${cert_subject} AND cert_common_name = ${cert_common_name} LIMIT 1" + " FROM users where cert_subject = ${cert_subject} AND \n" + " cert_common_name = ${cert_common_name} LIMIT 1" >>, <<"password_hash_algorithm">> => #{ <<"name">> => <<"sha256">>, @@ -518,12 +519,21 @@ init_seeds() -> ). create_user(Values) -> - Fields = [username, password_hash, salt, is_superuser_str, is_superuser_int, is_superuser_bool], + Fields = [ + username, + password_hash, + salt, + cert_subject, + cert_common_name, + is_superuser_str, + is_superuser_int, + is_superuser_bool + ], InsertQuery = - "INSERT INTO users(username, password_hash, salt," + "INSERT INTO users(username, password_hash, salt, cert_subject, cert_common_name, " "is_superuser_str, is_superuser_int, is_superuser_bool) " - "VALUES($1, $2, $3, $4, $5, $6)", + "VALUES($1, $2, $3, $4, $5, $6, $7, $8)", Params = [maps:get(F, Values, null) || F <- Fields], {ok, 1} = q(InsertQuery, Params), From 62561296c68e10b8f19842ab781e4f956f0d8204 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 25 Jul 2022 14:35:35 +0800 Subject: [PATCH 059/105] chore: update changes --- CHANGES-5.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 248939aee..4826526b6 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -17,7 +17,7 @@ * Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518) * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530) * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528) -* Support listen on an IPv6 address. [#8547](https://github.com/emqx/emqx/pull/8547) +* Support listen on an IPv6 address, e.g: [::1]:1883 or ::1:1883. [#8547](https://github.com/emqx/emqx/pull/8547) * GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472) **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, From a080c127a71646bdb0d8731db6ff5be4547e44b1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 25 Jul 2022 15:13:04 +0800 Subject: [PATCH 060/105] chore: `EMQ X` -> `EMQX` in icons --- README-CN.md | 4 ++-- README-JP.md | 2 +- README-RU.md | 4 ++-- README.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README-CN.md b/README-CN.md index 8e2a2d70f..772b4ba8a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -4,10 +4,10 @@ [![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) -[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) +[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) -[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) +[![Community](https://img.shields.io/badge/Community-EMQX-yellow)](https://askemq.com) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg) diff --git a/README-JP.md b/README-JP.md index d276316f5..bf0fc9188 100644 --- a/README-JP.md +++ b/README-JP.md @@ -4,7 +4,7 @@ [![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) -[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) +[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) diff --git a/README-RU.md b/README-RU.md index 2da6878a0..f047c90a1 100644 --- a/README-RU.md +++ b/README-RU.md @@ -4,10 +4,10 @@ [![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) -[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) +[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) -[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) +[![Community](https://img.shields.io/badge/Community-EMQX-yellow?logo=github)](https://github.com/emqx/emqx/discussions) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) [![The best IoT MQTT open source team looks forward to your joining](https://assets.emqx.com/images/github_readme_en_bg.png)](https://www.emqx.com/en/careers) diff --git a/README.md b/README.md index 0b335cb85..6441ec986 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) -[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) +[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) From dd59c850e01bf5a39f0d5d19c5dd9528a59dd521 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 25 Jul 2022 16:09:17 +0800 Subject: [PATCH 061/105] chore: make sure swagger's tags always titlecase --- apps/emqx_authn/src/emqx_authn_api.erl | 10 ++-------- apps/emqx_authn/src/emqx_authn_user_import_api.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_api_sources.erl | 9 +++++++++ .../src/emqx_auto_subscribe_api.erl | 2 ++ .../src/emqx_dashboard_error_code_api.erl | 2 ++ .../src/emqx_dashboard_monitor_api.erl | 8 ++++---- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 11 +++++++++-- apps/emqx_gateway/src/coap/emqx_coap_api.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_alarms.erl | 4 ++++ apps/emqx_management/src/emqx_mgmt_api_app.erl | 6 ++++++ apps/emqx_management/src/emqx_mgmt_api_banned.erl | 4 ++++ apps/emqx_management/src/emqx_mgmt_api_clients.erl | 12 ++++++++++++ apps/emqx_management/src/emqx_mgmt_api_cluster.erl | 3 +++ apps/emqx_management/src/emqx_mgmt_api_metrics.erl | 3 ++- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 4 ++++ apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 7 +++++++ apps/emqx_management/src/emqx_mgmt_api_publish.erl | 2 ++ apps/emqx_management/src/emqx_mgmt_api_stats.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_status.erl | 1 + .../src/emqx_mgmt_api_subscriptions.erl | 1 + apps/emqx_management/src/emqx_mgmt_api_sys.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_topics.erl | 3 +++ apps/emqx_management/src/emqx_mgmt_api_trace.erl | 8 ++++++++ apps/emqx_modules/src/emqx_telemetry_api.erl | 3 +++ apps/emqx_prometheus/src/emqx_prometheus_api.erl | 4 ++++ apps/emqx_slow_subs/src/emqx_slow_subs_api.erl | 9 +++++---- apps/emqx_statsd/src/emqx_statsd_api.erl | 2 +- 27 files changed, 103 insertions(+), 25 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index badf8024d..b07fb06ca 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -33,14 +33,8 @@ % Swagger --define(API_TAGS_GLOBAL, [ - ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"authentication config(global)">> -]). --define(API_TAGS_SINGLE, [ - ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"authentication config(single listener)">> -]). +-define(API_TAGS_GLOBAL, [<<"authentication">>]). +-define(API_TAGS_SINGLE, [<<"listener authentication">>]). -export([ api_spec/0, diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 48e623438..53e7b3d90 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -32,11 +32,11 @@ -define(API_TAGS_GLOBAL, [ ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"authentication config(global)">> + <<"authentication">> ]). -define(API_TAGS_SINGLE, [ ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"authentication config(single listener)">> + <<"listener authentication">> ]). -export([ diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 5f3873998..cb4c6f631 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -50,6 +50,8 @@ aggregate_metrics/1 ]). +-define(TAGS, [<<"Authorization">>]). + api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -70,6 +72,7 @@ schema("/authorization/sources") -> get => #{ description => ?DESC(authorization_sources_get), + tags => ?TAGS, responses => #{ 200 => mk( @@ -81,6 +84,7 @@ schema("/authorization/sources") -> post => #{ description => ?DESC(authorization_sources_post), + tags => ?TAGS, 'requestBody' => mk( hoconsc:union(authz_sources_type_refs()), #{desc => ?DESC(source_config)} @@ -101,6 +105,7 @@ schema("/authorization/sources/:type") -> get => #{ description => ?DESC(authorization_sources_type_get), + tags => ?TAGS, parameters => parameters_field(), responses => #{ @@ -114,6 +119,7 @@ schema("/authorization/sources/:type") -> put => #{ description => ?DESC(authorization_sources_type_put), + tags => ?TAGS, parameters => parameters_field(), 'requestBody' => mk(hoconsc:union(authz_sources_type_refs())), responses => @@ -125,6 +131,7 @@ schema("/authorization/sources/:type") -> delete => #{ description => ?DESC(authorization_sources_type_delete), + tags => ?TAGS, parameters => parameters_field(), responses => #{ @@ -139,6 +146,7 @@ schema("/authorization/sources/:type/status") -> get => #{ description => ?DESC(authorization_sources_type_status_get), + tags => ?TAGS, parameters => parameters_field(), responses => #{ @@ -159,6 +167,7 @@ schema("/authorization/sources/:type/move") -> post => #{ description => ?DESC(authorization_sources_type_move_post), + tags => ?TAGS, parameters => parameters_field(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl index d33861b09..4edb709b9 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl @@ -44,12 +44,14 @@ schema("/mqtt/auto_subscribe") -> 'operationId' => auto_subscribe, get => #{ description => ?DESC(list_auto_subscribe_api), + tags => [<<"Auto subscribe">>], responses => #{ 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe") } }, put => #{ description => ?DESC(update_auto_subscribe_api), + tags => [<<"Auto subscribe">>], 'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), responses => #{ 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index ff6f87258..2605ad91e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -51,6 +51,7 @@ schema("/error_codes") -> get => #{ security => [], description => <<"API Error Codes">>, + tags => [<<"Error Codes">>], responses => #{ 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code)) } @@ -62,6 +63,7 @@ schema("/error_codes/:code") -> get => #{ security => [], description => <<"API Error Codes">>, + tags => [<<"Error Codes">>], parameters => [ {code, hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index f1900dba5..bfb10692c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -37,7 +37,7 @@ schema("/monitor") -> #{ 'operationId' => monitor, get => #{ - tags => [dashboard], + tags => [<<"metrics">>], desc => <<"List monitor data.">>, parameters => [parameter_latest()], responses => #{ @@ -50,7 +50,7 @@ schema("/monitor/nodes/:node") -> #{ 'operationId' => monitor, get => #{ - tags => [dashboard], + tags => [<<"metrics">>], desc => <<"List the monitor data on the node.">>, parameters => [parameter_node(), parameter_latest()], responses => #{ @@ -63,7 +63,7 @@ schema("/monitor_current") -> #{ 'operationId' => monitor_current, get => #{ - tags => [dashboard], + tags => [<<"metrics">>], desc => <<"Current status. Gauge and rate.">>, responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}) @@ -74,7 +74,7 @@ schema("/monitor_current/nodes/:node") -> #{ 'operationId' => monitor_current, get => #{ - tags => [dashboard], + tags => [<<"metrics">>], desc => <<"Node current status. Gauge and rate.">>, parameters => [parameter_node()], responses => #{ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 3d65e569c..59b320368 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -338,10 +338,17 @@ to_spec(Meta, Params, RequestBody, Responses) -> maps:put('requestBody', RequestBody, Spec). generate_method_desc(Spec = #{desc := _Desc}) -> - trans_description(maps:remove(desc, Spec), Spec); + Spec1 = trans_description(maps:remove(desc, Spec), Spec), + trans_tags(Spec1); generate_method_desc(Spec = #{description := _Desc}) -> - trans_description(Spec, Spec); + Spec1 = trans_description(Spec, Spec), + trans_tags(Spec1); generate_method_desc(Spec) -> + trans_tags(Spec). + +trans_tags(Spec = #{tags := Tags}) -> + Spec#{tags => [string:titlecase(to_bin(Tag)) || Tag <- Tags]}; +trans_tags(Spec) -> Spec. parameters(Params, Module) -> diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index deba77445..7c0b13239 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -48,7 +48,7 @@ schema(?PREFIX ++ "/request") -> #{ operationId => request, post => #{ - tags => [<<"gateway|coap">>], + tags => [<<"coap">>], desc => ?DESC(send_coap_request), parameters => request_parameters(), requestBody => request_body(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index cc9b073b1..9135790bc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -26,6 +26,8 @@ -export([alarms/2]). +-define(TAGS, [<<"alarms">>]). + %% internal export (for query) -export([query/4]). @@ -40,6 +42,7 @@ schema("/alarms") -> 'operationId' => alarms, get => #{ description => ?DESC(list_alarms_api), + tags => ?TAGS, parameters => [ hoconsc:ref(emqx_dashboard_swagger, page), hoconsc:ref(emqx_dashboard_swagger, limit), @@ -59,6 +62,7 @@ schema("/alarms") -> }, delete => #{ description => ?DESC(delete_alarms_api), + tags => ?TAGS, responses => #{ 204 => ?DESC(delete_alarms_api_response204) } diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl index 2da060b55..89311a8d1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl @@ -22,6 +22,7 @@ -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). -export([api_key/2, api_key_by_name/2]). -export([validate_name/1]). +-define(TAGS, [<<"API keys">>]). namespace() -> "api_key". @@ -36,12 +37,14 @@ schema("/api_key") -> 'operationId' => api_key, get => #{ description => "Return api_key list", + tags => ?TAGS, responses => #{ 200 => delete([api_secret], fields(app)) } }, post => #{ description => "Create new api_key", + tags => ?TAGS, 'requestBody' => delete([created_at, api_key, api_secret], fields(app)), responses => #{ 200 => hoconsc:ref(app), @@ -54,6 +57,7 @@ schema("/api_key/:name") -> 'operationId' => api_key_by_name, get => #{ description => "Return the specific api_key", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 200 => delete([api_secret], fields(app)), @@ -62,6 +66,7 @@ schema("/api_key/:name") -> }, put => #{ description => "Update the specific api_key", + tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => delete([created_at, api_key, api_secret, name], fields(app)), responses => #{ @@ -71,6 +76,7 @@ schema("/api_key/:name") -> }, delete => #{ description => "Delete the specific api_key", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 204 => <<"Delete successfully">>, diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 1cf87b367..2eb8908c6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -39,6 +39,7 @@ ]). -define(TAB, emqx_banned). +-define(TAGS, [<<"Banned">>]). -define(BANNED_TYPES, [clientid, username, peerhost]). @@ -55,6 +56,7 @@ schema("/banned") -> 'operationId' => banned, get => #{ description => ?DESC(list_banned_api), + tags => ?TAGS, parameters => [ hoconsc:ref(emqx_dashboard_swagger, page), hoconsc:ref(emqx_dashboard_swagger, limit) @@ -68,6 +70,7 @@ schema("/banned") -> }, post => #{ description => ?DESC(create_banned_api), + tags => ?TAGS, 'requestBody' => hoconsc:mk(hoconsc:ref(ban)), responses => #{ 200 => [{data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})}], @@ -83,6 +86,7 @@ schema("/banned/:as/:who") -> 'operationId' => delete_banned, delete => #{ description => ?DESC(delete_banned_api), + tags => ?TAGS, parameters => [ {as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 956aecf79..0fd158dbd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -54,6 +54,7 @@ -export([do_subscribe/3]). -define(CLIENT_QTAB, emqx_channel_info). +-define(TAGS, [<<"Clients">>]). -define(CLIENT_QSCHEMA, [ {<<"node">>, atom}, @@ -100,6 +101,7 @@ schema("/clients") -> 'operationId' => clients, get => #{ description => <<"List clients">>, + tags => ?TAGS, parameters => [ hoconsc:ref(emqx_dashboard_swagger, page), hoconsc:ref(emqx_dashboard_swagger, limit), @@ -220,6 +222,7 @@ schema("/clients/:clientid") -> 'operationId' => client, get => #{ description => <<"Get clients info by client ID">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}), @@ -230,6 +233,7 @@ schema("/clients/:clientid") -> }, delete => #{ description => <<"Kick out client by client ID">>, + tags => ?TAGS, parameters => [ {clientid, hoconsc:mk(binary(), #{in => path})} ], @@ -246,6 +250,7 @@ schema("/clients/:clientid/authorization/cache") -> 'operationId' => authz_cache, get => #{ description => <<"Get client authz cache in the cluster.">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, authz_cache), #{}), @@ -256,6 +261,7 @@ schema("/clients/:clientid/authorization/cache") -> }, delete => #{ description => <<"Clean client authz cache in the cluster.">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ 204 => <<"Kick out client successfully">>, @@ -270,6 +276,7 @@ schema("/clients/:clientid/subscriptions") -> 'operationId' => subscriptions, get => #{ description => <<"Get client subscriptions">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ 200 => hoconsc:mk( @@ -286,6 +293,7 @@ schema("/clients/:clientid/subscribe") -> 'operationId' => subscribe, post => #{ description => <<"Subscribe">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, subscribe)), responses => #{ @@ -301,6 +309,7 @@ schema("/clients/:clientid/subscribe/bulk") -> 'operationId' => subscribe_batch, post => #{ description => <<"Subscribe">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscribe))), responses => #{ @@ -316,6 +325,7 @@ schema("/clients/:clientid/unsubscribe") -> 'operationId' => unsubscribe, post => #{ description => <<"Unsubscribe">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, unsubscribe)), responses => #{ @@ -331,6 +341,7 @@ schema("/clients/:clientid/unsubscribe/bulk") -> 'operationId' => unsubscribe_batch, post => #{ description => <<"Unsubscribe">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, unsubscribe))), responses => #{ @@ -346,6 +357,7 @@ schema("/clients/:clientid/keepalive") -> 'operationId' => set_keepalive, put => #{ description => <<"Set the online client keepalive by seconds">>, + tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, keepalive)), responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index e082fa745..74d5f3ba7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -40,6 +40,7 @@ schema("/cluster") -> 'operationId' => cluster_info, get => #{ description => "Get cluster info", + tags => [<<"Cluster">>], responses => #{ 200 => [ {name, ?HOCON(string(), #{desc => "Cluster name"})}, @@ -54,6 +55,7 @@ schema("/cluster/:node/invite") -> 'operationId' => invite_node, put => #{ description => "Invite node to cluster", + tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ 200 => <<"ok">>, @@ -66,6 +68,7 @@ schema("/cluster/:node/force_leave") -> 'operationId' => force_leave, delete => #{ description => "Force leave node from cluster", + tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ 204 => <<"Delete successfully">>, diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 69ce7723c..004913206 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -22,7 +22,7 @@ -import(hoconsc, [mk/2, ref/2]). -%% minirest/dashbaord_swagger behaviour callbacks +%% minirest/dashboard_swagger behaviour callbacks -export([ api_spec/0, paths/0, @@ -74,6 +74,7 @@ schema("/metrics") -> get => #{ description => <<"EMQX metrics">>, + tags => [<<"Metrics">>], parameters => [ {aggregate, diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 5e31bbb78..f1731db4d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -64,6 +64,7 @@ schema("/nodes") -> get => #{ description => <<"List EMQX nodes">>, + tags => [<<"Nodes">>], responses => #{ 200 => mk( @@ -79,6 +80,7 @@ schema("/nodes/:node") -> get => #{ description => <<"Get node info">>, + tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => #{ @@ -96,6 +98,7 @@ schema("/nodes/:node/metrics") -> get => #{ description => <<"Get node metrics">>, + tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => #{ @@ -113,6 +116,7 @@ schema("/nodes/:node/stats") -> get => #{ description => <<"Get node stats">>, + tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 6c625ba8c..88dd21518 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -48,6 +48,7 @@ ]). -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$"). +-define(TAGS, [<<"Plugins">>]). namespace() -> "plugins". @@ -72,6 +73,7 @@ schema("/plugins") -> "List all install plugins.
" "Plugins are launched in top-down order.
" "Using `POST /plugins/{name}/move` to change the boot order.", + tags => ?TAGS, responses => #{ 200 => hoconsc:array(hoconsc:ref(plugin)) } @@ -85,6 +87,7 @@ schema("/plugins/install") -> "Install a plugin(plugin-vsn.tar.gz)." "Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) " "to develop plugin.", + tags => ?TAGS, 'requestBody' => #{ content => #{ 'multipart/form-data' => #{ @@ -111,6 +114,7 @@ schema("/plugins/:name") -> 'operationId' => plugin, get => #{ description => "Describe a plugin according `release.json` and `README.md`.", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 200 => hoconsc:ref(plugin), @@ -119,6 +123,7 @@ schema("/plugins/:name") -> }, delete => #{ description => "Uninstall a plugin package.", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 204 => <<"Uninstall successfully">>, @@ -134,6 +139,7 @@ schema("/plugins/:name/:action") -> "start/stop a installed plugin.
" "- **start**: start the plugin.
" "- **stop**: stop the plugin.
", + tags => ?TAGS, parameters => [ hoconsc:ref(name), {action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})} @@ -149,6 +155,7 @@ schema("/plugins/:name/move") -> 'operationId' => update_boot_order, post => #{ description => "Setting the boot order of plugins.", + tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => move_request_body(), responses => #{200 => <<"OK">>} diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index b2b92d389..5442369ed 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -43,6 +43,7 @@ schema("/publish") -> 'operationId' => publish, post => #{ description => <<"Publish Message">>, + tags => [<<"Mqtt">>], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)), responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, publish_message_info)) @@ -54,6 +55,7 @@ schema("/publish/bulk") -> 'operationId' => publish_batch, post => #{ description => <<"Publish Messages">>, + tags => [<<"Mqtt">>], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}), responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message_info)), #{}) diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 6263380ad..e24c613b6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -50,7 +50,7 @@ schema("/stats") -> get => #{ description => <<"EMQX stats">>, - tags => [<<"stats">>], + tags => [<<"metrics">>], parameters => [ref(aggregate)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index 3a29d8567..5ece0bda4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -37,6 +37,7 @@ schema("/status") -> get => #{ description => <<"Node running status">>, + tags => [<<"Status">>], security => [], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 8bd418d43..7c7173a46 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -60,6 +60,7 @@ schema("/subscriptions") -> 'operationId' => subscriptions, get => #{ description => <<"List subscriptions">>, + tags => [<<"Subscriptions">>], parameters => parameters(), responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{}) diff --git a/apps/emqx_management/src/emqx_mgmt_api_sys.erl b/apps/emqx_management/src/emqx_mgmt_api_sys.erl index 0209e2d0e..df5ee15ab 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_sys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_sys.erl @@ -31,7 +31,7 @@ -export([sys/2]). --define(TAGS, [<<"sys">>]). +-define(TAGS, [<<"system topics">>]). namespace() -> "sys". diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 3b54a3bee..c357855b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -39,6 +39,7 @@ -define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND'). -define(TOPICS_QUERY_SCHEMA, [{<<"topic">>, binary}, {<<"node">>, atom}]). +-define(TAGS, [<<"Topics">>]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). @@ -51,6 +52,7 @@ schema("/topics") -> 'operationId' => topics, get => #{ description => <<"Topics list">>, + tags => ?TAGS, parameters => [ topic_param(query), node_param(), @@ -70,6 +72,7 @@ schema("/topics/:topic") -> 'operationId' => topic, get => #{ description => <<"Lookup topic info by name">>, + tags => ?TAGS, parameters => [topic_param(path)], responses => #{ 200 => hoconsc:mk(hoconsc:ref(topic), #{}), diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index a04b269f7..7a9ae5710 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -47,6 +47,7 @@ -define(TO_BIN(_B_), iolist_to_binary(_B_)). -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}). +-define(TAGS, [<<"Trace">>]). namespace() -> "trace". @@ -61,12 +62,14 @@ schema("/trace") -> 'operationId' => trace, get => #{ description => "List all trace", + tags => ?TAGS, responses => #{ 200 => hoconsc:ref(trace) } }, post => #{ description => "Create new trace", + tags => ?TAGS, 'requestBody' => delete([status, log_size], fields(trace)), responses => #{ 200 => hoconsc:ref(trace), @@ -82,6 +85,7 @@ schema("/trace") -> }, delete => #{ description => "Clear all traces", + tags => ?TAGS, responses => #{ 204 => <<"No Content">> } @@ -92,6 +96,7 @@ schema("/trace/:name") -> 'operationId' => delete_trace, delete => #{ description => "Delete trace by name", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 204 => <<"Delete successfully">>, @@ -104,6 +109,7 @@ schema("/trace/:name/stop") -> 'operationId' => update_trace, put => #{ description => "Stop trace by name", + tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ 200 => hoconsc:ref(trace), @@ -116,6 +122,7 @@ schema("/trace/:name/download") -> 'operationId' => download_trace_log, get => #{ description => "Download trace log by name", + tags => ?TAGS, parameters => [hoconsc:ref(name), hoconsc:ref(node)], responses => #{ 200 => @@ -134,6 +141,7 @@ schema("/trace/:name/log") -> 'operationId' => stream_log_file, get => #{ description => "view trace log", + tags => ?TAGS, parameters => [ hoconsc:ref(name), hoconsc:ref(bytes), diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index ce65f1052..a135ac3c3 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -52,12 +52,14 @@ schema("/telemetry/status") -> get => #{ description => ?DESC(get_telemetry_status_api), + tags => [<<"Telemetry">>], responses => #{200 => status_schema(?DESC(get_telemetry_status_api))} }, put => #{ description => ?DESC(update_telemetry_status_api), + tags => [<<"Telemetry">>], 'requestBody' => status_schema(?DESC(update_telemetry_status_api)), responses => #{ @@ -71,6 +73,7 @@ schema("/telemetry/data") -> get => #{ description => ?DESC(get_telemetry_data_api), + tags => [<<"Telemetry">>], responses => #{200 => mk(ref(?MODULE, telemetry), #{desc => ?DESC(get_telemetry_data_api)})} } diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index d024c5f69..a7971d2fd 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -34,6 +34,7 @@ ]). -define(SCHEMA_MODULE, emqx_prometheus_schema). +-define(TAGS, [<<"monitor">>]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -50,12 +51,14 @@ schema("/prometheus") -> get => #{ description => <<"Get Prometheus config info">>, + tags => ?TAGS, responses => #{200 => prometheus_config_schema()} }, put => #{ description => <<"Update Prometheus config">>, + tags => ?TAGS, 'requestBody' => prometheus_config_schema(), responses => #{200 => prometheus_config_schema()} @@ -67,6 +70,7 @@ schema("/prometheus/stats") -> get => #{ description => <<"Get Prometheus Data">>, + tags => ?TAGS, security => [], responses => #{200 => prometheus_data_schema()} diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl index ccd533600..cd2f76f11 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -25,6 +25,7 @@ -export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]). -export([slow_subs/2, get_history/0, settings/2]). +-define(TAGS, [<<"Slow subscriptions">>]). -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_mgmt_util, [bad_request/0]). @@ -44,14 +45,14 @@ schema(("/slow_subscriptions")) -> #{ 'operationId' => slow_subs, delete => #{ - tags => [<<"slow subs">>], + tags => ?TAGS, description => ?DESC(clear_records_api), parameters => [], 'requestBody' => [], responses => #{204 => <<"No Content">>} }, get => #{ - tags => [<<"slow subs">>], + tags => ?TAGS, description => ?DESC(get_records_api), parameters => [ ref(emqx_dashboard_swagger, page), @@ -65,12 +66,12 @@ schema("/slow_subscriptions/settings") -> #{ 'operationId' => settings, get => #{ - tags => [<<"slow subs">>], + tags => ?TAGS, description => ?DESC(get_setting_api), responses => #{200 => conf_schema()} }, put => #{ - tags => [<<"slow subs">>], + tags => ?TAGS, description => ?DESC(update_setting_api), 'requestBody' => conf_schema(), responses => #{200 => conf_schema()} diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 716147eca..0d9fcc24c 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -32,7 +32,7 @@ schema/1 ]). --define(API_TAG_STATSD, [<<"statsd">>]). +-define(API_TAG_STATSD, [<<"monitor">>]). -define(SCHEMA_MODULE, emqx_statsd_schema). -define(INTERNAL_ERROR, 'INTERNAL_ERROR'). From 99340e424e1257ed6a95ed51263b4d1942ae638c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 25 Jul 2022 16:14:03 +0800 Subject: [PATCH 062/105] chore: bump app.src version --- apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_slow_subs/src/emqx_slow_subs.app.src | 2 +- apps/emqx_statsd/src/emqx_statsd.app.src | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index 353928592..937f201bb 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ {description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index cc423f7c6..e446a572a 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 5a46ff92a..e87d293d3 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.0"}, + {vsn, "1.0.1"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 9e5829a11..21a972266 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "An OTP application"}, - {vsn, "5.0.0"}, + {vsn, "5.0.1"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ From 087a098d645142d56af0dcd398da2139f6b7144b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 16:26:51 +0800 Subject: [PATCH 063/105] chore: update apps/emqx_management/src/emqx_mgmt_api_publish.erl Co-authored-by: zhouzb --- apps/emqx_management/src/emqx_mgmt_api_publish.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index 5442369ed..c6f46b74f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -55,7 +55,7 @@ schema("/publish/bulk") -> 'operationId' => publish_batch, post => #{ description => <<"Publish Messages">>, - tags => [<<"Mqtt">>], + tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}), responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message_info)), #{}) From cc63479e698d4f6803810ee9557a288247073382 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 16:27:31 +0800 Subject: [PATCH 064/105] chore: update apps/emqx_management/src/emqx_mgmt_api_publish.erl Co-authored-by: zhouzb --- apps/emqx_management/src/emqx_mgmt_api_publish.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index c6f46b74f..bd214a87c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -43,7 +43,7 @@ schema("/publish") -> 'operationId' => publish, post => #{ description => <<"Publish Message">>, - tags => [<<"Mqtt">>], + tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)), responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, publish_message_info)) From a3296078c32fcb49fb9ebac859035312582d08cb Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 25 Jul 2022 16:36:36 +0800 Subject: [PATCH 065/105] chore: make sure swagger's tags always titlecase 2 --- apps/emqx_authn/src/emqx_authn_api.erl | 4 ++-- apps/emqx_authn/src/emqx_authn_user_import_api.erl | 11 ++--------- .../src/emqx_dashboard_error_code_api.erl | 4 ++-- .../emqx_dashboard/src/emqx_dashboard_monitor_api.erl | 8 ++++---- apps/emqx_gateway/src/coap/emqx_coap_api.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_alarms.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_stats.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_sys.erl | 2 +- apps/emqx_prometheus/src/emqx_prometheus_api.erl | 2 +- apps/emqx_statsd/src/emqx_statsd_api.erl | 2 +- 10 files changed, 16 insertions(+), 23 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index b07fb06ca..c7f8eb824 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -33,8 +33,8 @@ % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>]). --define(API_TAGS_SINGLE, [<<"listener authentication">>]). +-define(API_TAGS_GLOBAL, [<<"Authentication">>]). +-define(API_TAGS_SINGLE, [<<"Listener authentication">>]). -export([ api_spec/0, diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 53e7b3d90..6074aabeb 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -29,15 +29,8 @@ -define(NOT_FOUND, 'NOT_FOUND'). % Swagger - --define(API_TAGS_GLOBAL, [ - ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"authentication">> -]). --define(API_TAGS_SINGLE, [ - ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, - <<"listener authentication">> -]). +-define(API_TAGS_GLOBAL, [<<"authentication">>]). +-define(API_TAGS_SINGLE, [<<"listener authentication">>]). -export([ api_spec/0, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index 2605ad91e..139567828 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -51,7 +51,7 @@ schema("/error_codes") -> get => #{ security => [], description => <<"API Error Codes">>, - tags => [<<"Error Codes">>], + tags => [<<"Error codes">>], responses => #{ 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code)) } @@ -63,7 +63,7 @@ schema("/error_codes/:code") -> get => #{ security => [], description => <<"API Error Codes">>, - tags => [<<"Error Codes">>], + tags => [<<"Error codes">>], parameters => [ {code, hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index bfb10692c..e3ea870af 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -37,7 +37,7 @@ schema("/monitor") -> #{ 'operationId' => monitor, get => #{ - tags => [<<"metrics">>], + tags => [<<"Metrics">>], desc => <<"List monitor data.">>, parameters => [parameter_latest()], responses => #{ @@ -50,7 +50,7 @@ schema("/monitor/nodes/:node") -> #{ 'operationId' => monitor, get => #{ - tags => [<<"metrics">>], + tags => [<<"Metrics">>], desc => <<"List the monitor data on the node.">>, parameters => [parameter_node(), parameter_latest()], responses => #{ @@ -63,7 +63,7 @@ schema("/monitor_current") -> #{ 'operationId' => monitor_current, get => #{ - tags => [<<"metrics">>], + tags => [<<"Metrics">>], desc => <<"Current status. Gauge and rate.">>, responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}) @@ -74,7 +74,7 @@ schema("/monitor_current/nodes/:node") -> #{ 'operationId' => monitor_current, get => #{ - tags => [<<"metrics">>], + tags => [<<"Metrics">>], desc => <<"Node current status. Gauge and rate.">>, parameters => [parameter_node()], responses => #{ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 7c0b13239..f42bdcf9f 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -48,7 +48,7 @@ schema(?PREFIX ++ "/request") -> #{ operationId => request, post => #{ - tags => [<<"coap">>], + tags => [<<"CoAP gateway">>], desc => ?DESC(send_coap_request), parameters => request_parameters(), requestBody => request_body(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index 9135790bc..36845e4e7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -26,7 +26,7 @@ -export([alarms/2]). --define(TAGS, [<<"alarms">>]). +-define(TAGS, [<<"Alarms">>]). %% internal export (for query) -export([query/4]). diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index e24c613b6..b6462c713 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -50,7 +50,7 @@ schema("/stats") -> get => #{ description => <<"EMQX stats">>, - tags => [<<"metrics">>], + tags => [<<"Metrics">>], parameters => [ref(aggregate)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_sys.erl b/apps/emqx_management/src/emqx_mgmt_api_sys.erl index df5ee15ab..43fd9ee14 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_sys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_sys.erl @@ -31,7 +31,7 @@ -export([sys/2]). --define(TAGS, [<<"system topics">>]). +-define(TAGS, [<<"System topics">>]). namespace() -> "sys". diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index a7971d2fd..9a81f3ea3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -34,7 +34,7 @@ ]). -define(SCHEMA_MODULE, emqx_prometheus_schema). --define(TAGS, [<<"monitor">>]). +-define(TAGS, [<<"Monitor">>]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 0d9fcc24c..ee6007d7d 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -32,7 +32,7 @@ schema/1 ]). --define(API_TAG_STATSD, [<<"monitor">>]). +-define(API_TAG_STATSD, [<<"Monitor">>]). -define(SCHEMA_MODULE, emqx_statsd_schema). -define(INTERNAL_ERROR, 'INTERNAL_ERROR'). From 02c0755867176629dba06390372c39cd46b8204f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 25 Jul 2022 16:53:48 +0800 Subject: [PATCH 066/105] fix: dashboard failed test --- apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index d1dfe7988..912459b9d 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -196,7 +196,7 @@ t_in_mix(_Config) -> } ], ExpectMeta = #{ - tags => [tags, good], + tags => [<<"Tags">>, <<"Good">>], description => <<"good description">>, summary => <<"good summary">>, security => [], From 24f88679aeb8a555ab4a3ba7ba25a9b11f83fcd2 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 25 Jul 2022 10:45:06 +0200 Subject: [PATCH 067/105] feat: Added emqx command to check configs --- apps/emqx/rebar.config | 2 +- apps/emqx_prometheus/rebar.config | 2 +- bin/emqx | 16 +++++++++++++++- mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 0c0a09cc0..c5b591a79 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 2a0f20495..66afda52e 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -4,7 +4,7 @@ {emqx, {path, "../emqx"}}, %% FIXME: tag this as v3.1.3 {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}} + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/bin/emqx b/bin/emqx index 0d27d7168..cac5cf655 100755 --- a/bin/emqx +++ b/bin/emqx @@ -169,6 +169,9 @@ usage() { echo " --no-permanent Install release package VERSION but" echo " don't make it permanent" ;; + check_config) + echo "Checks the EMQX config without generating any files" + ;; *) echo "Usage: $REL_NAME COMMAND [help]" echo '' @@ -184,6 +187,7 @@ usage() { echo " Up/Down-grade: upgrade | downgrade | install | uninstall" echo " Install info: ertspath | root_dir" echo " Runtime info: pid | ping | versions" + echo " Validate Config: check_config" echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-erl" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" @@ -211,7 +215,7 @@ fi ## IS_BOOT_COMMAND is set for later to inspect node name and cookie from hocon config (or env variable) case "${COMMAND}" in - start|console|console_clean|foreground) + start|console|console_clean|foreground|check_config) IS_BOOT_COMMAND='yes' ;; ertspath) @@ -525,6 +529,12 @@ relx_start_command() { "$START_OPTION" } +# Function to check configs without generating them +check_config() { + ## this command checks the configs without generating any files + call_hocon -v -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf check_schema +} + # Function to generate app.config and vm.args # sets two environment variables CONF_FILE and ARGS_FILE generate_config() { @@ -1030,6 +1040,10 @@ case "${COMMAND}" in shift relx_nodetool "eval" "$@" ;; + + check_config) + check_config + ;; *) usage "$COMMAND" exit 1 diff --git a/mix.exs b/mix.exs index fc802cb5e..714279cfd 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.28.3", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.29.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 37dce676d..58b4b079b 100644 --- a/rebar.config +++ b/rebar.config @@ -67,7 +67,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 7261fbaa6b0fdb47ffa710df15d76eb13aa78e90 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 17:45:28 +0800 Subject: [PATCH 068/105] chore: Update apps/emqx_authn/src/emqx_authn_user_import_api.erl Co-authored-by: zhouzb --- apps/emqx_authn/src/emqx_authn_user_import_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 6074aabeb..0eea9f8b3 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -30,7 +30,7 @@ % Swagger -define(API_TAGS_GLOBAL, [<<"authentication">>]). --define(API_TAGS_SINGLE, [<<"listener authentication">>]). +-define(API_TAGS_SINGLE, [<<"Listener authentication">>]). -export([ api_spec/0, From 2466881460149de6d3b8981c0d9685f92ef9c82f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 17:45:38 +0800 Subject: [PATCH 069/105] chore: Update apps/emqx_authn/src/emqx_authn_user_import_api.erl Co-authored-by: zhouzb --- apps/emqx_authn/src/emqx_authn_user_import_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index 0eea9f8b3..47d4de81c 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -29,7 +29,7 @@ -define(NOT_FOUND, 'NOT_FOUND'). % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>]). +-define(API_TAGS_GLOBAL, [<<"Authentication">>]). -define(API_TAGS_SINGLE, [<<"Listener authentication">>]). -export([ From 1a236db91b17effd485c2af60cb119cecac5712b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 20 Jul 2022 19:40:19 +0300 Subject: [PATCH 070/105] chore(log): add authentication tracing --- CHANGES-5.0.md | 3 +- apps/emqx/include/emqx_authentication.hrl | 13 ++++ apps/emqx/include/logger.hrl | 18 +++-- apps/emqx/src/emqx_authentication.erl | 31 ++++++--- apps/emqx/src/emqx_trace/emqx_trace.erl | 15 ++-- .../src/emqx_trace/emqx_trace_formatter.erl | 53 ++++++++------- apps/emqx_authn/include/emqx_authn.hrl | 4 +- apps/emqx_authn/src/emqx_authn_utils.erl | 16 ++++- .../emqx_enhanced_authn_scram_mnesia.erl | 10 ++- .../src/simple_authn/emqx_authn_http.erl | 68 ++++++++++--------- .../src/simple_authn/emqx_authn_jwt.erl | 34 ++++++---- .../src/simple_authn/emqx_authn_mnesia.erl | 8 ++- .../src/simple_authn/emqx_authn_mongodb.erl | 7 +- .../src/simple_authn/emqx_authn_mysql.erl | 3 +- .../src/simple_authn/emqx_authn_pgsql.erl | 3 +- .../src/simple_authn/emqx_authn_redis.erl | 8 +-- 16 files changed, 182 insertions(+), 112 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e278c9502..6e22050fe 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -23,7 +23,8 @@ * Improve the dashboard listener startup log, the listener name is no longer spliced with port information, and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) -* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) +* Optimize performance of builtin database operations in processes with long message queue [#8439](https://github.com/emqx/emqx/pull/8439) +* Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554) # 5.0.3 diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl index dc501d1d8..6187b136c 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx/include/emqx_authentication.hrl @@ -17,6 +17,19 @@ -ifndef(EMQX_AUTHENTICATION_HRL). -define(EMQX_AUTHENTICATION_HRL, true). +-include_lib("emqx/include/logger.hrl"). + +-define(AUTHN_TRACE_TAG, "AUTHN"). + +-define(TRACE_AUTHN_PROVIDER(Msg), ?TRACE_AUTHN_PROVIDER(Msg, #{})). +-define(TRACE_AUTHN_PROVIDER(Msg, Meta), ?TRACE_AUTHN_PROVIDER(debug, Msg, Meta)). +-define(TRACE_AUTHN_PROVIDER(Level, Msg, Meta), + ?TRACE_AUTHN(Level, Msg, (Meta)#{provider => ?MODULE}) +). + +-define(TRACE_AUTHN(Msg, Meta), ?TRACE_AUTHN(debug, Msg, Meta)). +-define(TRACE_AUTHN(Level, Msg, Meta), ?TRACE(Level, ?AUTHN_TRACE_TAG, Msg, Meta)). + %% config root name all auth providers have to agree on. -define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). -define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 05e5359b5..d321330a8 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -42,17 +42,21 @@ -define(TRACE_FILTER, emqx_trace_filter). +-define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)). + %% Only evaluate when necessary -%% Always debug the trace events. --define(TRACE(Tag, Msg, Meta), begin - case persistent_term:get(?TRACE_FILTER, undefined) of - undefined -> ok; +-define(TRACE(Level, Tag, Msg, Meta), begin + case persistent_term:get(?TRACE_FILTER, []) of [] -> ok; - List -> emqx_trace:log(List, Msg, Meta#{trace_tag => Tag}) + %% We can't bind filter list to a variablebecause we pollute the calling scope with it. + %% We also don't want to wrap the macro body in a fun + %% beacause this adds overhead to the happy path. + %% So evaluate `persistent_term:get` twice. + _ -> emqx_trace:log(persistent_term:get(?TRACE_FILTER, []), Msg, (Meta)#{trace_tag => Tag}) end, ?SLOG( - debug, - (emqx_trace_formatter:format_meta(Meta))#{msg => Msg, tag => Tag}, + Level, + (emqx_trace_formatter:format_meta_map(Meta))#{msg => Msg, tag => Tag}, #{is_trace => false} ) end). diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index e7e89465d..964a97dfb 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -101,6 +101,14 @@ -define(CHAINS_TAB, emqx_authn_chains). +-define(TRACE_RESULT(Label, Result, Reason), begin + ?TRACE_AUTHN(Label, #{ + result => (Result), + reason => (Reason) + }), + Result +end). + -type chain_name() :: atom(). -type authenticator_id() :: binary(). -type position() :: front | rear | {before, authenticator_id()} | {'after', authenticator_id()}. @@ -216,14 +224,14 @@ when authenticate(#{enable_authn := false}, _AuthResult) -> inc_authenticate_metric('authentication.success.anonymous'), - ignore; + ?TRACE_RESULT("authentication_result", ignore, enable_authn_false); authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) -> case get_authenticators(Listener, global_chain(Protocol)) of {ok, ChainName, Authenticators} -> case get_enabled(Authenticators) of [] -> inc_authenticate_metric('authentication.success.anonymous'), - ignore; + ?TRACE_RESULT("authentication_result", ignore, empty_chain); NAuthenticators -> Result = do_authenticate(ChainName, NAuthenticators, Credential), @@ -235,11 +243,11 @@ authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthRe _ -> ok end, - Result + ?TRACE_RESULT("authentication_result", Result, chain_result) end; none -> inc_authenticate_metric('authentication.success.anonymous'), - ignore + ?TRACE_RESULT("authentication_result", ignore, no_chain) end. get_authenticators(Listener, Global) -> @@ -626,11 +634,11 @@ handle_create_authenticator(Chain, Config, Providers) -> do_authenticate(_ChainName, [], _) -> {stop, {error, not_authorized}}; do_authenticate( - ChainName, [#authenticator{id = ID, provider = Provider, state = State} | More], Credential + ChainName, [#authenticator{id = ID} = Authenticator | More], Credential ) -> MetricsID = metrics_id(ChainName, ID), emqx_metrics_worker:inc(authn_metrics, MetricsID, total), - try Provider:authenticate(Credential, State) of + try authenticate_with_provider(Authenticator, Credential) of ignore -> ok = emqx_metrics_worker:inc(authn_metrics, MetricsID, nomatch), do_authenticate(ChainName, More, Credential); @@ -651,8 +659,7 @@ do_authenticate( {stop, Result} catch Class:Reason:Stacktrace -> - ?SLOG(warning, #{ - msg => "unexpected_error_in_authentication", + ?TRACE_AUTHN(warning, "authenticator_error", #{ exception => Class, reason => Reason, stacktrace => Stacktrace, @@ -662,6 +669,14 @@ do_authenticate( do_authenticate(ChainName, More, Credential) end. +authenticate_with_provider(#authenticator{id = ID, provider = Provider, state = State}, Credential) -> + AuthnResult = Provider:authenticate(Credential, State), + ?TRACE_AUTHN("authenticator_result", #{ + authenticator => ID, + result => AuthnResult + }), + AuthnResult. + reply(Reply, State) -> {reply, Reply, State}. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 162ed4770..e6ee4260f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -92,15 +92,16 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -log(List, Msg, Meta0) -> - Meta = - case logger:get_process_metadata() of - undefined -> Meta0; - ProcMeta -> maps:merge(ProcMeta, Meta0) - end, - Log = #{level => debug, meta => Meta, msg => Msg}, +log(List, Msg, Meta) -> + Log = #{level => debug, meta => enrich_meta(Meta), msg => Msg}, log_filter(List, Log). +enrich_meta(Meta) -> + case logger:get_process_metadata() of + undefined -> Meta; + ProcMeta -> maps:merge(ProcMeta, Meta) + end. + log_filter([], _Log) -> ok; log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 33c718b72..de909ffda 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -16,7 +16,7 @@ -module(emqx_trace_formatter). -export([format/2]). --export([format_meta/1]). +-export([format_meta_map/1]). %%%----------------------------------------------------------------- %%% API @@ -31,32 +31,39 @@ format( ClientId = to_iolist(maps:get(clientid, Meta, "")), Peername = maps:get(peername, Meta, ""), MetaBin = format_meta(Meta, PEncode), - [Time, " [", Tag, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"]; + [Time, " [", Tag, "] ", ClientId, "@", Peername, " msg: ", Msg, ", ", MetaBin, "\n"]; format(Event, Config) -> emqx_logger_textfmt:format(Event, Config). -format_meta(Meta) -> +format_meta_map(Meta) -> Encode = emqx_trace_handler:payload_encode(), - do_format_meta(Meta, Encode). + format_meta_map(Meta, Encode). -format_meta(Meta0, Encode) -> - Meta1 = #{packet := Packet0, payload := Payload0} = do_format_meta(Meta0, Encode), - Packet = enrich(", packet: ", Packet0), - Payload = enrich(", payload: ", Payload0), - Meta2 = maps:without([msg, clientid, peername, packet, payload, trace_tag], Meta1), - case Meta2 =:= #{} of - true -> [Packet, Payload]; - false -> [Packet, ", ", map_to_iolist(Meta2), Payload] +format_meta_map(Meta, Encode) -> + format_meta_map(Meta, Encode, [{packet, fun format_packet/2}, {payload, fun format_payload/2}]). + +format_meta_map(Meta, _Encode, []) -> + Meta; +format_meta_map(Meta, Encode, [{Name, FormatFun} | Rest]) -> + case Meta of + #{Name := Value} -> + NewMeta = Meta#{Name => FormatFun(Value, Encode)}, + format_meta_map(NewMeta, Encode, Rest); + #{} -> + format_meta_map(Meta, Encode, Rest) end. -enrich(_, "") -> ""; -enrich(Key, IoData) -> [Key, IoData]. +format_meta(Meta0, Encode) -> + Meta1 = maps:without([msg, clientid, peername, trace_tag], Meta0), + Meta2 = format_meta_map(Meta1, Encode), + kvs_to_iolist(lists:sort(fun compare_meta_kvs/2, maps:to_list(Meta2))). -do_format_meta(Meta, Encode) -> - Meta#{ - packet => format_packet(maps:get(packet, Meta, undefined), Encode), - payload => format_payload(maps:get(payload, Meta, undefined), Encode) - }. +%% packet always goes first; payload always goes last +compare_meta_kvs(KV1, KV2) -> weight(KV1) =< weight(KV2). + +weight({packet, _}) -> {0, packet}; +weight({payload, _}) -> {2, payload}; +weight({K, _}) -> {1, K}. format_packet(undefined, _) -> ""; format_packet(Packet, Encode) -> emqx_packet:format(Packet, Encode). @@ -69,14 +76,14 @@ format_payload(_, hidden) -> "******". to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]); -to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"]; +to_iolist(SubMap) when is_map(SubMap) -> ["[", kvs_to_iolist(maps:to_list(SubMap)), "]"]; to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char). -map_to_iolist(Map) -> +kvs_to_iolist(KVs) -> lists:join( - ",", + ", ", lists:map( fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end, - maps:to_list(Map) + KVs ) ). diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index f07904b6b..d59eea1af 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -36,6 +36,6 @@ -type authenticator_id() :: binary(). --endif. - -define(RESOURCE_GROUP, <<"emqx_authn">>). + +-endif. diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 8d3d45b1b..17ad8a28b 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -33,7 +33,8 @@ bin/1, ensure_apps_started/1, cleanup_resources/0, - make_resource_id/1 + make_resource_id/1, + without_password/1 ]). -define(AUTHN_PLACEHOLDERS, [ @@ -199,10 +200,23 @@ make_resource_id(Name) -> NameBin = bin(Name), emqx_resource:generate_id(NameBin). +without_password(Credential) -> + without_password(Credential, [password, <<"password">>]). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +without_password(Credential, []) -> + Credential; +without_password(Credential, [Name | Rest]) -> + case maps:is_key(Name, Credential) of + true -> + without_password(Credential#{Name => <<"[password]">>}, Rest); + false -> + without_password(Credential, Rest) + end. + handle_var({var, Name}, undefined) -> error({cannot_get_variable, Name}); handle_var({var, <<"peerhost">>}, PeerHost) -> diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 9688ef65a..bc26140a6 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -331,7 +331,10 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S {continue, ServerFirstMessage, Cache}; ignore -> ignore; - {error, _Reason} -> + {error, Reason} -> + ?TRACE_AUTHN_PROVIDER("check_client_first_message_error", #{ + reason => Reason + }), {error, not_authorized} end. @@ -344,7 +347,10 @@ check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algori of {ok, ServerFinalMessage} -> {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; - {error, _Reason} -> + {error, Reason} -> + ?TRACE_AUTHN_PROVIDER("check_client_final_message_error", #{ + reason => Reason + }), {error, not_authorized} end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 8dccb9ee3..db3eb3c2f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -188,23 +188,22 @@ authenticate( } = State ) -> Request = generate_request(Credential, State), - case emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}) of + Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}), + ?TRACE_AUTHN_PROVIDER("http_response", #{ + request => request_for_log(Credential, State), + response => response_for_log(Response), + resource => ResourceId + }), + case Response of {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; {ok, 200, Headers, Body} -> handle_response(Headers, Body); {ok, _StatusCode, _Headers} = Response -> - log_response(ResourceId, Response), ignore; {ok, _StatusCode, _Headers, _Body} = Response -> - log_response(ResourceId, Response), ignore; - {error, Reason} -> - ?SLOG(error, #{ - msg => "http_server_query_failed", - resource => ResourceId, - reason => Reason - }), + {error, _Reason} -> ignore end. @@ -296,7 +295,8 @@ parse_config( cow_qs:parse_qs(to_bin(Query)) ), body_template => emqx_authn_utils:parse_deep(maps:get(body, Config, #{})), - request_timeout => RequestTimeout + request_timeout => RequestTimeout, + url => RawUrl }, {Config#{base_url => BaseUrl, pool_type => random}, State}. @@ -379,11 +379,6 @@ parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> parse_body(ContentType, _) -> {error, {unsupported_content_type, ContentType}}. -may_append_body(Output, {ok, _, _, Body}) -> - Output#{body => Body}; -may_append_body(Output, {ok, _, _}) -> - Output. - uri_encode(T) -> emqx_http_lib:uri_encode(to_list(T)). @@ -391,26 +386,33 @@ encode_path(Path) -> Parts = string:split(Path, "/", all), lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). -log_response(ResourceId, Other) -> - Output = may_append_body(#{resource => ResourceId}, Other), - case erlang:element(2, Other) of - Code5xx when Code5xx >= 500 andalso Code5xx < 600 -> - ?SLOG(error, Output#{ - msg => "http_server_error", - code => Code5xx - }); - Code4xx when Code4xx >= 400 andalso Code4xx < 500 -> - ?SLOG(warning, Output#{ - msg => "refused_by_http_server", - code => Code4xx - }); - OtherCode -> - ?SLOG(error, Output#{ - msg => "undesired_response_code", - code => OtherCode - }) +request_for_log(Credential, #{url := Url} = State) -> + SafeCredential = emqx_authn_utils:without_password(Credential), + case generate_request(SafeCredential, State) of + {PathQuery, Headers} -> + #{ + method => post, + base_url => Url, + path_query => PathQuery, + headers => Headers + }; + {PathQuery, Headers, Body} -> + #{ + method => post, + base_url => Url, + path_query => PathQuery, + headers => Headers, + mody => Body + } end. +response_for_log({ok, StatusCode, Headers}) -> + #{status => StatusCode, headers => Headers}; +response_for_log({ok, StatusCode, Headers, Body}) -> + #{status => StatusCode, headers => Headers, body => Body}; +response_for_log({error, Error}) -> + #{error => Error}. + to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 0017754a4..c6f2069ae 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -227,8 +227,7 @@ authenticate( ) -> case emqx_resource:query(ResourceId, get_jwks) of {error, Reason} -> - ?SLOG(error, #{ - msg => "get_jwks_failed", + ?TRACE_AUTHN_PROVIDER(error, "get_jwks_failed", #{ resource => ResourceId, reason => Reason }), @@ -350,10 +349,17 @@ verify(undefined, _, _, _) -> ignore; verify(JWT, JWKs, VerifyClaims, AclClaimName) -> case do_verify(JWT, JWKs, VerifyClaims) of - {ok, Extra} -> {ok, acl(Extra, AclClaimName)}; - {error, {missing_claim, _}} -> {error, bad_username_or_password}; - {error, invalid_signature} -> ignore; - {error, {claims, _}} -> {error, bad_username_or_password} + {ok, Extra} -> + {ok, acl(Extra, AclClaimName)}; + {error, {missing_claim, Claim}} -> + ?TRACE_AUTHN_PROVIDER("missing_jwt_claim", #{jwt => JWT, claim => Claim}), + {error, bad_username_or_password}; + {error, invalid_signature} -> + ?TRACE_AUTHN_PROVIDER("invalid_jwt_signature", #{jwks => JWKs, jwt => JWT}), + ignore; + {error, {claims, Claims}} -> + ?TRACE_AUTHN_PROVIDER("invalid_jwt_claims", #{jwt => JWT, claims => Claims}), + {error, bad_username_or_password} end. acl(Claims, AclClaimName) -> @@ -371,11 +377,11 @@ acl(Claims, AclClaimName) -> end, maps:merge(emqx_authn_utils:is_superuser(Claims), Acl). -do_verify(_JWS, [], _VerifyClaims) -> +do_verify(_JWT, [], _VerifyClaims) -> {error, invalid_signature}; -do_verify(JWS, [JWK | More], VerifyClaims) -> - try jose_jws:verify(JWK, JWS) of - {true, Payload, _JWS} -> +do_verify(JWT, [JWK | More], VerifyClaims) -> + try jose_jws:verify(JWK, JWT) of + {true, Payload, _JWT} -> Claims0 = emqx_json:decode(Payload, [return_maps]), Claims = try_convert_to_int(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]), case verify_claims(Claims, VerifyClaims) of @@ -385,11 +391,11 @@ do_verify(JWS, [JWK | More], VerifyClaims) -> {error, Reason} end; {false, _, _} -> - do_verify(JWS, More, VerifyClaims) + do_verify(JWT, More, VerifyClaims) catch - _:_Reason -> - ?TRACE("JWT", "authn_jwt_invalid_signature", #{jwk => JWK, jws => JWS}), - {error, invalid_signature} + _:Reason -> + ?TRACE_AUTHN_PROVIDER("jwt_verify_error", #{jwk => JWK, jwt => JWT, reason => Reason}), + do_verify(JWT, More, VerifyClaims) end. verify_claims(Claims, VerifyClaims0) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 81dc89c54..c3380b91f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -17,6 +17,7 @@ -module(emqx_authn_mnesia). -include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -158,6 +159,7 @@ authenticate( UserID = get_user_identity(Credential, Type), case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> + ?TRACE_AUTHN_PROVIDER("user_not_found"), ignore; [#user_info{password_hash = PasswordHash, salt = Salt, is_superuser = IsSuperuser}] -> case @@ -165,8 +167,10 @@ authenticate( Algorithm, Salt, PasswordHash, Password ) of - true -> {ok, #{is_superuser => IsSuperuser}}; - false -> {error, bad_username_or_password} + true -> + {ok, #{is_superuser => IsSuperuser}}; + false -> + {error, bad_username_or_password} end end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index a46d62c8c..f7249ae57 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -167,8 +167,7 @@ authenticate( undefined -> ignore; {error, Reason} -> - ?SLOG(error, #{ - msg => "mongodb_query_failed", + ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{ resource => ResourceId, collection => Collection, filter => Filter, @@ -180,11 +179,11 @@ authenticate( ok -> {ok, is_superuser(Doc, State)}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> - ?SLOG(error, #{ - msg => "cannot_find_password_hash_field", + ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{ resource => ResourceId, collection => Collection, filter => Filter, + document => Doc, password_hash_field => PasswordHashField }), ignore; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index d183748ff..e95302ad4 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -130,8 +130,7 @@ authenticate( {error, Reason} end; {error, Reason} -> - ?SLOG(error, #{ - msg => "mysql_query_failed", + ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{ resource => ResourceId, tmpl_token => TmplToken, params => Params, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 99f4fa43c..2962308ab 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -133,8 +133,7 @@ authenticate( {error, Reason} end; {error, Reason} -> - ?SLOG(error, #{ - msg => "postgresql_query_failed", + ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{ resource => ResourceId, params => Params, reason => Reason diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index d2003f26a..71cd292e6 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -128,13 +128,14 @@ authenticate(#{auth_method := _}, _) -> authenticate( #{password := Password} = Credential, #{ - cmd := {Command, KeyTemplate, Fields}, + cmd := {CommandName, KeyTemplate, Fields}, resource_id := ResourceId, password_hash_algorithm := Algorithm } ) -> NKey = emqx_authn_utils:render_str(KeyTemplate, Credential), - case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of + Command = [CommandName, NKey | Fields], + case emqx_resource:query(ResourceId, {cmd, Command}) of {ok, []} -> ignore; {ok, Values} -> @@ -150,8 +151,7 @@ authenticate( {error, Reason} end; {error, Reason} -> - ?SLOG(error, #{ - msg => "redis_query_failed", + ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{ resource => ResourceId, cmd => Command, keys => NKey, From 2c71d27a89ed0faf3a5e31c628f5d9822df69d16 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 25 Jul 2022 17:42:19 +0800 Subject: [PATCH 071/105] feat(listeners): change the `GET /listeners` api fields --- apps/emqx_dashboard/etc/emqx_dashboard.conf | 6 +- .../src/emqx_mgmt_api_listeners.erl | 213 ++++++++++++------ 2 files changed, 146 insertions(+), 73 deletions(-) diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 2d54431fa..856779500 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -1,7 +1,7 @@ dashboard { listeners.http { - bind: 18083 + bind = 18083 } - default_username: "admin" - default_password: "public" + default_username = "admin" + default_password = "public" } diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 842ac4bfb..c1733a9a6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -218,11 +218,13 @@ fields(listener_type_status) -> fields(listener_id_status) -> fields(listener_id) ++ [ + {type, ?HOCON(?ENUM(listeners_type()), #{desc => "Listener type", required => true})}, + {name, ?HOCON(string(), #{desc => "Listener name", required => true})}, {enable, ?HOCON(boolean(), #{desc => "Listener enable", required => true})}, {number, ?HOCON(typerefl:pos_integer(), #{desc => "ListenerId counter"})}, {bind, ?HOCON( - hoconsc:union([emqx_schema:ip_port(), integer()]), + emqx_schema:ip_port(), #{desc => "Listener bind addr", required => true} )}, {acceptors, ?HOCON(typerefl:pos_integer(), #{desc => "ListenerId acceptors"})}, @@ -231,12 +233,24 @@ fields(listener_id_status) -> ]; fields(status) -> [ + {running, + ?HOCON( + hoconsc:union([inconsistent, boolean()]), + #{desc => "Listener running status", required => true} + )}, {max_connections, ?HOCON(hoconsc:union([infinity, integer()]), #{desc => "Max connections"})}, {current_connections, ?HOCON(non_neg_integer(), #{desc => "Current connections"})} ]; fields(node_status) -> - fields(node) ++ fields(status); + [ + {"node", + ?HOCON(atom(), #{ + desc => "Node name", + example => "emqx@127.0.0.1" + })}, + {status, ?HOCON(?R_REF(status))} + ]; fields(Type) -> Listeners = listeners_info(#{bind => true}) ++ listeners_info(#{bind => false}), [Schema] = [S || #{ref := ?R_REF(_, T), schema := S} <- Listeners, T =:= Type], @@ -311,7 +325,7 @@ listener_type_status(get, _Request) -> Listeners = maps:to_list(listener_status_by_type(list_listeners(), #{})), List = lists:map( fun({Type, L}) -> - L1 = maps:without([bind, acceptors], L), + L1 = maps:without([bind, acceptors, name], L), L1#{type => Type} end, Listeners @@ -453,7 +467,7 @@ listener_status_by_id(NodeL) -> fun({Id, L}) -> L1 = maps:remove(ids, L), #{node_status := Nodes} = L1, - L1#{number => maps:size(Nodes), id => Id} + L1#{number => length(Nodes), id => Id} end, Listeners ). @@ -510,67 +524,75 @@ wrap_rpc(Res) -> format_status(Key, Node, Listener, Acc) -> #{ <<"id">> := Id, + <<"type">> := Type, + <<"enabled">> := Enabled, <<"running">> := Running, <<"max_connections">> := MaxConnections, <<"current_connections">> := CurrentConnections, <<"acceptors">> := Acceptors, <<"bind">> := Bind } = Listener, + {ok, #{name := Name}} = emqx_listeners:parse_listener_id(Id), GroupKey = maps:get(Key, Listener), case maps:find(GroupKey, Acc) of error -> Acc#{ GroupKey => #{ - enable => Running, + name => Name, + type => Type, + enable => Enabled, ids => [Id], acceptors => Acceptors, - bind => Bind, + bind => format_raw_bind(Bind), status => #{ + running => Running, max_connections => MaxConnections, current_connections => CurrentConnections }, - node_status => #{ - Node => #{ - max_connections => MaxConnections, - current_connections => CurrentConnections + node_status => [ + #{ + node => Node, + status => #{ + running => Running, + max_connections => MaxConnections, + current_connections => CurrentConnections + } } - } + ] } }; {ok, GroupValue} -> #{ ids := Ids, status := #{ + running := Running0, max_connections := MaxConnections0, current_connections := CurrentConnections0 }, node_status := NodeStatus0 } = GroupValue, - NodeStatus = - case maps:find(Node, NodeStatus0) of - error -> - NodeStatus0#{ - Node => #{ - max_connections => MaxConnections, - current_connections => CurrentConnections - } - }; - {ok, #{ - max_connections := PrevMax, - current_connections := PrevCurr - }} -> - NodeStatus0#{ - Node => #{ - max_connections => max_conn(MaxConnections, PrevMax), - current_connections => CurrentConnections + PrevCurr - } - } + NodeStatus = [ + #{ + node => Node, + status => #{ + running => Running, + max_connections => MaxConnections, + current_connections => CurrentConnections + } + } + | NodeStatus0 + ], + NRunning = + case Running == Running0 of + true -> Running0; + _ -> inconsistent end, Acc#{ GroupKey => GroupValue#{ ids => lists:usort([Id | Ids]), status => #{ + running => NRunning, max_connections => max_conn(MaxConnections0, MaxConnections), current_connections => CurrentConnections0 + CurrentConnections }, @@ -583,6 +605,12 @@ max_conn(_Int1, <<"infinity">>) -> <<"infinity">>; max_conn(<<"infinity">>, _Int) -> <<"infinity">>; max_conn(Int1, Int2) -> Int1 + Int2. +%% @doc returning a uniform format (ip_port string) is more +%% helpful to users +format_raw_bind(Bind) when is_integer(Bind) -> + <<"0.0.0.0:", (integer_to_binary(Bind))/binary>>; +format_raw_bind(Bind) when is_binary(Bind) -> Bind. + update(Path, Conf) -> wrap(emqx_conf:update(Path, {update, Conf}, ?OPTS(cluster))). @@ -605,17 +633,27 @@ listener_type_status_example() -> #{ enable => false, ids => ["tcp:demo"], - node_status => #{ - 'emqx@127.0.0.1' => #{ - current_connections => 11, - max_connections => 1024000 - }, - 'emqx@127.0.0.2' => #{ - current_connections => 10, - max_connections => 1024000 - } - }, + node_status => + [ + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 11, + max_connections => 1024000 + } + }, + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 10, + max_connections => 1024000 + } + } + ], status => #{ + running => true, current_connections => 21, max_connections => 2048000 }, @@ -624,17 +662,28 @@ listener_type_status_example() -> #{ enable => false, ids => ["ssl:default"], - node_status => #{ - 'emqx@127.0.0.1' => #{ - current_connections => 31, - max_connections => infinity - }, - 'emqx@127.0.0.2' => #{ - current_connections => 40, - max_connections => infinity - } - }, + node_status => + [ + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 31, + max_connections => infinity + } + }, + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 40, + max_connections => infinity + } + } + ], + status => #{ + running => true, current_connections => 71, max_connections => infinity }, @@ -649,18 +698,30 @@ listener_id_status_example() -> bind => <<"0.0.0.0:1884">>, enable => true, id => <<"tcp:demo">>, - node_status => #{ - 'emqx@127.0.0.1' => #{ - current_connections => 100, - max_connections => 1024000 - }, - 'emqx@127.0.0.2' => #{ - current_connections => 101, - max_connections => 1024000 - } - }, + type => <<"tcp">>, + name => <<"demo">>, + node_status => + [ + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 100, + max_connections => 1024000 + } + }, + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 101, + max_connections => 1024000 + } + } + ], number => 2, status => #{ + running => true, current_connections => 201, max_connections => 2048000 } @@ -670,18 +731,30 @@ listener_id_status_example() -> bind => <<"0.0.0.0:1883">>, enable => true, id => <<"tcp:default">>, - node_status => #{ - 'emqx@127.0.0.1' => #{ - current_connections => 300, - max_connections => infinity - }, - 'emqx@127.0.0.2' => #{ - current_connections => 201, - max_connections => infinity - } - }, + type => <<"tcp">>, + name => <<"default">>, + node_status => + [ + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 200, + max_connections => infinity + } + }, + #{ + node => 'emqx@127.0.0.1', + status => #{ + running => true, + current_connections => 301, + max_connections => infinity + } + } + ], number => 2, status => #{ + running => true, current_connections => 501, max_connections => infinity } From bad82b29ce4941afaf77588dfbfd3880bff6d39b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 26 Jul 2022 11:03:13 +0800 Subject: [PATCH 072/105] feat(gw): change the listeners api fields --- .../i18n/emqx_gateway_api_listeners_i18n.conf | 7 ++ .../src/emqx_gateway_api_listeners.erl | 83 ++++++++++++++----- apps/emqx_gateway/src/emqx_gateway_conf.erl | 23 ++--- apps/emqx_gateway/src/emqx_gateway_utils.erl | 8 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 11 +-- apps/emqx_management/src/emqx_mgmt_util.erl | 24 +++++- 6 files changed, 100 insertions(+), 56 deletions(-) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf index e4f7413d0..9c5de67c3 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf @@ -112,6 +112,13 @@ emqx_gateway_api_listeners { } } + listener_status { + desc { + en: """listener status """ + zh: """监听器状态""" + } + } + listener_node_status { desc { en: """listener status of each node in the cluster""" diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 0f0ec8606..473305610 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -81,7 +81,7 @@ paths() -> listeners(get, #{bindings := #{name := Name0}}) -> with_gateway(Name0, fun(GwName, _) -> - Result = get_cluster_listeners_info(GwName), + Result = lists:map(fun bind2str/1, get_cluster_listeners_info(GwName)), {200, Result} end); listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> @@ -119,7 +119,7 @@ listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> with_gateway(Name0, fun(_GwName, _) -> case emqx_gateway_conf:listener(ListenerId) of {ok, Listener} -> - {200, Listener}; + {200, bind2str(Listener)}; {error, not_found} -> return_http_error(404, "Listener not found"); {error, Reason} -> @@ -266,11 +266,14 @@ get_cluster_listeners_info(GwName) -> ClusterStatus ), - {MaxCons, CurrCons} = emqx_gateway_http:sum_cluster_connections(NodeStatus), + {MaxCons, CurrCons, Running} = aggregate_listener_status(NodeStatus), Listener#{ - max_connections => MaxCons, - current_connections => CurrCons, + status => #{ + running => Running, + max_connections => MaxCons, + current_connections => CurrCons + }, node_status => NodeStatus } end, @@ -292,20 +295,23 @@ do_listeners_cluster_status(Listeners) -> fun({Id, ListenOn}, Acc) -> BinId = erlang:atom_to_binary(Id), {ok, #{<<"max_connections">> := Max}} = emqx_gateway_conf:listener(BinId), - Curr = + {Running, Curr} = try esockd:get_current_connections({Id, ListenOn}) of - Int -> Int + Int -> {true, Int} catch %% not started error:not_found -> - 0 + {false, 0} end, Acc#{ Id => #{ node => Node, - current_connections => Curr, - %% XXX: Since it is taken from raw-conf, it is possible a string - max_connections => int(Max) + status => #{ + running => Running, + current_connections => Curr, + %% XXX: Since it is taken from raw-conf, it is possible a string + max_connections => int(Max) + } } } end, @@ -317,6 +323,31 @@ int(B) when is_binary(B) -> binary_to_integer(B); int(I) when is_integer(I) -> I. +aggregate_listener_status(NodeStatus) -> + aggregate_listener_status(NodeStatus, 0, 0, undefined). + +aggregate_listener_status( + [ + #{status := #{running := Running, max_connections := Max, current_connections := Current}} + | T + ], + MaxAcc, + CurrAcc, + RunningAcc +) -> + NRunning = aggregate_running(Running, RunningAcc), + aggregate_listener_status(T, MaxAcc + Max, Current + CurrAcc, NRunning); +aggregate_listener_status([], MaxAcc, CurrAcc, RunningAcc) -> + {MaxAcc, CurrAcc, RunningAcc}. + +aggregate_running(R, R) -> R; +aggregate_running(R, undefined) -> R; +aggregate_running(_, _) -> inconsistent. + +bind2str(Listener = #{bind := Bind}) -> + Listener#{bind := iolist_to_binary(emqx_gateway_utils:format_listenon(Bind))}; +bind2str(Listener = #{<<"bind">> := Bind}) -> + Listener#{<<"bind">> := iolist_to_binary(emqx_gateway_utils:format_listenon(Bind))}. %%-------------------------------------------------------------------- %% Swagger defines @@ -590,22 +621,25 @@ params_paging_in_qs() -> roots() -> [listener]. -fields(listener_node_status) -> +fields(listener_status) -> [ - {current_connections, mk(non_neg_integer(), #{desc => ?DESC(current_connections)})}, + {status, + mk(ref(emqx_mgmt_api_listeners, status), #{ + desc => ?DESC(listener_status) + })}, {node_status, mk(hoconsc:array(ref(emqx_mgmt_api_listeners, node_status)), #{ desc => ?DESC(listener_node_status) })} ]; fields(tcp_listener) -> - emqx_gateway_api:fields(tcp_listener) ++ fields(listener_node_status); + emqx_gateway_api:fields(tcp_listener) ++ fields(listener_status); fields(ssl_listener) -> - emqx_gateway_api:fields(ssl_listener) ++ fields(listener_node_status); + emqx_gateway_api:fields(ssl_listener) ++ fields(listener_status); fields(udp_listener) -> - emqx_gateway_api:fields(udp_listener) ++ fields(listener_node_status); + emqx_gateway_api:fields(udp_listener) ++ fields(listener_status); fields(dtls_listener) -> - emqx_gateway_api:fields(dtls_listener) ++ fields(listener_node_status); + emqx_gateway_api:fields(dtls_listener) ++ fields(listener_status); fields(_) -> []. @@ -623,12 +657,19 @@ listener_node_status_schema() -> examples_listener_list() -> Convert = fun(Cfg) -> Cfg#{ - current_connections => 0, + status => #{ + running => true, + max_connections => 1024000, + current_connections => 10 + }, node_status => [ #{ - node => <<"127.0.0.1">>, - current_connections => 0, - max_connections => 1024000 + node => <<"emqx@127.0.0.1">>, + status => #{ + running => true, + current_connections => 10, + max_connections => 1024000 + } } ] } diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 50fc069fc..5fe858fa9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -181,24 +181,11 @@ do_convert_listener(GwName, LType, Conf) -> do_convert_listener2(GwName, LType, LName, LConf) -> ListenerId = emqx_gateway_utils:listener_id(GwName, LType, LName), - Running = emqx_gateway_utils:is_running(ListenerId, LConf), - bind2str( - LConf#{ - id => ListenerId, - type => LType, - name => LName, - running => Running - } - ). - -bind2str(LConf = #{bind := Bind}) when is_integer(Bind) -> - maps:put(bind, integer_to_binary(Bind), LConf); -bind2str(LConf = #{<<"bind">> := Bind}) when is_integer(Bind) -> - maps:put(<<"bind">>, integer_to_binary(Bind), LConf); -bind2str(LConf = #{bind := Bind}) when is_binary(Bind) -> - LConf; -bind2str(LConf = #{<<"bind">> := Bind}) when is_binary(Bind) -> - LConf. + LConf#{ + id => ListenerId, + type => LType, + name => LName + }. get_bind(#{bind := Bind}) -> emqx_gateway_utils:parse_listenon(Bind); diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 6a491de3d..f8c0549bd 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -287,12 +287,8 @@ apply(F, A2) when -> erlang:apply(F, A2). -format_listenon(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format_listenon({Addr, Port}) when is_list(Addr) -> - io_lib:format("~ts:~w", [Addr, Port]); -format_listenon({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). +format_listenon(Term) -> + emqx_mgmt_util:format_listen_on(Term). parse_listenon(Port) when is_integer(Port) -> Port; diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index c846c9d58..e373f11cd 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -582,7 +582,7 @@ listeners([]) -> end, Info = [ - {listen_on, {string, format_listen_on(Bind)}}, + {listen_on, {string, emqx_mgmt_util:format_listen_on(Port)}}, {acceptors, Acceptors}, {proxy_protocol, ProxyProtocol}, {running, Running} @@ -802,15 +802,6 @@ indent_print({Key, {string, Val}}) -> indent_print({Key, Val}) -> emqx_ctl:print(" ~-16s: ~w~n", [Key, Val]). -format_listen_on(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format_listen_on({Addr, Port}) when is_list(Addr) -> - io_lib:format("~ts:~w", [Addr, Port]); -format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 4 -> - io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]); -format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 8 -> - io_lib:format("[~ts]:~w", [inet:ntoa(Addr), Port]). - name(Filter) -> iolist_to_binary(["CLI-", Filter]). diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 58fd6e952..2802046fb 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -43,7 +43,10 @@ batch_schema/1 ]). --export([urldecode/1]). +-export([ + urldecode/1, + format_listen_on/1 +]). -define(KB, 1024). -define(MB, (1024 * 1024)). @@ -86,6 +89,25 @@ merge_maps(Default, New) -> urldecode(S) -> emqx_http_lib:uri_decode(S). +-spec format_listen_on( + integer() | {tuple(), integer()} | string() | binary() +) -> io_lib:chars(). +format_listen_on(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format_listen_on({Addr, Port}) when is_list(Addr) -> + io_lib:format("~ts:~w", [Addr, Port]); +format_listen_on({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]); +format_listen_on(Str) when is_list(Str) -> + case emqx_schema:to_ip_port(Str) of + {ok, {Ip, Port}} -> + format_listen_on({Ip, Port}); + {error, _} -> + format_listen_on(list_to_integer(Str)) + end; +format_listen_on(Bin) when is_binary(Bin) -> + format_listen_on(binary_to_list(Bin)). + %%%============================================================================================== %% schema util schema(Ref) when is_atom(Ref) -> From 53fda85c93b6070f3a3643f2db2c522da8e90098 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 26 Jul 2022 11:10:40 +0800 Subject: [PATCH 073/105] fix: conflicts resolving legacy --- apps/emqx_management/src/emqx_mgmt_cli.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index e373f11cd..be3de5d90 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -582,7 +582,7 @@ listeners([]) -> end, Info = [ - {listen_on, {string, emqx_mgmt_util:format_listen_on(Port)}}, + {listen_on, {string, emqx_mgmt_util:format_listen_on(Bind)}}, {acceptors, Acceptors}, {proxy_protocol, ProxyProtocol}, {running, Running} From 38b5882dc8dcc0fc9cee1e5d868403be8085770f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 26 Jul 2022 11:40:52 +0800 Subject: [PATCH 074/105] test(gw): using ip_port in listener tests --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 90a060cd7..fb3207944 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -340,7 +340,7 @@ t_listeners_tcp(_) -> LisConf = #{ name => <<"def">>, type => <<"tcp">>, - bind => <<"61613">> + bind => <<"127.0.0.1:61613">> }, {201, _} = request(post, "/gateway/stomp/listeners", LisConf), {200, ConfResp} = request(get, "/gateway/stomp/listeners"), @@ -348,7 +348,7 @@ t_listeners_tcp(_) -> {200, ConfResp1} = request(get, "/gateway/stomp/listeners/stomp:tcp:def"), assert_confs(LisConf, ConfResp1), - LisConf2 = maps:merge(LisConf, #{bind => <<"61614">>}), + LisConf2 = maps:merge(LisConf, #{bind => <<"127.0.0.1:61614">>}), {200, _} = request( put, "/gateway/stomp/listeners/stomp:tcp:def", @@ -369,7 +369,7 @@ t_listeners_authn(_) -> #{ name => <<"def">>, type => <<"tcp">>, - bind => <<"61613">> + bind => <<"127.0.0.1:61613">> } ] }, @@ -405,7 +405,7 @@ t_listeners_authn_data_mgmt(_) -> #{ name => <<"def">>, type => <<"tcp">>, - bind => <<"61613">> + bind => <<"127.0.0.1:61613">> } ] }, From 889e25d64e331712b481a95bf99adbdd6bef4840 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 26 Jul 2022 12:00:20 +0800 Subject: [PATCH 075/105] chore: update changes --- CHANGES-5.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index ae1184f81..f16ef442d 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -31,6 +31,8 @@ * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) * Optimize performance of builtin database operations in processes with long message queue [#8439](https://github.com/emqx/emqx/pull/8439) * Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554) +* Standardize the '/listeners' and `/gateway//listeners` API fields. + It will introduce some incompatible updates, see [#8571](https://github.com/emqx/emqx/pull/8571) # 5.0.3 From bcc78950a93cc227b4288562b108915760ace19d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 26 Jul 2022 14:33:04 +0800 Subject: [PATCH 076/105] fix(limiter): fix schema error --- .../emqx_limiter/src/emqx_limiter_schema.erl | 99 +++++++++---------- apps/emqx/src/emqx_schema.erl | 6 +- .../src/emqx_retainer_schema.erl | 2 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 23d4d4e4c..89c927950 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -31,8 +31,7 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0, - bucket_fields/1 + infinity_value/0 ]). -define(KILOBYTE, 1024). @@ -92,7 +91,7 @@ fields(limiter) -> {Type, ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), - default => make_limiter_default(Type) + default => #{} })} || Type <- types() ] ++ @@ -179,17 +178,12 @@ fields(client_opts) -> } )} ]; -fields({client_fields, Types}) -> - [ - {Type, - ?HOCON(?R_REF(client_opts), #{ - desc => ?DESC(Type), - required => false - })} - || Type <- Types - ]; -fields({bucket_fields, Types}) -> - bucket_fields(Types). +fields(listener_fields) -> + bucket_fields([bytes_in, message_in, connection, message_routing], listener_client_fields); +fields(listener_client_fields) -> + client_fields([bytes_in, message_in, connection, message_routing]); +fields(Type) -> + bucket_field(Type). desc(limiter) -> "Settings for the rate limiter."; @@ -202,38 +196,6 @@ desc(client_opts) -> desc(_) -> undefined. -bucket_fields(Type) when is_atom(Type) -> - fields(bucket_opts) ++ - [ - {client, - ?HOCON( - ?R_REF(?MODULE, client_opts), - #{ - desc => ?DESC(client), - required => false - } - )} - ]; -bucket_fields(Types) -> - [ - {Type, - ?HOCON(?R_REF(?MODULE, bucket_opts), #{ - desc => ?DESC(?MODULE, Type), - required => false - })} - || Type <- Types - ] ++ - [ - {client, - ?HOCON( - ?R_REF(?MODULE, {client_fields, Types}), - #{ - desc => ?DESC(client), - required => false - } - )} - ]. - %% default period is 100ms default_period() -> 100. @@ -366,7 +328,44 @@ apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). -make_limiter_default(connection) -> - #{<<"rate">> => <<"1000/s">>}; -make_limiter_default(_) -> - #{}. +bucket_field(Type) when is_atom(Type) -> + fields(bucket_opts) ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, client_opts), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. +bucket_fields(Types, ClientRef) -> + [ + {Type, + ?HOCON(?R_REF(?MODULE, bucket_opts), #{ + desc => ?DESC(?MODULE, Type), + required => false + })} + || Type <- Types + ] ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, ClientRef), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. + +client_fields(Types) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + required => false + })} + || Type <- Types + ]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 81f5c922a..f7c18ae8e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1621,11 +1621,13 @@ base_listener(Bind) -> sc( ?R_REF( emqx_limiter_schema, - {bucket_fields, [bytes_in, message_in, connection, message_routing]} + listener_fields ), #{ desc => ?DESC(base_listener_limiter), - default => #{} + default => #{ + <<"connection">> => #{<<"rate">> => <<"1000/s">>, <<"capacity">> => 1000} + } } )}, {"enable_authn", diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 986eb4105..51dbf496b 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - ?R_REF(emqx_limiter_schema, {bucket_fields, internal}), + ?R_REF(emqx_limiter_schema, internal), batch_deliver_limiter, undefined )} From bc716884e09b31c75c5629fe0313ecc46910d605 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 26 Jul 2022 15:16:53 +0800 Subject: [PATCH 077/105] fix(limiter): fix spellcheck error --- apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 89c927950..bce87e2ba 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -193,6 +193,14 @@ desc(bucket_opts) -> "Settings for the bucket."; desc(client_opts) -> "Settings for the client in bucket level."; +desc(client_fields) -> + "Fields of the client level."; +desc(listener_fields) -> + "Fields of the listener."; +desc(listener_client_fields) -> + "Fields of the client level of the listener."; +desc(internal) -> + "Internal limiter."; desc(_) -> undefined. From 257e310931b085f1d96914a12f0c6e5017fb3554 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 26 Jul 2022 18:09:18 +0800 Subject: [PATCH 078/105] chore: ensure the `bind` output style e.g: - Configured as `1883`, printed as `:1883` - Configured as `0.0.0.0:1883`, printed as `:1883` - Configured as `127.0.0.1:1883`, printed as `127.0.0.1:1883` - Configured as `::1:1883`, printed as `[::1]:1883` - Configured as `[::1]:1883`, printed as `[::1]:1883` --- apps/emqx/src/emqx_listeners.erl | 45 ++++++++++++------- apps/emqx_dashboard/src/emqx_dashboard.erl | 4 +- .../src/emqx_gateway_api_listeners.erl | 4 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 8 +--- .../src/exproto/emqx_exproto_impl.erl | 2 +- .../src/emqx_mgmt_api_listeners.erl | 8 +--- apps/emqx_management/src/emqx_mgmt_cli.erl | 2 +- apps/emqx_management/src/emqx_mgmt_util.erl | 24 +--------- 8 files changed, 40 insertions(+), 57 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 3c508bacc..09b923d0c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -54,7 +54,7 @@ -export([pre_config_update/3, post_config_update/5]). --export([format_addr/1]). +-export([format_bind/1]). -define(CONF_KEY_PATH, [listeners, '?', '?']). -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]). @@ -201,14 +201,14 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) -> ?tp(listener_started, #{type => Type, bind => Bind}), console_print( "Listener ~ts on ~ts started.~n", - [listener_id(Type, ListenerName), format_addr(Bind)] + [listener_id(Type, ListenerName), format_bind(Bind)] ), ok; {error, {already_started, Pid}} -> {error, {already_started, Pid}}; {error, Reason} -> ListenerId = listener_id(Type, ListenerName), - BindStr = format_addr(Bind), + BindStr = format_bind(Bind), ?ELOG( "Failed to start listener ~ts on ~ts: ~0p.~n", [ListenerId, BindStr, Reason] @@ -261,19 +261,19 @@ stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> ok -> console_print( "Listener ~ts on ~ts stopped.~n", - [listener_id(Type, ListenerName), format_addr(Bind)] + [listener_id(Type, ListenerName), format_bind(Bind)] ), ok; {error, not_found} -> ?ELOG( "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_addr(Bind), already_stopped] + [listener_id(Type, ListenerName), format_bind(Bind), already_stopped] ), ok; {error, Reason} -> ?ELOG( "Failed to stop listener ~ts on ~ts: ~0p~n", - [listener_id(Type, ListenerName), format_addr(Bind), Reason] + [listener_id(Type, ListenerName), format_bind(Bind), Reason] ), {error, Reason} end. @@ -492,17 +492,32 @@ merge_default(Options) -> [{tcp_options, ?MQTT_SOCKOPTS} | Options] end. -format_addr(Port) when is_integer(Port) -> - io_lib:format("~w", [Port]); +-spec format_bind( + integer() | {tuple(), integer()} | string() | binary() +) -> io_lib:chars(). +format_bind(Port) when is_integer(Port) -> + io_lib:format(":~w", [Port]); %% Print only the port number when bound on all interfaces -format_addr({{0, 0, 0, 0}, Port}) -> - format_addr(Port); -format_addr({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) -> - format_addr(Port); -format_addr({Addr, Port}) when is_list(Addr) -> +format_bind({{0, 0, 0, 0}, Port}) -> + format_bind(Port); +format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) -> + format_bind(Port); +format_bind({Addr, Port}) when is_list(Addr) -> io_lib:format("~ts:~w", [Addr, Port]); -format_addr({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). +format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 -> + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]); +format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 8 -> + io_lib:format("[~ts]:~w", [inet:ntoa(Addr), Port]); +%% Support string, binary type for Port or IP:Port +format_bind(Str) when is_list(Str) -> + case emqx_schema:to_ip_port(Str) of + {ok, {Ip, Port}} -> + format_bind({Ip, Port}); + {error, _} -> + format_bind(list_to_integer(Str)) + end; +format_bind(Bin) when is_binary(Bin) -> + format_bind(binary_to_list(Bin)). listener_id(Type, ListenerName) -> list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 042c9d5d8..9bf981323 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -92,7 +92,7 @@ start_listeners(Listeners) -> case minirest:start(Name, RanchOptions, Minirest) of {ok, _} -> ?ULOG("Listener ~ts on ~ts started.~n", [ - Name, emqx_listeners:format_addr(Bind) + Name, emqx_listeners:format_bind(Bind) ]), Acc; {error, _Reason} -> @@ -114,7 +114,7 @@ stop_listeners(Listeners) -> case minirest:stop(Name) of ok -> ?ULOG("Stop listener ~ts on ~ts successfully.~n", [ - Name, emqx_listeners:format_addr(Port) + Name, emqx_listeners:format_bind(Port) ]); {error, not_found} -> ?SLOG(warning, #{msg => "stop_listener_failed", name => Name, port => Port}) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 473305610..79734bfc0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -345,9 +345,9 @@ aggregate_running(R, undefined) -> R; aggregate_running(_, _) -> inconsistent. bind2str(Listener = #{bind := Bind}) -> - Listener#{bind := iolist_to_binary(emqx_gateway_utils:format_listenon(Bind))}; + Listener#{bind := iolist_to_binary(emqx_listeners:format_bind(Bind))}; bind2str(Listener = #{<<"bind">> := Bind}) -> - Listener#{<<"bind">> := iolist_to_binary(emqx_gateway_utils:format_listenon(Bind))}. + Listener#{<<"bind">> := iolist_to_binary(emqx_listeners:format_bind(Bind))}. %%-------------------------------------------------------------------- %% Swagger defines diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index f8c0549bd..15359dea6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -37,7 +37,6 @@ -export([ apply/2, - format_listenon/1, parse_listenon/1, unix_ts_to_rfc3339/1, unix_ts_to_rfc3339/2, @@ -165,7 +164,7 @@ start_listener( {Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg ) -> - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + ListenOnStr = emqx_listeners:format_bind(ListenOn), ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = maps:merge(Cfg, ModCfg), @@ -243,7 +242,7 @@ stop_listeners(GwName, Listeners) -> -spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok. stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), - ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), + ListenOnStr = emqx_listeners:format_bind(ListenOn), case StopRet of ok -> console_print( @@ -287,9 +286,6 @@ apply(F, A2) when -> erlang:apply(F, A2). -format_listenon(Term) -> - emqx_mgmt_util:format_listen_on(Term). - parse_listenon(Port) when is_integer(Port) -> Port; parse_listenon(IpPort) when is_tuple(IpPort) -> diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 9965956f7..78e70def0 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -167,7 +167,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> )} ] end, - ListenOnStr = emqx_listeners:format_addr(ListenOn), + ListenOnStr = emqx_listeners:format_bind(ListenOn), case grpc:start_server(GwName, ListenOn, Services, SvrOptions) of {ok, _SvrPid} -> console_print( diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index c1733a9a6..24fe710c7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -543,7 +543,7 @@ format_status(Key, Node, Listener, Acc) -> enable => Enabled, ids => [Id], acceptors => Acceptors, - bind => format_raw_bind(Bind), + bind => iolist_to_binary(emqx_listeners:format_bind(Bind)), status => #{ running => Running, max_connections => MaxConnections, @@ -605,12 +605,6 @@ max_conn(_Int1, <<"infinity">>) -> <<"infinity">>; max_conn(<<"infinity">>, _Int) -> <<"infinity">>; max_conn(Int1, Int2) -> Int1 + Int2. -%% @doc returning a uniform format (ip_port string) is more -%% helpful to users -format_raw_bind(Bind) when is_integer(Bind) -> - <<"0.0.0.0:", (integer_to_binary(Bind))/binary>>; -format_raw_bind(Bind) when is_binary(Bind) -> Bind. - update(Path, Conf) -> wrap(emqx_conf:update(Path, {update, Conf}, ?OPTS(cluster))). diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index be3de5d90..60b2f3b15 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -582,7 +582,7 @@ listeners([]) -> end, Info = [ - {listen_on, {string, emqx_mgmt_util:format_listen_on(Bind)}}, + {listen_on, {string, emqx_listeners:format_bind(Bind)}}, {acceptors, Acceptors}, {proxy_protocol, ProxyProtocol}, {running, Running} diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 2802046fb..58fd6e952 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -43,10 +43,7 @@ batch_schema/1 ]). --export([ - urldecode/1, - format_listen_on/1 -]). +-export([urldecode/1]). -define(KB, 1024). -define(MB, (1024 * 1024)). @@ -89,25 +86,6 @@ merge_maps(Default, New) -> urldecode(S) -> emqx_http_lib:uri_decode(S). --spec format_listen_on( - integer() | {tuple(), integer()} | string() | binary() -) -> io_lib:chars(). -format_listen_on(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format_listen_on({Addr, Port}) when is_list(Addr) -> - io_lib:format("~ts:~w", [Addr, Port]); -format_listen_on({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]); -format_listen_on(Str) when is_list(Str) -> - case emqx_schema:to_ip_port(Str) of - {ok, {Ip, Port}} -> - format_listen_on({Ip, Port}); - {error, _} -> - format_listen_on(list_to_integer(Str)) - end; -format_listen_on(Bin) when is_binary(Bin) -> - format_listen_on(binary_to_list(Bin)). - %%%============================================================================================== %% schema util schema(Ref) when is_atom(Ref) -> From b4be9b6faea876acba8589bcb9bfb32d2b07a9cf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 26 Jul 2022 09:50:14 -0300 Subject: [PATCH 079/105] chore: bump ekka -> 0.13.3 Mainly: https://github.com/emqx/erlang-rocksdb/pull/7 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index c5b591a79..b33840aaa 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 714279cfd..aba53d8e8 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.3", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, diff --git a/rebar.config b/rebar.config index 58b4b079b..289d261ba 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} From 932387844235d3841d744b7985e2544451bf1aa9 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 27 Jul 2022 16:23:54 +0800 Subject: [PATCH 080/105] chore: bump vsn --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 75d5852c9..338b0aba6 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.4"). +-define(EMQX_RELEASE_CE, "5.0.4-beta.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). From 23eeb21b13296b6b9d0383dc546dbae104006185 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 27 Jul 2022 15:41:56 +0800 Subject: [PATCH 081/105] fix(webhook): the 'max_retries' opt means request retry times --- CHANGES-5.0.md | 1 + .../i18n/emqx_bridge_webhook_schema.conf | 11 ++ apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 1 - apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 +- .../src/emqx_bridge_webhook_schema.erl | 118 ++++++++++-------- .../i18n/emqx_connector_http.conf | 11 -- .../src/emqx_connector_http.erl | 50 ++++---- mix.exs | 2 +- rebar.config | 2 +- scripts/relup-test/relup.lux | 2 +- 11 files changed, 107 insertions(+), 99 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index ae1184f81..e305bf854 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -23,6 +23,7 @@ **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`, which will carry the paging meta information. +* Fix the issue that webhook leaks TCP connections. [ehttpc#34](https://github.com/emqx/ehttpc/pull/34), [#8580](https://github.com/emqx/emqx/pull/8580) ## Enhancements diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf index cd2cafd78..fcc817bef 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf @@ -127,6 +127,17 @@ HTTP 请求的正文。
} } + config_max_retries { + desc { + en: """HTTP request max retry times if failed.""" + zh: """HTTP 请求失败最大重试次数""" + } + label: { + en: "HTTP Request Max Retries" + zh: "HTTP 请求重试次数" + } + } + desc_type { desc { en: """The Bridge Type""" diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 70550efe4..fe19ed066 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index bc9b6c5a2..37a42ab3d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -225,7 +225,6 @@ info_example_basic(webhook, _) -> request_timeout => <<"15s">>, connect_timeout => <<"15s">>, max_retries => 3, - retry_interval => <<"10s">>, pool_type => <<"random">>, pool_size => 4, enable_pipelining => 100, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 678aa1f10..d19cc8426 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -238,7 +238,8 @@ parse_confs( method := Method, body := Body, headers := Headers, - request_timeout := ReqTimeout + request_timeout := ReqTimeout, + max_retries := Retry } = Conf ) -> {BaseUrl, Path} = parse_url(Url), @@ -251,7 +252,8 @@ parse_confs( method => Method, body => Body, headers => Headers, - request_timeout => ReqTimeout + request_timeout => ReqTimeout, + max_retries => Retry } }; parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl index 972ba86bc..f11247d68 100644 --- a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl @@ -14,60 +14,7 @@ namespace() -> "bridge". roots() -> []. fields("config") -> - basic_config() ++ - [ - {url, - mk( - binary(), - #{ - required => true, - desc => ?DESC("config_url") - } - )}, - {local_topic, - mk( - binary(), - #{desc => ?DESC("config_local_topic")} - )}, - {method, - mk( - method(), - #{ - default => post, - desc => ?DESC("config_method") - } - )}, - {headers, - mk( - map(), - #{ - default => #{ - <<"accept">> => <<"application/json">>, - <<"cache-control">> => <<"no-cache">>, - <<"connection">> => <<"keep-alive">>, - <<"content-type">> => <<"application/json">>, - <<"keep-alive">> => <<"timeout=5">> - }, - desc => ?DESC("config_headers") - } - )}, - {body, - mk( - binary(), - #{ - default => <<"${payload}">>, - desc => ?DESC("config_body") - } - )}, - {request_timeout, - mk( - emqx_schema:duration_ms(), - #{ - default => <<"15s">>, - desc => ?DESC("config_request_timeout") - } - )} - ]; + basic_config() ++ request_config(); fields("post") -> [ type_field(), @@ -106,6 +53,69 @@ basic_config() -> ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). +request_config() -> + [ + {url, + mk( + binary(), + #{ + required => true, + desc => ?DESC("config_url") + } + )}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("config_local_topic")} + )}, + {method, + mk( + method(), + #{ + default => post, + desc => ?DESC("config_method") + } + )}, + {headers, + mk( + map(), + #{ + default => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keep-alive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">> + }, + desc => ?DESC("config_headers") + } + )}, + {body, + mk( + binary(), + #{ + default => <<"${payload}">>, + desc => ?DESC("config_body") + } + )}, + {max_retries, + mk( + non_neg_integer(), + #{ + default => 2, + desc => ?DESC("config_max_retries") + } + )}, + {request_timeout, + mk( + emqx_schema:duration_ms(), + #{ + default => <<"15s">>, + desc => ?DESC("config_request_timeout") + } + )} + ]. + %%====================================================================================== type_field() -> diff --git a/apps/emqx_connector/i18n/emqx_connector_http.conf b/apps/emqx_connector/i18n/emqx_connector_http.conf index 0e29a15d3..8664d324f 100644 --- a/apps/emqx_connector/i18n/emqx_connector_http.conf +++ b/apps/emqx_connector/i18n/emqx_connector_http.conf @@ -41,17 +41,6 @@ base URL 只包含host和port。
} } - retry_interval { - desc { - en: "Interval between retries." - zh: "重试之间的间隔时间。" - } - label: { - en: "Retry Interval" - zh: "重试间隔" - } - } - pool_type { desc { en: "The type of the pool. Can be one of `random`, `hash`." diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index f9e63dc57..59b4ddffa 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -88,22 +88,6 @@ fields(config) -> desc => ?DESC("connect_timeout") } )}, - {max_retries, - sc( - non_neg_integer(), - #{ - default => 5, - desc => ?DESC("max_retries") - } - )}, - {retry_interval, - sc( - emqx_schema:duration(), - #{ - default => "1s", - desc => ?DESC("retry_interval") - } - )}, {pool_type, sc( pool_type(), @@ -147,6 +131,14 @@ fields("request") -> {path, hoconsc:mk(binary(), #{required => false, desc => ?DESC("path")})}, {body, hoconsc:mk(binary(), #{required => false, desc => ?DESC("body")})}, {headers, hoconsc:mk(map(), #{required => false, desc => ?DESC("headers")})}, + {max_retries, + sc( + non_neg_integer(), + #{ + required => false, + desc => ?DESC("max_retries") + } + )}, {request_timeout, sc( emqx_schema:duration_ms(), @@ -182,8 +174,6 @@ on_start( path := BasePath }, connect_timeout := ConnectTimeout, - max_retries := MaxRetries, - retry_interval := RetryInterval, pool_type := PoolType, pool_size := PoolSize } = Config @@ -206,8 +196,6 @@ on_start( {host, Host}, {port, Port}, {connect_timeout, ConnectTimeout}, - {retry, MaxRetries}, - {retry_timeout, RetryInterval}, {keepalive, 30000}, {pool_type, PoolType}, {pool_size, PoolSize}, @@ -247,17 +235,23 @@ on_query(InstId, {send_message, Msg}, AfterQuery, State) -> path := Path, body := Body, headers := Headers, - request_timeout := Timeout + request_timeout := Timeout, + max_retries := Retry } = process_request(Request, Msg), - on_query(InstId, {Method, {Path, Headers, Body}, Timeout}, AfterQuery, State) + on_query( + InstId, + {undefined, Method, {Path, Headers, Body}, Timeout, Retry}, + AfterQuery, + State + ) end; on_query(InstId, {Method, Request}, AfterQuery, State) -> - on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State); + on_query(InstId, {undefined, Method, Request, 5000, 2}, AfterQuery, State); on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> - on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); + on_query(InstId, {undefined, Method, Request, Timeout, 2}, AfterQuery, State); on_query( InstId, - {KeyOrNum, Method, Request, Timeout}, + {KeyOrNum, Method, Request, Timeout, Retry}, AfterQuery, #{pool_name := PoolName, base_path := BasePath} = State ) -> @@ -275,7 +269,8 @@ on_query( end, Method, NRequest, - Timeout + Timeout, + Retry ) of {error, Reason} -> @@ -368,7 +363,8 @@ preprocess_request( path => emqx_plugin_libs_rule:preproc_tmpl(Path), body => emqx_plugin_libs_rule:preproc_tmpl(Body), headers => preproc_headers(Headers), - request_timeout => maps:get(request_timeout, Req, 30000) + request_timeout => maps:get(request_timeout, Req, 30000), + max_retries => maps:get(max_retries, Req, 2) }. preproc_headers(Headers) when is_map(Headers) -> diff --git a/mix.exs b/mix.exs index 714279cfd..98269bfde 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do {:lc, github: "emqx/lc", tag: "0.3.1"}, {:redbug, "2.0.7"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, - {:ehttpc, github: "emqx/ehttpc", tag: "0.2.1"}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0"}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, diff --git a/rebar.config b/rebar.config index 58b4b079b..7acbf90dd 100644 --- a/rebar.config +++ b/rebar.config @@ -49,7 +49,7 @@ , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.1"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.3.0"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} diff --git a/scripts/relup-test/relup.lux b/scripts/relup-test/relup.lux index 8db8169f8..c05425e71 100644 --- a/scripts/relup-test/relup.lux +++ b/scripts/relup-test/relup.lux @@ -15,7 +15,7 @@ ?SH-PROMPT ## create a webhook data bridge with id "my_webhook" - !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","body":"","method":"post","url":"http://webhook.emqx.io:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","request_timeout":"5s","max_retries":3,"type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status' + !curl --user admin:public --silent --show-error 'http://localhost:18083/api/v5/bridges' -X 'POST' -H 'Content-Type: application/json' --data-binary '{"name":"my_webhook","body":"","method":"post","url":"http://webhook.emqx.io:7077/counter","headers":{"content-type":"application/json"},"pool_size":4,"enable_pipelining":100,"connect_timeout":"5s","type":"webhook","ssl":{"enable":false,"verify":"verify_none"}}' | jq '.status' ?connected ?SH-PROMPT From 5a7d0fff88fb8d2ccae42a0699772d641e2560e2 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:07:56 +0200 Subject: [PATCH 082/105] fix(windows): Build without rocksdb on windows --- build | 1 + 1 file changed, 1 insertion(+) diff --git a/build b/build index 1a7165ced..07c16b69e 100755 --- a/build +++ b/build @@ -57,6 +57,7 @@ if [ "${SYSTEM}" = 'windows' ]; then # windows does not like the find FIND="/usr/bin/find" TAR="/usr/bin/tar" + export BUILD_WITHOUT_ROCKSDB="on" else FIND='find' TAR='tar' From aa8026104a5c2c8dce7b14d426d51fc72928fb4e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 27 Jul 2022 12:04:43 -0300 Subject: [PATCH 083/105] chore(mix): configure ee apps in `mix.exs` --- mix.exs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index aba53d8e8..2177450f1 100644 --- a/mix.exs +++ b/mix.exs @@ -30,17 +30,17 @@ defmodule EMQXUmbrella.MixProject do """ def project() do - check_profile!() + profile_info = check_profile!() [ app: :emqx_mix, version: pkg_vsn(), - deps: deps(), + deps: deps(profile_info), releases: releases() ] end - defp deps() do + defp deps(profile_info) do # we need several overrides here because dependencies specify # other exact versions, and not ranges. [ @@ -89,7 +89,8 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.11.2", override: true, runtime: false} - ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() + ] ++ + umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() end defp umbrella_apps() do @@ -105,6 +106,24 @@ defmodule EMQXUmbrella.MixProject do end) end + defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do + "lib-ee/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.map(fn path -> + app = + path + |> String.trim_leading("lib-ee/") + |> String.to_atom() + + {app, path: path, manager: :rebar3, override: true} + end) + end + + defp enterprise_apps(_profile_info) do + [] + end + defp releases() do [ emqx: fn -> From 9cb37e06a2c081ffc6b4f42304885e051f1a4bdd Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 27 Jul 2022 16:36:03 -0300 Subject: [PATCH 084/105] ci: add `mix format` to `make fmt` --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index bde3e6d76..9f656eac3 100644 --- a/Makefile +++ b/Makefile @@ -249,3 +249,4 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) fmt: $(REBAR) @./scripts/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' @./scripts/erlfmt -w 'rebar.config.erl' + @mix format From b780d944abfec3e4148748dce7e304a166364426 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Jul 2022 13:39:32 +0800 Subject: [PATCH 085/105] chore: update dashboard vsn to v1.0.5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bde3e6d76..7eeb2ca0d 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1 +export EMQX_DASHBOARD_VERSION ?= v1.0.5 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 5835120d8e6f039271f63ac6fdeff1dd5dc1cc4a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Jul 2022 13:41:53 +0800 Subject: [PATCH 086/105] chore: relealse v5.0.4 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 338b0aba6..75d5852c9 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.4-beta.1"). +-define(EMQX_RELEASE_CE, "5.0.4"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). From aff3afb3854bd75a4cd9d956c578dad89bcab587 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Jul 2022 10:23:14 -0300 Subject: [PATCH 087/105] ci: use `Path.basename` --- mix.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 2177450f1..d67868ff8 100644 --- a/mix.exs +++ b/mix.exs @@ -99,7 +99,7 @@ defmodule EMQXUmbrella.MixProject do |> Enum.map(fn path -> app = path - |> String.trim_leading("apps/") + |> Path.basename() |> String.to_atom() {app, path: path, manager: :rebar3, override: true} @@ -113,7 +113,7 @@ defmodule EMQXUmbrella.MixProject do |> Enum.map(fn path -> app = path - |> String.trim_leading("lib-ee/") + |> Path.basename() |> String.to_atom() {app, path: path, manager: :rebar3, override: true} From 80d35feb3315de00ad38d31a6f530112c86402d6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Jul 2022 11:16:33 -0300 Subject: [PATCH 088/105] fix(license): change schema to allow updating license to different type Before this change, if a license in `emqx.conf` was of type `key`/`file` and then changed to the opposite type, such change would be saved to `{cluster,local}-overrides.conf`. When the node restarts after that, the configs are merged naively and becomes invalid, as it contains both the `key` and `file` keys, and crashes. --- bin/emqx | 2 +- bin/nodetool | 4 +- lib-ee/emqx_license/etc/emqx_license.conf | 1 + lib-ee/emqx_license/src/emqx_license.erl | 8 +- .../emqx_license/src/emqx_license_schema.erl | 34 +++++++- .../emqx_license/test/emqx_license_SUITE.erl | 82 +++++++++++++++++-- .../test/emqx_license_checker_SUITE.erl | 2 +- .../test/emqx_license_cli_SUITE.erl | 4 +- .../test/emqx_license_installer_SUITE.erl | 2 +- .../test/emqx_license_parser_SUITE.erl | 2 +- .../test/emqx_license_parser_legacy_SUITE.erl | 2 +- .../test/emqx_license_resources_SUITE.erl | 2 +- 12 files changed, 121 insertions(+), 24 deletions(-) diff --git a/bin/emqx b/bin/emqx index cac5cf655..a70c676fd 100755 --- a/bin/emqx +++ b/bin/emqx @@ -416,7 +416,7 @@ call_hocon() { ## and parsing HOCON config + environment variables is a non-trivial task CONF_KEYS=( 'node.data_dir' 'node.name' 'node.cookie' 'node.db_backend' 'cluster.proto_dist' ) if [ "$IS_ENTERPRISE" = 'yes' ]; then - CONF_KEYS+=( 'license.file' 'license.key' ) + CONF_KEYS+=( 'license.type' 'license.file' 'license.key' ) fi if [ "$IS_BOOT_COMMAND" = 'yes' ]; then diff --git a/bin/nodetool b/bin/nodetool index 773f855a0..0711f32d6 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -25,9 +25,9 @@ main(Args) -> %% forward the call to hocon_cli hocon_cli:main(Rest); ["check_license_key", Key] -> - check_license(#{key => list_to_binary(Key)}); + check_license(#{type => key, key => list_to_binary(Key)}); ["check_license_file", File] -> - check_license(#{file => list_to_binary(File)}); + check_license(#{type => file, file => list_to_binary(File)}); _ -> do(Args) end. diff --git a/lib-ee/emqx_license/etc/emqx_license.conf b/lib-ee/emqx_license/etc/emqx_license.conf index 476444ea0..b5684b740 100644 --- a/lib-ee/emqx_license/etc/emqx_license.conf +++ b/lib-ee/emqx_license/etc/emqx_license.conf @@ -1,4 +1,5 @@ license { + type = key # The default license has 1000 connections limit, it is issued on 20220419 and valid for 5 years (1825 days) key = "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZGVmYXVsdAoyMDIyMDQxOQoxODI1CjEwMDAK.MEQCICbgRVijCQov2hrvZXR1mk9Oa+tyV1F5oJ6iOZeSHjnQAiB9dUiVeaZekDOjztk+NCWjhk4PG8tWfw2uFZWruSzD6g==" connection_low_watermark = 75%, diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index 787a8b283..e7a0cce48 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -130,7 +130,7 @@ do_update({file, Filename}, Conf) -> {ok, Content} -> case emqx_license_parser:parse(Content) of {ok, _License} -> - maps:remove(<<"key">>, Conf#{<<"file">> => Filename}); + maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => Filename}); {error, Reason} -> erlang:throw(Reason) end; @@ -140,7 +140,7 @@ do_update({file, Filename}, Conf) -> do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> case emqx_license_parser:parse(Content) of {ok, _License} -> - maps:remove(<<"file">>, Conf#{<<"key">> => Content}); + maps:remove(<<"file">>, Conf#{<<"type">> => key, <<"key">> => Content}); {error, Reason} -> erlang:throw(Reason) end; @@ -151,12 +151,12 @@ do_update(_Other, Conf) -> check_max_clients_exceeded(MaxClients) -> emqx_license_resources:connection_count() > MaxClients * 1.1. -read_license(#{file := Filename}) -> +read_license(#{type := file, file := Filename}) -> case file:read_file(Filename) of {ok, Content} -> emqx_license_parser:parse(Content); {error, _} = Error -> Error end; -read_license(#{key := Content}) -> +read_license(#{type := key, key := Content}) -> emqx_license_parser:parse(Content). handle_config_update_result({error, _} = Error) -> diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index d6517ab88..88d245eb3 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -14,14 +14,15 @@ -export([roots/0, fields/1, validations/0, desc/1]). +-export([ + license_type/0 +]). + roots() -> [ {license, hoconsc:mk( - hoconsc:union([ - hoconsc:ref(?MODULE, key_license), - hoconsc:ref(?MODULE, file_license) - ]), + license_type(), #{ desc => "EMQX Enterprise license.\n" @@ -36,16 +37,35 @@ roots() -> fields(key_license) -> [ + {type, #{ + type => key, + required => true + }}, {key, #{ type => string(), %% so it's not logged sensitive => true, + required => true, desc => "License string" + }}, + {file, #{ + type => string(), + required => false }} | common_fields() ]; fields(file_license) -> [ + {type, #{ + type => file, + required => true + }}, + {key, #{ + type => string(), + %% so it's not logged + sensitive => true, + required => false + }}, {file, #{ type => string(), desc => "Path to the license file" @@ -77,6 +97,12 @@ common_fields() -> validations() -> [{check_license_watermark, fun check_license_watermark/1}]. +license_type() -> + hoconsc:union([ + hoconsc:ref(?MODULE, key_license), + hoconsc:ref(?MODULE, file_license) + ]). + check_license_watermark(Conf) -> case hocon_maps:get("license.connection_low_watermark", Conf) of undefined -> diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index bc73f9071..ba898b360 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -28,28 +28,65 @@ end_per_suite(_) -> init_per_testcase(Case, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), set_invalid_license_file(Case), - Config. + Paths = set_override_paths(Case), + Paths ++ Config. -end_per_testcase(Case, _Config) -> +end_per_testcase(Case, Config) -> restore_valid_license_file(Case), + clean_overrides(Case, Config), + ok. + +set_override_paths(TestCase) when + TestCase =:= t_change_from_file_to_key; + TestCase =:= t_change_from_key_to_file +-> + LocalOverridePath = filename:join([ + "/tmp", + "local-" ++ atom_to_list(TestCase) ++ ".conf" + ]), + ClusterOverridePath = filename:join([ + "/tmp", + "local-" ++ atom_to_list(TestCase) ++ ".conf" + ]), + application:set_env(emqx, local_override_conf_file, LocalOverridePath), + application:set_env(emqx, cluster_override_conf_file, ClusterOverridePath), + [ + {local_override_path, LocalOverridePath}, + {cluster_override_path, ClusterOverridePath} + ]; +set_override_paths(_TestCase) -> + []. + +clean_overrides(TestCase, Config) when + TestCase =:= t_change_from_file_to_key; + TestCase =:= t_change_from_key_to_file +-> + LocalOverridePath = ?config(local_override_path, Config), + ClusterOverridePath = ?config(cluster_override_path, Config), + file:delete(LocalOverridePath), + file:delete(ClusterOverridePath), + application:unset_env(emqx, local_override_conf_file), + application:unset_env(emqx, cluster_override_conf_file), + ok; +clean_overrides(_TestCase, _Config) -> ok. set_invalid_license_file(t_read_license_from_invalid_file) -> - Config = #{file => "/invalid/file"}, + Config = #{type => file, file => "/invalid/file"}, emqx_config:put([license], Config); set_invalid_license_file(_) -> ok. restore_valid_license_file(t_read_license_from_invalid_file) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); restore_valid_license_file(_) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config), - RawConfig = #{<<"file">> => emqx_license_test_lib:default_license()}, + RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, emqx_config:put_raw([<<"license">>], RawConfig); set_special_configs(_) -> ok. @@ -183,6 +220,39 @@ t_check_not_loaded(_Config) -> emqx_license:check(#{}, #{}) ). +t_change_from_file_to_key(_Config) -> + %% precondition + ?assertMatch(#{file := _}, emqx_conf:get([license])), + + OldConf = emqx_conf:get_raw([]), + + %% this saves updated config to `{cluster,local}-overrrides.conf' + {ok, LicenseValue} = file:read_file(emqx_license_test_lib:default_license()), + {ok, _NewConf} = emqx_license:update_key(LicenseValue), + + %% assert that `{cluster,local}-overrides.conf' merge correctly + ?assertEqual(ok, emqx_config:init_load(emqx_license_schema, OldConf, #{})), + + ok. + +t_change_from_key_to_file(_Config) -> + Config = #{type => key, key => <<"some key">>}, + emqx_config:put([license], Config), + RawConfig = #{<<"type">> => key, <<"key">> => <<"some key">>}, + emqx_config:put_raw([<<"license">>], RawConfig), + + %% precondition + ?assertMatch(#{type := key, key := _}, emqx_conf:get([license])), + OldConf = emqx_conf:get_raw([]), + + %% this saves updated config to `{cluster,local}-overrrides.conf' + {ok, _NewConf} = emqx_license:update_file(emqx_license_test_lib:default_license()), + + %% assert that `{cluster,local}-overrides.conf' merge correctly + ?assertEqual(ok, emqx_config:init_load(emqx_license_schema, OldConf, #{})), + + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl index 8842db7f9..0e10b684d 100644 --- a/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl @@ -35,7 +35,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. diff --git a/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl index ab7fd2dc8..5cf11adda 100644 --- a/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl @@ -31,9 +31,9 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config), - RawConfig = #{<<"file">> => emqx_license_test_lib:default_license()}, + RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, emqx_config:put_raw([<<"license">>], RawConfig); set_special_configs(_) -> ok. diff --git a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl index e92c082be..e62c4d814 100644 --- a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl @@ -31,7 +31,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. diff --git a/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl index 3b4e78a49..e9868cdc1 100644 --- a/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl @@ -30,7 +30,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. diff --git a/lib-ee/emqx_license/test/emqx_license_parser_legacy_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_parser_legacy_SUITE.erl index b1c45ade7..61f3c4cd8 100644 --- a/lib-ee/emqx_license/test/emqx_license_parser_legacy_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_parser_legacy_SUITE.erl @@ -30,7 +30,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. diff --git a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl index a6411902e..fc4e19e5b 100644 --- a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl @@ -31,7 +31,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{file => emqx_license_test_lib:default_license()}, + Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. From f8a1bd0715a2039e8eebaf741923a26469119973 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Jul 2022 17:27:50 -0300 Subject: [PATCH 089/105] feat(license): copy license file to cluster when updating it --- apps/emqx/test/emqx_common_test_helpers.erl | 3 +- lib-ee/emqx_license/src/emqx_license.erl | 98 ++++++-- .../src/emqx_license_resources.erl | 2 +- .../src/proto/emqx_license_proto_v2.erl | 30 +++ .../emqx_license/test/emqx_license_SUITE.erl | 235 +++++++++++++++++- .../test/emqx_license_resources_SUITE.erl | 8 +- 6 files changed, 343 insertions(+), 33 deletions(-) create mode 100644 lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d0ee99d6d..b3292ded3 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -595,6 +595,7 @@ setup_node(Node, Opts) when is_map(Opts) -> EnvHandler = maps:get(env_handler, Opts, fun(_) -> ok end), ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true), LoadSchema = maps:get(load_schema, Opts, true), + SchemaMod = maps:get(schema_mod, Opts, emqx_schema), LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps), Env = maps:get(env, Opts, []), Conf = maps:get(conf, Opts, []), @@ -630,7 +631,7 @@ setup_node(Node, Opts) when is_map(Opts) -> %% Otherwise, configuration get's loaded and all preset env in envhandler is lost LoadSchema andalso begin - emqx_config:init_load(emqx_schema), + emqx_config:init_load(SchemaMod), application:set_env(emqx, init_config_load_done, true) end, diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index e7a0cce48..24b2cc709 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -22,7 +22,9 @@ read_license/0, read_license/1, update_file/1, - update_key/1 + update_key/1, + license_dir/0, + save_and_backup_license/1 ]). -define(CONF_KEY_PATH, [license]). @@ -54,15 +56,29 @@ unload() -> emqx_conf:remove_handler(?CONF_KEY_PATH), emqx_license_cli:unload(). +-spec license_dir() -> file:filename(). +license_dir() -> + filename:join([emqx:data_dir(), licenses]). + +%% Subdirectory relative to data dir. +-spec relative_license_path() -> file:filename(). +relative_license_path() -> + filename:join([licenses, "emqx.lic"]). + -spec update_file(binary() | string()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_file(Filename) when is_binary(Filename); is_list(Filename) -> - Result = emqx_conf:update( - ?CONF_KEY_PATH, - {file, Filename}, - #{rawconf_with_defaults => true, override_to => local} - ), - handle_config_update_result(Result). + case file:read_file(Filename) of + {ok, Contents} -> + Result = emqx_conf:update( + ?CONF_KEY_PATH, + {file, Contents}, + #{rawconf_with_defaults => true, override_to => local} + ), + handle_config_update_result(Result); + {error, Error} -> + {error, Error} + end. -spec update_key(binary() | string()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. @@ -125,18 +141,14 @@ del_license_hook() -> _ = emqx_hooks:del('client.connect', {?MODULE, check, []}), ok. -do_update({file, Filename}, Conf) -> - case file:read_file(Filename) of - {ok, Content} -> - case emqx_license_parser:parse(Content) of - {ok, _License} -> - maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => Filename}); - {error, Reason} -> - erlang:throw(Reason) - end; - {error, Reason} -> - erlang:throw({invalid_license_file, Reason}) - end; +do_update({file, NewContents}, Conf) -> + Res = emqx_license_proto_v2:save_and_backup_license(mria_mnesia:running_nodes(), NewContents), + %% assert + true = lists:all(fun(X) -> X =:= {ok, ok} end, Res), + %% Must be relative to the data dir, since different nodes might + %% have different data directories configured... + LicensePath = relative_license_path(), + maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => LicensePath}); do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> case emqx_license_parser:parse(Content) of {ok, _License} -> @@ -148,17 +160,61 @@ do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> do_update(_Other, Conf) -> Conf. +save_and_backup_license(NewLicenseKey) -> + %% Must be relative to the data dir, since different nodes might + %% have different data directories configured... + CurrentLicensePath = filename:join(emqx:data_dir(), relative_license_path()), + LicenseDir = filename:dirname(CurrentLicensePath), + case filelib:ensure_dir(CurrentLicensePath) of + ok -> ok; + {error, EnsureError} -> throw({error_creating_license_dir, EnsureError}) + end, + case file:read_file(CurrentLicensePath) of + {ok, NewLicenseKey} -> + %% same contents; nothing to do. + ok; + {ok, _OldContents} -> + Time = calendar:system_time_to_rfc3339(erlang:system_time(second)), + BackupPath = filename:join([ + LicenseDir, + "emqx.lic." ++ Time ++ ".backup" + ]), + case file:copy(CurrentLicensePath, BackupPath) of + {ok, _} -> ok; + {error, CopyError} -> throw({error_backing_up_license, CopyError}) + end, + ok; + {error, enoent} -> + ok; + {error, Error} -> + throw({error_reading_existing_license, Error}) + end, + case file:write_file(CurrentLicensePath, NewLicenseKey) of + ok -> ok; + {error, WriteError} -> throw({error_writing_license, WriteError}) + end, + ok. + check_max_clients_exceeded(MaxClients) -> emqx_license_resources:connection_count() > MaxClients * 1.1. read_license(#{type := file, file := Filename}) -> case file:read_file(Filename) of - {ok, Content} -> emqx_license_parser:parse(Content); - {error, _} = Error -> Error + {ok, Content} -> + emqx_license_parser:parse(Content); + {error, _} = Error -> + %% Could be a relative path in data folder after update. + FilenameDataDir = filename:join(emqx:data_dir(), Filename), + case file:read_file(FilenameDataDir) of + {ok, Content} -> emqx_license_parser:parse(Content); + _Error -> Error + end end; read_license(#{type := key, key := Content}) -> emqx_license_parser:parse(Content). +handle_config_update_result({error, {post_config_update, ?MODULE, Error}}) -> + {error, Error}; handle_config_update_result({error, _} = Error) -> Error; handle_config_update_result({ok, #{post_config_update := #{emqx_license := Result}}}) -> diff --git a/lib-ee/emqx_license/src/emqx_license_resources.erl b/lib-ee/emqx_license/src/emqx_license_resources.erl index 551601923..4aaf853a3 100644 --- a/lib-ee/emqx_license/src/emqx_license_resources.erl +++ b/lib-ee/emqx_license/src/emqx_license_resources.erl @@ -128,6 +128,6 @@ ensure_timer(#{check_peer_interval := CheckInterval} = State) -> remote_connection_count() -> Nodes = mria_mnesia:running_nodes() -- [node()], - Results = emqx_license_proto_v1:remote_connection_counts(Nodes), + Results = emqx_license_proto_v2:remote_connection_counts(Nodes), Counts = [Count || {ok, Count} <- Results], lists:sum(Counts). diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl new file mode 100644 index 000000000..6af1cea77 --- /dev/null +++ b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_proto_v2). + +-behaviour(emqx_bpapi). + +-include_lib("emqx/include/bpapi.hrl"). + +-export([introduced_in/0]). + +-export([ + remote_connection_counts/1, + save_and_backup_license/2 +]). + +-define(TIMEOUT, 500). +-define(BACKUP_TIMEOUT, 15_000). + +introduced_in() -> + "5.0.5". + +-spec remote_connection_counts(list(node())) -> list({atom(), term()}). +remote_connection_counts(Nodes) -> + erpc:multicall(Nodes, emqx_license_resources, local_connection_count, [], ?TIMEOUT). + +-spec save_and_backup_license(list(node()), binary()) -> list({atom(), term()}). +save_and_backup_license(Nodes, NewLicenseKey) -> + erpc:multicall(Nodes, emqx_license, save_and_backup_license, [NewLicenseKey], ?BACKUP_TIMEOUT). diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index ba898b360..a648595d2 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -29,11 +29,13 @@ init_per_testcase(Case, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), set_invalid_license_file(Case), Paths = set_override_paths(Case), - Paths ++ Config. + Config0 = setup_test(Case, Config), + Paths ++ Config0 ++ Config. end_per_testcase(Case, Config) -> restore_valid_license_file(Case), clean_overrides(Case, Config), + teardown_test(Case, Config), ok. set_override_paths(TestCase) when @@ -71,6 +73,114 @@ clean_overrides(TestCase, Config) when clean_overrides(_TestCase, _Config) -> ok. +setup_test(TestCase, Config) when + TestCase =:= t_update_file_cluster_backup +-> + DataDir = ?config(data_dir, Config), + {LicenseKey, _License} = mk_license( + [ + %% license format version + "220111", + %% license type + "0", + %% customer type + "10", + %% customer name + "Foo", + %% customer email + "contact@foo.com", + %% deplayment name + "bar-deployment", + %% start date + "20220111", + %% days + "100000", + %% max connections + "19" + ] + ), + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, [emqx_conf, emqx_license]}, + {load_schema, false}, + {schema_mod, emqx_enterprise_conf_schema}, + {env_handler, fun + (emqx) -> + emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + %% emqx_config:save_schema_mod_and_names(emqx_license_schema), + application:set_env(emqx, boot_modules, []), + application:set_env( + emqx, + data_dir, + filename:join([ + DataDir, + TestCase, + node() + ]) + ), + ok; + (emqx_conf) -> + emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + %% emqx_config:save_schema_mod_and_names(emqx_license_schema), + application:set_env( + emqx, + data_dir, + filename:join([ + DataDir, + TestCase, + node() + ]) + ), + ok; + (emqx_license) -> + LicensePath = filename:join(emqx_license:license_dir(), "emqx.lic"), + filelib:ensure_dir(LicensePath), + ok = file:write_file(LicensePath, LicenseKey), + LicConfig = #{type => file, file => LicensePath}, + emqx_config:put([license], LicConfig), + RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, + emqx_config:put_raw([<<"license">>], RawConfig), + ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]), + %% meck:expect(emqx_license, read_license, fun() -> {ok, License} end), + meck:expect( + emqx_license_parser, + parse, + fun(X) -> + emqx_license_parser:parse( + X, + emqx_license_test_lib:public_key_pem() + ) + end + ), + ok; + (_) -> + ok + end} + ] + ), + Nodes = [emqx_common_test_helpers:start_slave(Name, Opts) || {Name, Opts} <- Cluster], + [{nodes, Nodes}, {cluster, Cluster}, {old_license, LicenseKey}]; +setup_test(_TestCase, _Config) -> + []. + +teardown_test(TestCase, Config) when + TestCase =:= t_update_file_cluster_backup +-> + Nodes = ?config(nodes, Config), + lists:foreach( + fun(N) -> + LicenseDir = erpc:call(N, emqx_license, license_dir, []), + {ok, _} = emqx_common_test_helpers:stop_slave(N), + ok = file:del_dir_r(LicenseDir), + ok + end, + Nodes + ), + ok; +teardown_test(_TestCase, _Config) -> + ok. + set_invalid_license_file(t_read_license_from_invalid_file) -> Config = #{type => file, file => "/invalid/file"}, emqx_config:put([license], Config); @@ -91,13 +201,17 @@ set_special_configs(emqx_license) -> set_special_configs(_) -> ok. +assert_on_nodes(Nodes, RunFun, CheckFun) -> + Res = [{N, erpc:call(N, RunFun)} || N <- Nodes], + lists:foreach(CheckFun, Res). + %%------------------------------------------------------------------------------ %% Tests %%------------------------------------------------------------------------------ t_update_file(_Config) -> ?assertMatch( - {error, {invalid_license_file, enoent}}, + {error, enoent}, emqx_license:update_file("/unknown/path") ), @@ -112,6 +226,115 @@ t_update_file(_Config) -> emqx_license:update_file(emqx_license_test_lib:default_license()) ). +t_update_file_cluster_backup(Config) -> + OldLicenseKey = ?config(old_license, Config), + Nodes = [N1 | _] = ?config(nodes, Config), + + %% update the license file for the cluster + {NewLicenseKey, NewDecodedLicense} = mk_license( + [ + %% license format version + "220111", + %% license type + "0", + %% customer type + "10", + %% customer name + "Foo", + %% customer email + "contact@foo.com", + %% deplayment name + "bar-deployment", + %% start date + "20220111", + %% days + "100000", + %% max connections + "190" + ] + ), + NewLicensePath = "tmp_new_license.lic", + ok = file:write_file(NewLicensePath, NewLicenseKey), + {ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]), + + assert_on_nodes( + Nodes, + fun() -> + Conf = emqx_conf:get([license]), + emqx_license:read_license(Conf) + end, + fun({N, Res}) -> + ?assertMatch({ok, _}, Res, #{node => N}), + {ok, License} = Res, + ?assertEqual(NewDecodedLicense, License, #{node => N}) + end + ), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + file:list_dir(LicenseDir) + end, + fun({N, Res}) -> + ?assertMatch({ok, _}, Res, #{node => N}), + {ok, DirContents} = Res, + %% the now current license + ?assert(lists:member("emqx.lic", DirContents), #{node => N, dir_contents => DirContents}), + %% the backed up old license + ?assert( + lists:any( + fun + ("emqx.lic." ++ Suffix) -> lists:suffix(".backup", Suffix); + (_) -> false + end, + DirContents + ), + #{node => N, dir_contents => DirContents} + ) + end + ), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + {ok, DirContents} = file:list_dir(LicenseDir), + [BackupLicensePath0] = [ + F + || "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F) + ], + BackupLicensePath = "emqx.lic." ++ BackupLicensePath0, + {ok, BackupLicense} = file:read_file(filename:join(LicenseDir, BackupLicensePath)), + {ok, NewLicense} = file:read_file(filename:join(LicenseDir, "emqx.lic")), + #{ + backup => BackupLicense, + new => NewLicense + } + end, + fun({N, #{backup := BackupLicense, new := NewLicense}}) -> + ?assertEqual(OldLicenseKey, BackupLicense, #{node => N}), + ?assertEqual(NewLicenseKey, NewLicense, #{node => N}) + end + ), + + %% uploading the same license twice should not generate extra backups. + {ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + {ok, DirContents} = file:list_dir(LicenseDir), + [F || "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F)] + end, + fun({N, Backups}) -> + ?assertMatch([_], Backups, #{node => N}) + end + ), + + ok. + t_update_value(_Config) -> ?assertMatch( {error, [_ | _]}, @@ -132,7 +355,7 @@ t_read_license_from_invalid_file(_Config) -> ). t_check_exceeded(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", "0", @@ -161,7 +384,7 @@ t_check_exceeded(_Config) -> ). t_check_ok(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", "0", @@ -190,7 +413,7 @@ t_check_ok(_Config) -> ). t_check_expired(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", %% Official customer @@ -263,4 +486,4 @@ mk_license(Fields) -> EncodedLicense, emqx_license_test_lib:public_key_pem() ), - License. + {EncodedLicense, License}. diff --git a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl index fc4e19e5b..84a63e611 100644 --- a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl @@ -59,9 +59,9 @@ t_connection_count(_Config) -> meck:new(emqx_cm, [passthrough]), meck:expect(emqx_cm, get_connected_client_count, fun() -> 10 end), - meck:new(emqx_license_proto_v1, [passthrough]), + meck:new(emqx_license_proto_v2, [passthrough]), meck:expect( - emqx_license_proto_v1, + emqx_license_proto_v2, remote_connection_counts, fun(_Nodes) -> [{ok, 5}, {error, some_error}] @@ -82,8 +82,8 @@ t_connection_count(_Config) -> end ), - meck:unload(emqx_license_proto_v1), + meck:unload(emqx_license_proto_v2), meck:unload(emqx_cm). t_emqx_license_proto(_Config) -> - ?assert("5.0.0" =< emqx_license_proto_v1:introduced_in()). + ?assert("5.0.0" =< emqx_license_proto_v2:introduced_in()). From 7c34011c1238183cf242dd8fa3bcad45cd8bc1da Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Jul 2022 17:45:44 -0300 Subject: [PATCH 090/105] chore: bump app vsns --- lib-ee/emqx_license/src/emqx_license.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 7d10277d8..9aba01e96 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.0"}, + {vsn, "5.0.1"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib]}, From 8e08f045aa2912082fcf80799e31998bb0c22cb4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Jul 2022 17:59:54 -0300 Subject: [PATCH 091/105] chore(bpapi): add new vsn --- apps/emqx/priv/bpapi.versions | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 12e70e36a..d1176e7ac 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -14,6 +14,7 @@ {emqx_gateway_cm,1}. {emqx_gateway_http,1}. {emqx_license,1}. +{emqx_license,2}. {emqx_management,1}. {emqx_management,2}. {emqx_mgmt_api_plugins,1}. From c56f84b997517ee5efb5ba038668121f980f0c52 Mon Sep 17 00:00:00 2001 From: lukas Date: Thu, 21 Jul 2022 19:18:42 +0200 Subject: [PATCH 092/105] feat(helm): add ssl support for helm chart --- deploy/charts/emqx/README.md | 167 ++++++++++-------- deploy/charts/emqx/templates/StatefulSet.yaml | 14 +- deploy/charts/emqx/templates/certificate.yaml | 16 ++ deploy/charts/emqx/values.yaml | 9 + 4 files changed, 135 insertions(+), 71 deletions(-) create mode 100644 deploy/charts/emqx/templates/certificate.yaml diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 496d52061..ed331619d 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -1,92 +1,121 @@ # Introduction -This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. + +This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. # Prerequisites + + Kubernetes 1.6+ + Helm # Installing the Chart + To install the chart with the release name `my-emqx`: -+ From github - ``` - $ git clone https://github.com/emqx/emqx.git - $ cd emqx/deploy/charts/emqx - $ helm install my-emqx . - ``` ++ From github + ``` + $ git clone https://github.com/emqx/emqx.git + $ cd emqx/deploy/charts/emqx + $ helm install my-emqx . + ``` -+ From chart repos - ``` - helm repo add emqx https://repos.emqx.io/charts - helm install my-emqx emqx/emqx - ``` - > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. ++ From chart repos + ``` + helm repo add emqx https://repos.emqx.io/charts + helm install my-emqx emqx/emqx + ``` + > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. # Uninstalling the Chart + To uninstall/delete the `my-emqx` deployment: + ``` $ helm del my-emqx ``` # Configuration + The following table lists the configurable parameters of the emqx chart and their default values. -| Parameter | Description | Default Value | -| --- | --- | --- | -| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3| -| `image.repository` | EMQX Image name |emqx/emqx| -| `image.pullPolicy` | The image pull policy |IfNotPresent| -| `image.pullSecrets ` | The image pull secrets |`[]` (does not add image pull secrets to deployed pods)| -| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | -| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | -`podAnnotations ` | Annotations for pod | `{}` -`podManagementPolicy`| To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` -| `persistence.enabled` | Enable EMQX persistence using PVC |false| -| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)| -| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template |""| -| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce| -| `persistence.size` | PVC Storage Request for EMQX volume |20Mi| -| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`| -| `resources` | CPU/Memory resource requests/limits |{}| -| `nodeSelector` | Node labels for pod assignment |`{}`| -| `tolerations` | Toleration labels for pod assignment |`[]`| -| `affinity` | Map of node/pod affinities |`{}`| -| `service.type` | Kubernetes Service type. |ClusterIP| -| `service.mqtt` | Port for MQTT. |1883| -| `service.mqttssl` | Port for MQTT(SSL). |8883| -| `service.mgmt` | Port for mgmt API. |8081| -| `service.ws` | Port for WebSocket/HTTP. |8083| -| `service.wss` | Port for WSS/HTTPS. |8084| -| `service.dashboard` | Port for dashboard. |18083| -| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. |nil| -| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil| -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. |nil| -| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. |nil| -| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. |nil| -| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. |nil| -| `service.loadBalancerIP` | loadBalancerIP for Service | nil | -| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | -| `service.externalIPs` | ExternalIPs for the service | [] | -| `service.annotations` | Service annotations | {}(evaluated as a template)| -| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | -| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | -| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | -| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| Parameter | Description | Default Value | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | +| `image.repository` | EMQX Image name | emqx/emqx | +| `image.pullPolicy` | The image pull policy | IfNotPresent | +| `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | +| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | +| `podAnnotations ` | Annotations for pod | `{}` | +| `podManagementPolicy` | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` | +| `persistence.enabled` | Enable EMQX persistence using PVC | false | +| `persistence.storageClass` | Storage class of backing PVC | `nil` (uses alpha storage class annotation) | +| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | "" | +| `persistence.accessMode` | PVC Access Mode for EMQX volume | ReadWriteOnce | +| `persistence.size` | PVC Storage Request for EMQX volume | 20Mi | +| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. | `{}` | +| `resources` | CPU/Memory resource requests/limits | {} | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `service.type` | Kubernetes Service type. | ClusterIP | +| `service.mqtt` | Port for MQTT. | 1883 | +| `service.mqttssl` | Port for MQTT(SSL). | 8883 | +| `service.mgmt` | Port for mgmt API. | 8081 | +| `service.ws` | Port for WebSocket/HTTP. | 8083 | +| `service.wss` | Port for WSS/HTTPS. | 8084 | +| `service.dashboard` | Port for dashboard. | 18083 | +| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | +| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | +| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | +| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | +| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | +| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | +| `service.loadBalancerIP` | loadBalancerIP for Service | nil | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | +| `service.externalIPs` | ExternalIPs for the service | [] | +| `service.annotations` | Service annotations | {}(evaluated as a template) | +| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | +| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | +| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | +| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | +| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | +| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| `ssl.enabled` | Enable SSL support | false | +| `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | +| `ssl.existingName` | Name of existing certificate | emqx-tls | +| `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | +| `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | +| `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | ## EMQX specific settings -The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values. -Parameter | Description | Default Value ---- | --- | --- -`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` + +The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their +default values. +Parameter | Description | Default Value +--- | --- | --- +`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items +expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix +can be omitted) or using the configuration +files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil` + +## SSL settings +`cert-manager` generates secrets with certificate data using the keys `tls.crt` and `tls.key`. The helm chart always mounts those keys as files to `/tmp/ssl/` +which needs to explicitly configured by either changing the emqx config file or by passing the following environment variables: + +``` + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__CERTFILE: /tmp/ssl/tls.crt + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__KEYFILE: /tmp/ssl/tls.key +``` + +If you chose to use an existing certificate, make sure, you update the filenames accordingly. + diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 80f7e2c0a..d44c88a86 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -53,6 +53,11 @@ spec: {{- end }} spec: volumes: + {{- if .Values.ssl.enabled }} + - name: ssl-cert + secret: + secretName: {{ include "emqx.fullname" . }}-tls + {{- end }} {{- if not .Values.persistence.enabled }} - name: emqx-data emptyDir: {} @@ -124,12 +129,17 @@ spec: volumeMounts: - name: emqx-data mountPath: "/opt/emqx/data" - {{ if .Values.emqxLicenseSecretName }} + {{- if .Values.ssl.enabled }} + - name: ssl-cert + mountPath: /tmp/ssl + readOnly: true + {{- end}} + {{ if .Values.emqxLicenseSecretName }} - name: emqx-license mountPath: "/opt/emqx/etc/emqx.lic" subPath: "emqx.lic" readOnly: true - {{ end }} + {{- end }} readinessProbe: httpGet: path: /api/v5/status diff --git a/deploy/charts/emqx/templates/certificate.yaml b/deploy/charts/emqx/templates/certificate.yaml new file mode 100644 index 000000000..36b7f6521 --- /dev/null +++ b/deploy/charts/emqx/templates/certificate.yaml @@ -0,0 +1,16 @@ +{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "emqx.fullname" . }}-tls +spec: + secretName: {{ include "emqx.fullname" . }}-tls + issuerRef: + name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} + kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + dnsNames: + {{- range .Values.ssl.dnsnames }} + - {{ . }} + {{- end }} +{{- end -}} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 7ed4f4995..94e7eeb3c 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -203,3 +203,12 @@ containerSecurityContext: metrics: enabled: false type: prometheus + +ssl: + enabled: false + useExisting: false + existingName: emqx-tls + dnsnames: {} + issuer: + name: letsencrypt-dns + kind: ClusterIssuer From 78deee68460a6b93dfec61e94052ab6484914242 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 1 Aug 2022 17:35:48 +0800 Subject: [PATCH 093/105] fix(channel): Adjust the timing of the `client.connected` event --- apps/emqx/src/emqx_channel.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index cdd4b1a9e..39bd81b40 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%-------------------------------------------------------------------- %% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -357,7 +357,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> }, case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of {ok, Properties, NChannel2} -> - process_connect(Properties, ensure_connected(NChannel2)); + process_connect(Properties, NChannel2); {continue, Properties, NChannel2} -> handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2); {error, ReasonCode} -> @@ -381,7 +381,7 @@ handle_in( {ok, NProperties, NChannel} -> case ConnState of connecting -> - process_connect(NProperties, ensure_connected(NChannel)); + process_connect(NProperties, NChannel); _ -> handle_out( auth, @@ -611,7 +611,7 @@ process_connect( case emqx_cm:open_session(CleanStart, ClientInfo, ConnInfo) of {ok, #{session := Session, present := false}} -> NChannel = Channel#channel{session = Session}, - handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), NChannel = Channel#channel{ @@ -619,7 +619,7 @@ process_connect( resuming = true, pendings = Pendings1 }, - handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, ensure_connected(NChannel)); {error, client_id_unavailable} -> handle_out(connack, ?RC_CLIENT_IDENTIFIER_NOT_VALID, Channel); {error, Reason} -> From a78760dbac4de67e2c1c0ff92e33e6100b934836 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 1 Aug 2022 09:59:23 -0300 Subject: [PATCH 094/105] chore: bump app vsns --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index fed9e6bc2..b7e65a042 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index c3b7b4f13..4e1a3518f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 1640f4cc9..5a823067a 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx]}, From 4a1ede7e7ec2e565e4ff94a11a2a4d879ed5b498 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 29 Jul 2022 15:18:31 -0300 Subject: [PATCH 095/105] test: fix flaky gateway tests --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 4 ++++ apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index fb3207944..aac140d3e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -239,6 +239,7 @@ t_gateway_exproto_with_ssl(_) -> t_authn(_) -> GwConf = #{name => <<"stomp">>}, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{ @@ -263,6 +264,7 @@ t_authn(_) -> t_authn_data_mgmt(_) -> GwConf = #{name => <<"stomp">>}, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{ @@ -271,6 +273,7 @@ t_authn_data_mgmt(_) -> user_id_type => <<"clientid">> }, {201, _} = request(post, "/gateway/stomp/authentication", AuthConf), + ct:sleep(500), {200, ConfResp} = request(get, "/gateway/stomp/authentication"), assert_confs(AuthConf, ConfResp), @@ -374,6 +377,7 @@ t_listeners_authn(_) -> ] }, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {200, ConfResp} = request(get, "/gateway/stomp"), assert_confs(GwConf, ConfResp), diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 4633b421e..dff7b3fd4 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -47,6 +47,7 @@ end_per_suite(_Conf) -> init_per_testcase(_CaseName, Conf) -> _ = emqx_gateway_conf:unload_gateway(stomp), + ct:sleep(500), Conf. %%-------------------------------------------------------------------- @@ -282,6 +283,7 @@ t_load_remove_authn(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_1), assert_confs( @@ -314,6 +316,7 @@ t_load_remove_listeners(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_listener( <<"stomp">>, @@ -371,6 +374,7 @@ t_load_remove_listener_authn(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_authn( <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_1 From cffaf95d00a39908fa4687afa4e21cd08cb86949 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Aug 2022 17:20:58 +0800 Subject: [PATCH 096/105] chore: update apps/emqx/src/emqx_channel.erl --- apps/emqx/src/emqx_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 39bd81b40..eff03e8ed 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1,4 +1,4 @@ -%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- %% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); From 8e53a617910888d214ec73dad0d6fe587a8f4b83 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 2 Aug 2022 19:49:06 +0800 Subject: [PATCH 097/105] fix: return 503 instead of crash when dashboard generate router timeout --- apps/emqx_dashboard/src/emqx_dashboard_listener.erl | 7 ++++++- apps/emqx_dashboard/src/emqx_dashboard_middleware.erl | 3 ++- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 8bd5a4eb3..3c53c2e3f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -38,7 +38,12 @@ ]). is_ready(Timeout) -> - ready =:= gen_server:call(?MODULE, is_ready, Timeout). + try + ready =:= gen_server:call(?MODULE, is_ready, Timeout) + catch + exit:{timeout, _} -> + false + end. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl index a2cf9db6f..67f907bbb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl @@ -43,5 +43,6 @@ check_dispatch_ready(Env) -> true; true -> %% dashboard should always ready, if not, is_ready/1 will block until ready. - emqx_dashboard_listener:is_ready(timer:seconds(15)) + %% if not ready, dashboard will return 503. + emqx_dashboard_listener:is_ready(timer:seconds(20)) end. diff --git a/mix.exs b/mix.exs index 714279cfd..a218e091d 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.6", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.4", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index 58b4b079b..854ba0d59 100644 --- a/rebar.config +++ b/rebar.config @@ -57,7 +57,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.6"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.4"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From b19e8fb3cd9e5685a424d74d8978e6e817cdb3e0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 29 Jul 2022 13:42:24 -0300 Subject: [PATCH 098/105] feat(license): add HTTP API for license --- .../i18n/emqx_license_http_api.conf | 23 ++ .../src/emqx_license_http_api.erl | 142 ++++++++++++ .../emqx_license/test/emqx_license_SUITE.erl | 1 - .../test/emqx_license_http_api_SUITE.erl | 210 ++++++++++++++++++ .../test/emqx_license_test_lib.erl | 26 +++ scripts/merge-i18n.escript | 8 +- 6 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 lib-ee/emqx_license/i18n/emqx_license_http_api.conf create mode 100644 lib-ee/emqx_license/src/emqx_license_http_api.erl create mode 100644 lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf new file mode 100644 index 000000000..6b2c7a687 --- /dev/null +++ b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf @@ -0,0 +1,23 @@ +emqx_license_http_api { + desc_license_info_api { + desc { + en: "Get license info" + zh: "获取许可证信息" + } + label: { + en: "License info" + zh: "许可证信息" + } + } + + desc_license_upload_api { + desc { + en: "Upload a license file or key" + zh: "上传许可证文件或钥匙" + } + label: { + en: "Update license" + zh: "更新许可证" + } + } +} diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl new file mode 100644 index 000000000..b204583ba --- /dev/null +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -0,0 +1,142 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_http_api). + +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-export([ + namespace/0, + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + '/license'/2, + '/license/upload'/2 +]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + +namespace() -> "license_http_api". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/license", + "/license/upload" + ]. + +schema("/license") -> + #{ + 'operationId' => '/license', + get => #{ + tags => [<<"license">>], + summary => <<"Get license info">>, + description => ?DESC("desc_license_info_api"), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + map(), + #{ + sample_license_info => #{ + value => #{ + customer => "Foo", + customer_type => 10, + deployment => "bar-deployment", + email => "contact@foo.com", + expiry => false, + expiry_at => "2295-10-27", + max_connections => 10, + start_at => "2022-01-11", + type => "trial" + } + } + } + ) + } + } + }; +schema("/license/upload") -> + #{ + 'operationId' => '/license/upload', + post => #{ + tags => [<<"license">>], + summary => <<"Upload license">>, + description => ?DESC("desc_license_upload_api"), + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_license_schema:license_type(), + #{ + license_key => #{ + summary => <<"License key string">>, + value => #{ + <<"key">> => <<"xxx">>, + <<"connection_low_watermark">> => "75%", + <<"connection_high_watermark">> => "80%" + } + }, + license_file => #{ + summary => <<"Path to a license file">>, + value => #{ + <<"file">> => <<"/path/to/license">>, + <<"connection_low_watermark">> => "75%", + <<"connection_high_watermark">> => "80%" + } + } + } + ), + responses => #{ + 200 => <<"ok">>, + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"bad request">>), + 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"file not found">>) + } + } + }. + +'/license'(get, _Params) -> + License = maps:from_list(emqx_license_checker:dump()), + {200, License}. + +'/license/upload'(post, #{body := #{<<"file">> := Filepath}}) -> + case emqx_license:update_file(Filepath) of + {error, enoent} -> + ?SLOG(error, #{ + msg => "license_file_not_found", + path => Filepath + }), + {404, <<"file not found">>}; + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_license_file", + reason => Error, + path => Filepath + }), + {400, <<"bad request">>}; + {ok, _} -> + ?SLOG(info, #{ + msg => "updated_license_file", + path => Filepath + }), + {200, <<"ok">>} + end; +'/license/upload'(post, #{body := #{<<"key">> := Key}}) -> + case emqx_license:update_key(Key) of + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_license_key", + reason => Error + }), + {400, <<"bad request">>}; + {ok, _} -> + ?SLOG(info, #{msg => "updated_license_key"}), + {200, <<"ok">>} + end; +'/license/upload'(post, _Params) -> + {400, <<"bad request">>}. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index a648595d2..08b3cb692 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -142,7 +142,6 @@ setup_test(TestCase, Config) when RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, emqx_config:put_raw([<<"license">>], RawConfig), ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]), - %% meck:expect(emqx_license, read_license, fun() -> {ok, License} end), meck:expect( emqx_license_parser, parse, diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl new file mode 100644 index 000000000..06bf35867 --- /dev/null +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -0,0 +1,210 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_http_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + emqx_config:save_schema_mod_and_names(emqx_license_schema), + ok = meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect( + emqx_license_parser, + parse, + fun(X) -> + emqx_license_parser:parse( + X, + emqx_license_test_lib:public_key_pem() + ) + end + ), + emqx_common_test_helpers:start_apps([emqx_license, emqx_dashboard], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_common_test_helpers:stop_apps([emqx_license, emqx_dashboard]), + ok = meck:unload([emqx_license_parser]), + Config = #{type => file, file => emqx_license_test_lib:default_license()}, + emqx_config:put([license], Config), + RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, + emqx_config:put_raw([<<"license">>], RawConfig), + ok. + +set_special_configs(emqx_dashboard) -> + emqx_dashboard_api_test_helpers:set_default_config(<<"license_admin">>); +set_special_configs(emqx_license) -> + LicenseKey = emqx_license_test_lib:make_license(#{max_connections => "100"}), + Config = #{type => key, key => LicenseKey}, + emqx_config:put([license], Config), + RawConfig = #{<<"type">> => key, <<"key">> => LicenseKey}, + emqx_config:put_raw([<<"license">>], RawConfig); +set_special_configs(_) -> + ok. + +init_per_testcase(_TestCase, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + Config. + +end_per_testcase(_TestCase, _Config) -> + {ok, _} = reset_license(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +request(Method, Uri, Body) -> + emqx_dashboard_api_test_helpers:request(<<"license_admin">>, Method, Uri, Body). + +uri(Segments) -> + emqx_dashboard_api_test_helpers:uri(Segments). + +get_license() -> + maps:from_list(emqx_license_checker:dump()). + +default_license() -> + emqx_license_test_lib:make_license(#{max_connections => "100"}). + +reset_license() -> + emqx_license:update_key(default_license()). + +assert_untouched_license() -> + ?assertMatch( + #{max_connections := 100}, + get_license() + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_license_info(_Config) -> + Res = request(get, uri(["license"]), []), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, + ?assertEqual( + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 100, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) + ), + ok. + +t_license_upload_file_success(_Config) -> + NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + Path = "/tmp/new.lic", + ok = file:write_file(Path, NewKey), + try + ?assertEqual( + {ok, 200, <<"ok">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + ?assertMatch( + #{max_connections := 999}, + get_license() + ), + ok + after + ok = file:delete(Path), + ok + end. + +t_license_upload_file_not_found(_Config) -> + ?assertEqual( + {ok, 404, <<"file not found">>}, + request( + post, + uri(["license", "upload"]), + #{file => "/tmp/inexistent.lic"} + ) + ), + assert_untouched_license(), + ok. + +t_license_upload_file_reading_error(_Config) -> + %% eisdir + Path = "/tmp/", + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + assert_untouched_license(), + ok. + +t_license_upload_file_bad_license(_Config) -> + Path = "/tmp/bad.lic", + ok = file:write_file(Path, <<"bad key">>), + try + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + assert_untouched_license(), + ok + after + ok = file:delete(Path), + ok + end. + +t_license_upload_key_success(_Config) -> + NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + ?assertEqual( + {ok, 200, <<"ok">>}, + request( + post, + uri(["license", "upload"]), + #{key => NewKey} + ) + ), + ?assertMatch( + #{max_connections := 999}, + get_license() + ), + ok. + +t_license_upload_key_bad_key(_Config) -> + BadKey = <<"bad key">>, + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{key => BadKey} + ) + ), + assert_untouched_license(), + ok. diff --git a/lib-ee/emqx_license/test/emqx_license_test_lib.erl b/lib-ee/emqx_license/test/emqx_license_test_lib.erl index d3f2b5bd7..af3912f75 100644 --- a/lib-ee/emqx_license/test/emqx_license_test_lib.erl +++ b/lib-ee/emqx_license/test/emqx_license_test_lib.erl @@ -47,6 +47,32 @@ test_key(Filename, Format) -> public_key:pem_entry_decode(PemEntry) end. +make_license(Values0 = #{}) -> + Defaults = #{ + license_format => "220111", + license_type => "0", + customer_type => "10", + name => "Foo", + email => "contact@foo.com", + deployment => "bar-deployment", + start_date => "20220111", + days => "100000", + max_connections => "10" + }, + Values1 = maps:merge(Defaults, Values0), + Keys = [ + license_format, + license_type, + customer_type, + name, + email, + deployment, + start_date, + days, + max_connections + ], + Values = lists:map(fun(K) -> maps:get(K, Values1) end, Keys), + make_license(Values); make_license(Values) -> Key = private_key(), Text = string:join(Values, "\n"), diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 9f8ac91ff..e98631cfc 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -4,10 +4,12 @@ main(_) -> BaseConf = <<"">>, - Cfgs = get_all_cfgs("apps/"), - Conf = [merge(BaseConf, Cfgs), + Cfgs0 = get_all_cfgs("apps/"), + Cfgs1 = get_all_cfgs("lib-ee/"), + Conf0 = merge(BaseConf, Cfgs0), + Conf = [merge(Conf0, Cfgs1), io_lib:nl() - ], + ], ok = file:write_file("apps/emqx_dashboard/priv/i18n.conf", Conf). merge(BaseConf, Cfgs) -> From 1a66e53c4901335dbb91486314bfc0c64d72abd9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 1 Aug 2022 09:27:41 -0300 Subject: [PATCH 099/105] chore(license): change api responses after review --- .../src/emqx_license_http_api.erl | 62 ++++++--- .../test/emqx_license_http_api_SUITE.erl | 123 +++++++++++++----- 2 files changed, 129 insertions(+), 56 deletions(-) diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index b204583ba..e11631004 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -47,17 +47,7 @@ schema("/license") -> map(), #{ sample_license_info => #{ - value => #{ - customer => "Foo", - customer_type => 10, - deployment => "bar-deployment", - email => "contact@foo.com", - expiry => false, - expiry_at => "2295-10-27", - max_connections => 10, - start_at => "2022-01-11", - type => "trial" - } + value => sample_license_info_response() } } ) @@ -93,13 +83,36 @@ schema("/license/upload") -> } ), responses => #{ - 200 => <<"ok">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"bad request">>), - 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"file not found">>) + 200 => emqx_dashboard_swagger:schema_with_examples( + map(), + #{ + sample_license_info => #{ + value => sample_license_info_response() + } + } + ), + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad license key">>), + 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"File not found">>) } } }. +sample_license_info_response() -> + #{ + customer => "Foo", + customer_type => 10, + deployment => "bar-deployment", + email => "contact@foo.com", + expiry => false, + expiry_at => "2295-10-27", + max_connections => 10, + start_at => "2022-01-11", + type => "trial" + }. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + '/license'(get, _Params) -> License = maps:from_list(emqx_license_checker:dump()), {200, License}. @@ -111,20 +124,28 @@ schema("/license/upload") -> msg => "license_file_not_found", path => Filepath }), - {404, <<"file not found">>}; + {404, error_msg(?NOT_FOUND, <<"File not found">>)}; + {error, Error} when is_atom(Error) -> + ?SLOG(error, #{ + msg => "bad_license_file", + reason => Error, + path => Filepath + }), + {400, error_msg(?BAD_REQUEST, emqx_misc:explain_posix(Error))}; {error, Error} -> ?SLOG(error, #{ msg => "bad_license_file", reason => Error, path => Filepath }), - {400, <<"bad request">>}; + {400, error_msg(?BAD_REQUEST, <<"Bad license file">>)}; {ok, _} -> ?SLOG(info, #{ msg => "updated_license_file", path => Filepath }), - {200, <<"ok">>} + License = maps:from_list(emqx_license_checker:dump()), + {200, License} end; '/license/upload'(post, #{body := #{<<"key">> := Key}}) -> case emqx_license:update_key(Key) of @@ -133,10 +154,11 @@ schema("/license/upload") -> msg => "bad_license_key", reason => Error }), - {400, <<"bad request">>}; + {400, error_msg(?BAD_REQUEST, <<"Bad license key">>)}; {ok, _} -> ?SLOG(info, #{msg => "updated_license_key"}), - {200, <<"ok">>} + License = maps:from_list(emqx_license_checker:dump()), + {200, License} end; '/license/upload'(post, _Params) -> - {400, <<"bad request">>}. + {400, error_msg(?BAD_REQUEST, <<"Invalid request params">>)}. diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 06bf35867..cb34f8f50 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -117,13 +117,26 @@ t_license_upload_file_success(_Config) -> Path = "/tmp/new.lic", ok = file:write_file(Path, NewKey), try + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, ?assertEqual( - {ok, 200, <<"ok">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 999, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -136,13 +149,20 @@ t_license_upload_file_success(_Config) -> end. t_license_upload_file_not_found(_Config) -> + Res = request( + post, + uri(["license", "upload"]), + #{file => "/tmp/inexistent.lic"} + ), + + ?assertMatch({ok, 404, _}, Res), + {ok, 404, Payload} = Res, ?assertEqual( - {ok, 404, <<"file not found">>}, - request( - post, - uri(["license", "upload"]), - #{file => "/tmp/inexistent.lic"} - ) + #{ + <<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"File not found">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -150,13 +170,19 @@ t_license_upload_file_not_found(_Config) -> t_license_upload_file_reading_error(_Config) -> %% eisdir Path = "/tmp/", + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Illegal operation on a directory">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -165,13 +191,19 @@ t_license_upload_file_bad_license(_Config) -> Path = "/tmp/bad.lic", ok = file:write_file(Path, <<"bad key">>), try + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Bad license file">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok @@ -182,13 +214,26 @@ t_license_upload_file_bad_license(_Config) -> t_license_upload_key_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + Res = request( + post, + uri(["license", "upload"]), + #{key => NewKey} + ), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, ?assertEqual( - {ok, 200, <<"ok">>}, - request( - post, - uri(["license", "upload"]), - #{key => NewKey} - ) + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 999, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -198,13 +243,19 @@ t_license_upload_key_success(_Config) -> t_license_upload_key_bad_key(_Config) -> BadKey = <<"bad key">>, + Res = request( + post, + uri(["license", "upload"]), + #{key => BadKey} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{key => BadKey} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Bad license key">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. From 889829e47d7647d174cabfae80dda67c4b11a769 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 08:57:00 -0300 Subject: [PATCH 100/105] docs: use better translation --- lib-ee/emqx_license/i18n/emqx_license_http_api.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf index 6b2c7a687..67ec1381f 100644 --- a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf +++ b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf @@ -13,7 +13,7 @@ emqx_license_http_api { desc_license_upload_api { desc { en: "Upload a license file or key" - zh: "上传许可证文件或钥匙" + zh: "上传许可证文件或密钥" } label: { en: "Update license" From f0f1a7ad05929fb4b82292c892b71a14f338a525 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 09:34:56 -0300 Subject: [PATCH 101/105] chore: update changelog --- CHANGES-5.0.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 4dd74823c..1f237c6b1 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -1,3 +1,14 @@ +# 5.0.5 + +## Bug fixes + +* Allow changing the license type from key to file (and vice-versa). [#8598](https://github.com/emqx/emqx/pull/8598) + +## Enhancements + +* The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) +* Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) + # 5.0.4 ## Bug fixes From 4869225395951b8a16b120d0066abad6bc2999f9 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:24:03 +0800 Subject: [PATCH 102/105] chore: update Running/Stopped to running/stopped --- CHANGES-5.0.md | 1 + apps/emqx_management/src/emqx_mgmt.erl | 4 ++-- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 11 +++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 1f237c6b1..8f5592d4a 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -8,6 +8,7 @@ * The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) * Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) +* Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) # 5.0.4 diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index cdf3bf504..8f73d5767 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -138,7 +138,7 @@ node_info() -> max_fds, lists:usort(lists:flatten(erlang:system_info(check_io))) ), connections => ets:info(emqx_channel, size), - node_status => 'Running', + node_status => 'running', uptime => proplists:get_value(uptime, BrokerInfo), version => iolist_to_binary(proplists:get_value(version, BrokerInfo)), role => mria_rlog:role() @@ -156,7 +156,7 @@ node_info(Node) -> wrap_rpc(emqx_management_proto_v2:node_info(Node)). stopped_node_info(Node) -> - #{name => Node, node_status => 'Stopped'}. + #{name => Node, node_status => 'stopped'}. %%-------------------------------------------------------------------- %% Brokers diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index f1731db4d..dda82aeb3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -189,8 +189,8 @@ fields(node_info) -> )}, {node_status, mk( - enum(['Running', 'Stopped']), - #{desc => <<"Node status">>, example => "Running"} + enum(['running', 'stopped']), + #{desc => <<"Node status">>, example => "running"} )}, {otp_release, mk( @@ -288,19 +288,18 @@ get_stats(Node) -> %% internal function format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - {ok, SysPathBinary} = file:get_cwd(), - SysPath = list_to_binary(SysPathBinary), + RootDir = code:root_dir(), LogPath = case log_path() of undefined -> <<"log.file_handler.default.enable is false,only log to console">>; Path -> - filename:join(SysPath, Path) + filename:join(RootDir, Path) end, Info#{ memory_total := emqx_mgmt_util:kmg(Total), memory_used := emqx_mgmt_util:kmg(Used), - sys_path => SysPath, + sys_path => RootDir, log_path => LogPath }. From a19514f0009d989fd8aa5b8846d393d780e91e39 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:34:30 +0800 Subject: [PATCH 103/105] chore: bump emqx_management to 5.0.3 --- apps/emqx_management/src/emqx_management.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index e5b769b6f..9de47ca50 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, From fd7c97735dc22145ce0c2d40e5eaca7add89ba67 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:58:20 +0800 Subject: [PATCH 104/105] chore: make sure path is binary --- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index dda82aeb3..e0f0912df 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -288,7 +288,7 @@ get_stats(Node) -> %% internal function format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - RootDir = code:root_dir(), + RootDir = list_to_binary(code:root_dir()), LogPath = case log_path() of undefined -> From abcdad39e571728d5f2935879a7a53f65385017b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 16:29:06 +0800 Subject: [PATCH 105/105] chore: bump fvt_test's minirest to latest(1.3.6) --- .ci/fvt_tests/http_server/README.md | 1 - .ci/fvt_tests/http_server/rebar.config | 2 +- .../http_server/src/http_server.app.src | 4 +- .ci/fvt_tests/http_server/src/http_server.erl | 130 +++++++++++++----- 4 files changed, 96 insertions(+), 41 deletions(-) diff --git a/.ci/fvt_tests/http_server/README.md b/.ci/fvt_tests/http_server/README.md index ea14939b3..93619927f 100644 --- a/.ci/fvt_tests/http_server/README.md +++ b/.ci/fvt_tests/http_server/README.md @@ -27,4 +27,3 @@ ok + POST `/counter` 计数器加一 - diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 4085df159..8ddb3a7ab 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ - {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "1.3.6"}}} ]}. {shell, [ diff --git a/.ci/fvt_tests/http_server/src/http_server.app.src b/.ci/fvt_tests/http_server/src/http_server.app.src index 420aff9d3..6be2382b6 100644 --- a/.ci/fvt_tests/http_server/src/http_server.app.src +++ b/.ci/fvt_tests/http_server/src/http_server.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, http_server, - [{description, "An OTP application"}, - {vsn, "0.1.0"}, + [{description, "An HTTP server application"}, + {vsn, "0.2.0"}, {registered, []}, % {mod, {http_server_app, []}}, {modules, []}, diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl index 4aaa25b95..c7097854e 100644 --- a/.ci/fvt_tests/http_server/src/http_server.erl +++ b/.ci/fvt_tests/http_server/src/http_server.erl @@ -10,51 +10,107 @@ stop/0 ]). --rest_api(#{ - name => get_counter, - method => 'GET', - path => "/counter", - func => get_counter, - descr => "Check counter" -}). --rest_api(#{ - name => add_counter, - method => 'POST', - path => "/counter", - func => add_counter, - descr => "Counter plus one" -}). +-behavior(minirest_api). --export([ - get_counter/2, - add_counter/2 -]). +-export([api_spec/0]). +-export([counter/2]). + +api_spec() -> + { + [counter_api()], + [] + }. + +counter_api() -> + MetaData = #{ + get => #{ + description => "Get counter", + summary => "Get counter", + responses => #{ + 200 => #{ + content => #{ + 'application/json' => + #{ + type => object, + properties => #{ + code => #{type => integer, example => 0}, + data => #{type => integer, example => 0} + } + } + } + } + } + }, + post => #{ + description => "Add counter", + summary => "Add counter", + 'requestBody' => #{ + content => #{ + 'application/json' => #{ + schema => + #{ + type => object, + properties => #{ + payload => #{type => string, example => <<"sample payload">>}, + id => #{type => integer, example => 0} + } + } + } + } + }, + responses => #{ + 200 => #{ + content => #{ + 'application/json' => + #{ + type => object, + properties => #{ + code => #{type => integer, example => 0} + } + } + } + } + } + } + }, + {"/counter", MetaData, counter}. + +counter(get, _Params) -> + V = ets:info(relup_test_message, size), + {200, #{<<"content-type">> => <<"text/plain">>}, #{<<"code">> => 0, <<"data">> => V}}; +counter(post, #{body := Params}) -> + case Params of + #{<<"payload">> := _, <<"id">> := Id} -> + ets:insert(relup_test_message, {Id, maps:remove(<<"id">>, Params)}), + {200, #{<<"code">> => 0}}; + _ -> + io:format("discarded: ~p\n", [Params]), + {200, #{<<"code">> => -1}} + end. start() -> application:ensure_all_started(minirest), _ = spawn(fun ets_owner/0), - Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}], - Dispatch = [{"/[...]", minirest, Handlers}], - minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 7077}]}, Dispatch). + RanchOptions = #{ + max_connections => 512, + num_acceptors => 4, + socket_opts => [{send_timeout, 5000}, {port, 7077}, {backlog, 512}] + }, + Minirest = #{ + base_path => "", + modules => [?MODULE], + dispatch => [{"/[...]", ?MODULE, []}], + protocol => http, + ranch_options => RanchOptions, + middlewares => [cowboy_router, cowboy_handler] + }, + Res = minirest:start(?MODULE, Minirest), + minirest:update_dispatch(?MODULE), + Res. stop() -> ets:delete(relup_test_message), - minirest:stop_http(?MODULE). - -get_counter(_Binding, _Params) -> - V = ets:info(relup_test_message, size), - return({ok, V}). - -add_counter(_Binding, Params) -> - case lists:keymember(<<"payload">>, 1, Params) of - true -> - {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params), - ets:insert(relup_test_message, {ID, Params1}); - _ -> - io:format("discarded: ~p\n", [Params]), - ok - end, - return(). + minirest:stop(?MODULE). ets_owner() -> ets:new(relup_test_message, [named_table, public]),