diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 716c4c709..9662b174d 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -54,7 +54,7 @@ services: KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, + KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD: password @@ -66,8 +66,8 @@ services: volumes: - emqx-shared-secret:/var/lib/secret - ./kafka/jaas.conf:/etc/kafka/jaas.conf - - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh + - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf - command: run_add_scram_users.sh + command: kafka-entrypoint.sh diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 66e7ec308..ce4f28ba7 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -15,6 +15,8 @@ services: - 8087:8087 - 13306:3306 - 13307:3307 + - 15432:5432 + - 15433:5433 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh similarity index 83% rename from .ci/docker-compose-file/kafka/run_add_scram_users.sh rename to .ci/docker-compose-file/kafka/kafka-entrypoint.sh index 4b51fee0d..445fd65c9 100755 --- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh +++ b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh @@ -22,6 +22,7 @@ sleep 3 echo "+++++++ Starting Kafka ++++++++" +# fork start Kafka start-kafka.sh & SERVER=localhost @@ -41,6 +42,12 @@ echo "+++++++ Run config commands ++++++++" kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=password],SCRAM-SHA-512=[password=password]' --entity-type users --entity-name emqxuser +echo "+++++++ Creating Kafka Topics ++++++++" + +# create topics after re-configuration +# there seem to be a race condition when creating the topics (too early) +env KAFKA_CREATE_TOPICS="$KAFKA_CREATE_TOPICS_NG" KAFKA_PORT="$PORT1" create-topics.sh + echo "+++++++ Wait until Kafka ports are down ++++++++" bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1 diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 34bc5b1db..f4b11116b 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -29,5 +29,17 @@ "listen": "0.0.0.0:6379", "upstream": "redis:6379", "enabled": true + }, + { + "name": "pgsql_tcp", + "listen": "0.0.0.0:5432", + "upstream": "pgsql:5432", + "enabled": true + }, + { + "name": "pgsql_tls", + "listen": "0.0.0.0:5433", + "upstream": "pgsql-tls:5432", + "enabled": true } ] diff --git a/.editorconfig b/.editorconfig index c563aa10d..719028b4d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,8 +20,3 @@ indent_size = 4 # Tab indentation (no size specified) [Makefile] indent_style = tab - -# Matches the exact files either package.json or .travis.yml -[{.travis.yml}] -indent_style = space -indent_size = 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ec5efa18..a74fcb27e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,42 +1,38 @@ -## MQTT -/apps/emqx_connector/src/mqtt/ @qzhuyan -/apps/emqx/*/*mqtt* @qzhuyan +## Default +* @emqx/emqx-review-board ## apps -/apps/emqx/ @lafirest @thalesmg @HJianBo @ieQu1 -/apps/emqx_authn/ @savonarola @JimMoen @HJianBo -/apps/emqx_authz/ @savonarola @JimMoen @HJianBo -/apps/emqx_auto_subscribe/ @thalesmg @HJianBo -/apps/emqx_bridge/ @terry-xiaoyu @thalesmg -/apps/emqx_conf/ @ieQu1 @thalesmg -/apps/emqx_connector/ @terry-xiaoyu @JimMoen -/apps/emqx_dashboard/ @lafirest @JimMoen -/apps/emqx_exhook/ @lafirest @HJianBo @JimMoen -/apps/emqx_gateway/ @HJianBo @lafirest -/apps/emqx_machine/ @thalesmg @terry-xiaoyu @ieQu1 -/apps/emqx_management/ @HJianBo @lafirest @sstrigler -/apps/emqx_modules/ @thalesmg @terry-xiaoyu @HJianBo -/apps/emqx_plugin_libs/ @terry-xiaoyu @lafirest -/apps/emqx_plugins/ @thalesmg @JimMoen @ieQu1 -/apps/emqx_prometheus/ @JimMoen @ieQu1 -/apps/emqx_psk/ @lafirest @thalesmg @terry-xiaoyu -/apps/emqx_resource/ @terry-xiaoyu @thalesmg -/apps/emqx_retainer/ @lafirest @ieQu1 @thalesmg -/apps/emqx_rule_engine/ @terry-xiaoyu @HJianBo @kjellwinblad -/apps/emqx_slow_subs/ @lafirest @HJianBo -/apps/emqx_statsd/ @JimMoen @HJianBo +/apps/emqx/ @emqx/emqx-review-board @lafirest @thalesmg +/apps/emqx_authn/ @emqx/emqx-review-board @JimMoen @savonarola +/apps/emqx_authz/ @emqx/emqx-review-board @JimMoen @savonarola +/apps/emqx_auto_subscribe/ @emqx/emqx-review-board @thalesmg +/apps/emqx_bridge/ @emqx/emqx-review-board @thalesmg +/apps/emqx_conf/ @emqx/emqx-review-board @thalesmg +/apps/emqx_connector/ @emqx/emqx-review-board @JimMoen +/apps/emqx_dashboard/ @emqx/emqx-review-board @JimMoen @lafirest +/apps/emqx_exhook/ @emqx/emqx-review-board @JimMoen @lafirest +/apps/emqx_gateway/ @emqx/emqx-review-board @lafirest +/apps/emqx_machine/ @emqx/emqx-review-board @thalesmg +/apps/emqx_management/ @emqx/emqx-review-board @lafirest @sstrigler +/apps/emqx_modules/ @emqx/emqx-review-board @thalesmg +/apps/emqx_plugin_libs/ @emqx/emqx-review-board @lafirest +/apps/emqx_plugins/ @emqx/emqx-review-board @JimMoen @thalesmg +/apps/emqx_prometheus/ @emqx/emqx-review-board @JimMoen +/apps/emqx_psk/ @emqx/emqx-review-board @lafirest @thalesmg +/apps/emqx_resource/ @emqx/emqx-review-board @thalesmg +/apps/emqx_retainer/ @emqx/emqx-review-board @lafirest @thalesmg +/apps/emqx_rule_engine/ @emqx/emqx-review-board @kjellwinblad +/apps/emqx_slow_subs/ @emqx/emqx-review-board @lafirest +/apps/emqx_statsd/ @emqx/emqx-review-board @JimMoen ## other -/lib-ee/ @thalesmg -/bin/ @zmstone @thalesmg @terry-xiaoyu @id -/rel/ @zmstone @thalesmg @id +/lib-ee/ @emqx/emqx-review-board @thalesmg +/bin/ @emqx/emqx-review-board @thalesmg @id +/rel/ @emqx/emqx-review-board @thalesmg @id ## CI -/.github/ @id -/.ci/ @id -/scripts/ @id -/build @id -/deploy/ @id - -## Default -* @zmstone +/.github/ @emqx/emqx-review-board @id +/.ci/ @emqx/emqx-review-board @id +/scripts/ @emqx/emqx-review-board @id +/build @emqx/emqx-review-board @id +/deploy/ @emqx/emqx-review-board @id diff --git a/.github/PULL_REQUEST_TEMPLATE/v5.md b/.github/PULL_REQUEST_TEMPLATE/v5.md index a0db01e1a..d4104ca5d 100644 --- a/.github/PULL_REQUEST_TEMPLATE/v5.md +++ b/.github/PULL_REQUEST_TEMPLATE/v5.md @@ -5,7 +5,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] Added tests for the changes - [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes/` dir +- [ ] Change log has been added to `changes//(feat|fix)-.en.md` and `.zh.md` files - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 7fce82cf8..36deef717 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -127,9 +127,18 @@ jobs: ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall echo "EMQX uninstalled" - uses: actions/upload-artifact@v3 + if: success() with: name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ + - name: Send notification to Slack + uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.event_name == 'schedule' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + payload: | + {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} mac: needs: prepare @@ -165,9 +174,18 @@ jobs: apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v3 + if: success() with: name: ${{ matrix.profile }} path: _packages/${{ matrix.profile }}/ + - name: Send notification to Slack + uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.event_name == 'schedule' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + payload: | + {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} linux: needs: prepare @@ -215,7 +233,7 @@ jobs: elixir: 1.13.4 release_with: elixir - profile: emqx - otp: 24.3.4.2-1 # TODO: 25.1.2-2 + otp: 25.1.2-2 arch: amd64 os: amzn2 build_machine: ubuntu-20.04 @@ -271,9 +289,18 @@ jobs: --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v3 + if: success() with: name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ + - name: Send notification to Slack + uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.event_name == 'schedule' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + payload: | + {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} publish_artifacts: runs-on: ubuntu-20.04 diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 4ee8efa2e..91a087c95 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -162,7 +162,7 @@ jobs: INFLUXDB_TAG: 2.5.0 PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - run: ./scripts/ct/run.sh --app ${{ matrix.app }} + run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} - uses: actions/upload-artifact@v3 with: name: coverdata @@ -170,7 +170,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.profile }}-${{ matrix.prefix }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }} path: source/_build/test/logs ct: @@ -213,7 +213,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.profile }}-${{ matrix.prefix }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }} path: source/_build/test/logs make_cover: diff --git a/.gitignore b/.gitignore index d01c764d0..4780aab38 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi *.conf.rendered lux_logs/ /.prepare +bom.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 118e9a046..272a602e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ Must be one of the following: - **chore**: Updating grunt tasks etc; no production code change - **perf**: A code change that improves performance - **test**: Adding missing tests, refactoring tests; no production code change -- **build**: Changes that affect the CI/CD pipeline or build system or external dependencies (example scopes: travis, jenkins, makefile) +- **build**: Changes that affect the CI/CD pipeline or build system or external dependencies (example scopes: jenkins, makefile) - **ci**: Changes provided by DevOps for CI purposes. - **revert**: Reverts a previous commit. diff --git a/Makefile b/Makefile index c2b33786b..180028899 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.4 +export EMQX_DASHBOARD_VERSION ?= v1.1.5 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.9 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 @@ -88,6 +88,7 @@ define gen-app-ct-target $1-ct: $(REBAR) @$(SCRIPTS)/pre-compile.sh $(PROFILE) @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ + --readable=$(CT_READABLE) \ --name $(CT_NODE_NAME) \ --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ --suite $(shell $(SCRIPTS)/find-suites.sh $1) diff --git a/NOTICE b/NOTICE index 613c5d83a..f5991f663 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ EMQX, highly scalable, highly available distributed MQTT messaging platform for IoT. -Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. This product contains code developed at EMQ Technologies Co., Ltd. Visit https://www.emqx.come to learn more. diff --git a/README-CN.md b/README-CN.md index 7e8cdd8a7..193e5ab98 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,7 +1,7 @@ # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) diff --git a/README-RU.md b/README-RU.md index 8a35177af..fb5ff9608 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,7 +1,7 @@ # Брокер EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) diff --git a/README.md b/README.md index 1831ced60..94baba04f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) diff --git a/apps/emqx/NOTICE b/apps/emqx/NOTICE index 613c5d83a..f5991f663 100644 --- a/apps/emqx/NOTICE +++ b/apps/emqx/NOTICE @@ -1,5 +1,5 @@ EMQX, highly scalable, highly available distributed MQTT messaging platform for IoT. -Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. This product contains code developed at EMQ Technologies Co., Ltd. Visit https://www.emqx.come to learn more. diff --git a/apps/emqx/include/bpapi.hrl b/apps/emqx/include/bpapi.hrl index b4e52d50f..1373e0381 100644 --- a/apps/emqx/include/bpapi.hrl +++ b/apps/emqx/include/bpapi.hrl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -ifndef(EMQX_BPAPI_HRL). -define(EMQX_BPAPI_HRL, true). diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index b1b1e2a00..64cd4687b 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index 4d5ba8179..693bc91b5 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl index 6187b136c..2b1f4d33f 100644 --- a/apps/emqx/include/emqx_authentication.hrl +++ b/apps/emqx/include/emqx_authentication.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_hooks.hrl b/apps/emqx/include/emqx_hooks.hrl index 3785a13de..1665492c5 100644 --- a/apps/emqx/include/emqx_hooks.hrl +++ b/apps/emqx/include/emqx_hooks.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 537e5a85a..e4f9111da 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index db7576b67..c4d235caf 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index cea7f076c..e749ebc2e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.13"). +-define(EMQX_RELEASE_CE, "5.0.14"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-beta.6"). diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 7cd16d338..858ce96ce 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index d321330a8..e93aa46f4 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/include/types.hrl b/apps/emqx/include/types.hrl index a36ad28b2..ec56a9300 100644 --- a/apps/emqx/include/types.hrl +++ b/apps/emqx/include/types.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8ff4fea58..82be1cb9a 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/bpapi/emqx_bpapi.erl b/apps/emqx/src/bpapi/emqx_bpapi.erl index ce6ad5cb1..3d58cd754 100644 --- a/apps/emqx/src/bpapi/emqx_bpapi.erl +++ b/apps/emqx/src/bpapi/emqx_bpapi.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/bpapi/emqx_bpapi.hrl b/apps/emqx/src/bpapi/emqx_bpapi.hrl index b2d2d218f..872f320dd 100644 --- a/apps/emqx/src/bpapi/emqx_bpapi.hrl +++ b/apps/emqx/src/bpapi/emqx_bpapi.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/bpapi/emqx_bpapi_trans.erl b/apps/emqx/src/bpapi/emqx_bpapi_trans.erl index b1e9fb5f2..06c96638c 100644 --- a/apps/emqx/src/bpapi/emqx_bpapi_trans.erl +++ b/apps/emqx/src/bpapi/emqx_bpapi_trans.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 70b8ef473..babf9c431 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index c281b11cc..bd7617e74 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index ed246cff5..6e4aa9922 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 3fa781e6d..efe9bee37 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 7368b442d..6122ff596 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl index b5967a21d..8b0bf90b4 100644 --- a/apps/emqx/src/emqx_alarm_handler.erl +++ b/apps/emqx/src/emqx_alarm_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index b6194eb31..6188d8030 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index ffce81787..017bc982d 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index b867800ae..98471e152 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_authentication_sup.erl b/apps/emqx/src/emqx_authentication_sup.erl index 5c2a554ec..d28101fe7 100644 --- a/apps/emqx/src/emqx_authentication_sup.erl +++ b/apps/emqx/src/emqx_authentication_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_authn_authz_metrics_sup.erl b/apps/emqx/src/emqx_authn_authz_metrics_sup.erl index 8fd55108b..46f9a04ed 100644 --- a/apps/emqx/src/emqx_authn_authz_metrics_sup.erl +++ b/apps/emqx/src/emqx_authn_authz_metrics_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_authz_cache.erl b/apps/emqx/src/emqx_authz_cache.erl index 668435d36..6555266a5 100644 --- a/apps/emqx/src/emqx_authz_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 3b5bf18f6..758c570da 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_base62.erl b/apps/emqx/src/emqx_base62.erl index 059b20d52..30e9c1050 100644 --- a/apps/emqx/src/emqx_base62.erl +++ b/apps/emqx/src/emqx_base62.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl index a287604c4..2fe09942c 100644 --- a/apps/emqx/src/emqx_batch.erl +++ b/apps/emqx/src/emqx_batch.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_boot.erl b/apps/emqx/src/emqx_boot.erl index 7bd18e22f..6cbac558f 100644 --- a/apps/emqx/src/emqx_boot.erl +++ b/apps/emqx/src/emqx_boot.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 2e0a69dba..7b8e3dddd 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_broker_bench.erl b/apps/emqx/src/emqx_broker_bench.erl index 18f74a523..9855df702 100644 --- a/apps/emqx/src/emqx_broker_bench.erl +++ b/apps/emqx/src/emqx_broker_bench.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 72a44f711..91b4c4994 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index 63c39d229..a43ee771f 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index b2d44847e..b6be52c6e 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 5adc7a811..66e9a2aee 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -1,5 +1,5 @@ %%------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_cm.hrl b/apps/emqx/src/emqx_cm.hrl index ec23e21e9..3896ea30a 100644 --- a/apps/emqx/src/emqx_cm.hrl +++ b/apps/emqx/src/emqx_cm.hrl @@ -1,5 +1,5 @@ %%------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl index 57569fdee..a7dfa2761 100644 --- a/apps/emqx/src/emqx_cm_locker.erl +++ b/apps/emqx/src/emqx_cm_locker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl index ebd4b2977..8a5639486 100644 --- a/apps/emqx/src/emqx_cm_registry.erl +++ b/apps/emqx/src/emqx_cm_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index c1e945ed7..221e78c40 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f5ff64d4d..2fa39d094 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -362,8 +362,8 @@ schema_default(Schema) -> []; ?LAZY(?ARRAY(_)) -> []; - ?LAZY(?UNION(Unions)) -> - case [A || ?ARRAY(A) <- Unions] of + ?LAZY(?UNION(Members)) -> + case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of [_ | _] -> []; _ -> #{} end; @@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) -> required => false, format => map, apply_override_envs => true, - remove_env_meta => true, check_lazy => true }, hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts). @@ -413,6 +412,31 @@ check_config(SchemaMod, RawConf) -> check_config(SchemaMod, RawConf, #{}). check_config(SchemaMod, RawConf, Opts0) -> + try + do_check_config(SchemaMod, RawConf, Opts0) + catch + throw:{Schema, Errors} -> + compact_errors(Schema, Errors) + end. + +%% HOCON tries to be very informative about all the detailed errors +%% it's maybe too much when reporting to the user +-spec compact_errors(any(), any()) -> no_return(). +compact_errors(Schema, [Error0 | More]) when is_map(Error0) -> + Error1 = Error0#{discarded_errors_count => length(More)}, + Error = + case is_atom(Schema) of + true -> + Error1#{schema_module => Schema}; + false -> + Error1 + end, + throw(Error); +compact_errors(Schema, Errors) -> + %% unexpected, we need the stacktrace reported, hence error + error({Schema, Errors}). + +do_check_config(SchemaMod, RawConf, Opts0) -> Opts1 = #{ return_plain => true, format => map, diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 0311418a9..a0a99b62e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -245,7 +245,7 @@ process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> BinKeyPath = bin_path(ConfKeyPath), case check_permissions(update, BinKeyPath, NewRawConf, Opts) of allow -> - OverrideConf = update_override_config(NewRawConf, Opts), + OverrideConf = merge_to_override_config(NewRawConf, Opts), {ok, NewRawConf, OverrideConf, Opts}; {deny, Reason} -> {error, {permission_denied, Reason}} @@ -447,9 +447,10 @@ remove_from_override_config(BinKeyPath, Opts) -> OldConf = emqx_config:read_override_conf(Opts), emqx_map_lib:deep_remove(BinKeyPath, OldConf). -update_override_config(_RawConf, #{persistent := false}) -> +%% apply new config on top of override config +merge_to_override_config(_RawConf, #{persistent := false}) -> undefined; -update_override_config(RawConf, Opts) -> +merge_to_override_config(RawConf, Opts) -> OldConf = emqx_config:read_override_conf(Opts), maps:merge(OldConf, RawConf). diff --git a/apps/emqx/src/emqx_congestion.erl b/apps/emqx/src/emqx_congestion.erl index 1e25ab391..11ca0327b 100644 --- a/apps/emqx/src/emqx_congestion.erl +++ b/apps/emqx/src/emqx_congestion.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e0e86c145..0c6481399 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl index 1087d9282..53eb5b888 100644 --- a/apps/emqx/src/emqx_ctl.erl +++ b/apps/emqx/src/emqx_ctl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_datetime.erl b/apps/emqx/src/emqx_datetime.erl index a33ba537f..60f40130b 100644 --- a/apps/emqx/src/emqx_datetime.erl +++ b/apps/emqx/src/emqx_datetime.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_exclusive_subscription.erl b/apps/emqx/src/emqx_exclusive_subscription.erl index 7a2d65472..afb6317b7 100644 --- a/apps/emqx/src/emqx_exclusive_subscription.erl +++ b/apps/emqx/src/emqx_exclusive_subscription.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 7e72c488f..64e4ed6c3 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 10f0aa135..5999e93f3 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_gc.erl b/apps/emqx/src/emqx_gc.erl index 53ba53d22..61087ba29 100644 --- a/apps/emqx/src/emqx_gc.erl +++ b/apps/emqx/src/emqx_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl index 9b51e92c8..fea4e70b0 100644 --- a/apps/emqx/src/emqx_guid.erl +++ b/apps/emqx/src/emqx_guid.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 7af9020dd..7e9dbca77 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index b07353302..1784d8ea3 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_inflight.erl b/apps/emqx/src/emqx_inflight.erl index 1aa0ed763..a3ff0ab10 100644 --- a/apps/emqx/src/emqx_inflight.erl +++ b/apps/emqx/src/emqx_inflight.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_json.erl b/apps/emqx/src/emqx_json.erl index 4be39152c..7827b98c9 100644 --- a/apps/emqx/src/emqx_json.erl +++ b/apps/emqx/src/emqx_json.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_keepalive.erl b/apps/emqx/src/emqx_keepalive.erl index 3b3a3e46b..9ba11e23f 100644 --- a/apps/emqx/src/emqx_keepalive.erl +++ b/apps/emqx/src/emqx_keepalive.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl index 3c8b5d1d3..21ed8576a 100644 --- a/apps/emqx/src/emqx_kernel_sup.erl +++ b/apps/emqx/src/emqx_kernel_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl index 16f7b03c8..8b2831766 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index d7a959e40..83bc2ec72 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl index 71a1ff315..cefd94881 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl index ad9f6a7cc..fe30e41e9 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index 74b6c7b87..ea02152a9 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_correction.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_correction.erl index 5dda06ba3..013c23e61 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_correction.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_correction.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl index b4c0839fe..33ba0e511 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index a6ca9f2c8..297bdffb0 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index bce87e2ba..fa67e1977 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index c5e919296..44663ceeb 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl index d75bf1082..cba11ede2 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl index a8808151e..df594c0bf 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4a6ea2046..fb6096e80 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl index f780f2217..114a1af49 100644 --- a/apps/emqx/src/emqx_logger.erl +++ b/apps/emqx/src/emqx_logger.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 03adeed04..22cf75153 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -221,7 +221,7 @@ best_effort_json_obj(Map, Config) -> end. json([], _) -> - "[]"; + ""; json(<<"">>, _) -> "\"\""; json(A, _) when is_atom(A) -> atom_to_binary(A, utf8); @@ -376,4 +376,19 @@ p_config() -> ] ). +best_effort_json_test() -> + ?assertEqual( + <<"{}">>, + emqx_logger_jsonfmt:best_effort_json([]) + ), + ?assertEqual( + <<"{\n \"key\": []\n}">>, + emqx_logger_jsonfmt:best_effort_json(#{key => []}) + ), + ?assertEqual( + <<"[\n {\n \"key\": []\n }\n]">>, + emqx_logger_jsonfmt:best_effort_json([#{key => []}]) + ), + ok. + -endif. diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl index 0bb4f30c6..3695929d9 100644 --- a/apps/emqx/src/emqx_logger_textfmt.erl +++ b/apps/emqx/src/emqx_logger_textfmt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index 6484d4269..5455fe9e7 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index 03f7ca6a2..509d4c90d 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 7e841a2cf..c2e297623 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_metrics_worker.erl b/apps/emqx/src/emqx_metrics_worker.erl index ab6a0b1a6..5f41346cb 100644 --- a/apps/emqx/src/emqx_metrics_worker.erl +++ b/apps/emqx/src/emqx_metrics_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ -behaviour(gen_server). +-include_lib("stdlib/include/ms_transform.hrl"). + %% API functions -export([ start_link/1, @@ -29,8 +31,16 @@ -export([ inc/3, inc/4, + observe/4, get/3, + get_gauge/3, + set_gauge/5, + shift_gauge/5, + get_gauges/2, + delete_gauges/2, get_rate/2, + get_slide/2, + get_slide/3, get_counters/2, create_metrics/3, create_metrics/4, @@ -60,7 +70,16 @@ -define(SAMPLING, 1). -endif. --export_type([metrics/0, handler_name/0, metric_id/0]). +-export_type([metrics/0, handler_name/0, metric_id/0, metric_spec/0]). + +% Default +-type metric_type() :: + %% Simple counter + counter + %% Sliding window average + | slide. + +-type metric_spec() :: {metric_type(), atom()}. -type rate() :: #{ current := float(), @@ -68,14 +87,22 @@ last5m := float() }. -type metrics() :: #{ - counters := #{atom() => integer()}, - rate := #{atom() => rate()} + counters := #{metric_name() => integer()}, + gauges := #{metric_name() => integer()}, + slides := #{metric_name() => number()}, + rate := #{metric_name() => rate()} }. -type handler_name() :: atom(). +%% metric_id() is actually a resource id -type metric_id() :: binary() | atom(). +-type metric_name() :: atom(). +-type worker_id() :: term(). -define(CntrRef(Name), {?MODULE, Name}). -define(SAMPCOUNT_5M, (?SECS_5M div ?SAMPLING)). +-define(GAUGE_TABLE(NAME), + list_to_atom(atom_to_list(?MODULE) ++ "_" ++ atom_to_list(NAME) ++ "_gauge") +). -record(rate, { max = 0 :: number(), @@ -89,9 +116,22 @@ last5m_smpl = [] :: list() }). +-record(slide_datapoint, { + sum :: non_neg_integer(), + samples :: non_neg_integer(), + time :: non_neg_integer() +}). + +-record(slide, { + %% Total number of samples through the history + n_samples = 0 :: non_neg_integer(), + datapoints = [] :: [#slide_datapoint{}] +}). + -record(state, { metric_ids = sets:new(), - rates :: undefined | #{metric_id() => #rate{}} + rates :: #{metric_id() => #{metric_name() => #rate{}}} | undefined, + slides = #{} :: #{metric_id() => #{metric_name() => #slide{}}} }). %%------------------------------------------------------------------------------ @@ -112,13 +152,18 @@ child_spec(ChldName, Name) -> modules => [emqx_metrics_worker] }. --spec create_metrics(handler_name(), metric_id(), [atom()]) -> ok | {error, term()}. +-spec create_metrics(handler_name(), metric_id(), [metric_spec() | metric_name()]) -> + ok | {error, term()}. create_metrics(Name, Id, Metrics) -> - create_metrics(Name, Id, Metrics, Metrics). + Metrics1 = desugar(Metrics), + Counters = filter_counters(Metrics1), + create_metrics(Name, Id, Metrics1, Counters). --spec create_metrics(handler_name(), metric_id(), [atom()], [atom()]) -> ok | {error, term()}. +-spec create_metrics(handler_name(), metric_id(), [metric_spec() | metric_name()], [atom()]) -> + ok | {error, term()}. create_metrics(Name, Id, Metrics, RateMetrics) -> - gen_server:call(Name, {create_metrics, Id, Metrics, RateMetrics}). + Metrics1 = desugar(Metrics), + gen_server:call(Name, {create_metrics, Id, Metrics1, RateMetrics}). -spec clear_metrics(handler_name(), metric_id()) -> ok. clear_metrics(Name, Id) -> @@ -135,13 +180,13 @@ has_metrics(Name, Id) -> _ -> true end. --spec get(handler_name(), metric_id(), atom() | integer()) -> number(). +-spec get(handler_name(), metric_id(), metric_name() | integer()) -> number(). get(Name, Id, Metric) -> case get_ref(Name, Id) of not_found -> 0; Ref when is_atom(Metric) -> - counters:get(Ref, idx_metric(Name, Id, Metric)); + counters:get(Ref, idx_metric(Name, Id, counter, Metric)); Ref when is_integer(Metric) -> counters:get(Ref, Metric) end. @@ -156,26 +201,158 @@ get_counters(Name, Id) -> fun(_Metric, Index) -> get(Name, Id, Index) end, - get_indexes(Name, Id) + get_indexes(Name, counter, Id) ). +-spec get_slide(handler_name(), metric_id()) -> map(). +get_slide(Name, Id) -> + gen_server:call(Name, {get_slide, Id}). + +%% Get the average for a specified sliding window period. +%% +%% It will only account for the samples recorded in the past `Window' seconds. +-spec get_slide(handler_name(), metric_id(), non_neg_integer()) -> number(). +get_slide(Name, Id, Window) -> + gen_server:call(Name, {get_slide, Id, Window}). + -spec reset_counters(handler_name(), metric_id()) -> ok. reset_counters(Name, Id) -> - Indexes = maps:values(get_indexes(Name, Id)), - Ref = get_ref(Name, Id), - lists:foreach(fun(Idx) -> counters:put(Ref, Idx, 0) end, Indexes). + case get_ref(Name, Id) of + not_found -> + ok; + Ref -> + #{size := Size} = counters:info(Ref), + lists:foreach(fun(Idx) -> counters:put(Ref, Idx, 0) end, lists:seq(1, Size)) + end. -spec get_metrics(handler_name(), metric_id()) -> metrics(). get_metrics(Name, Id) -> - #{rate => get_rate(Name, Id), counters => get_counters(Name, Id)}. + #{ + rate => get_rate(Name, Id), + counters => get_counters(Name, Id), + gauges => get_gauges(Name, Id), + slides => get_slide(Name, Id) + }. -spec inc(handler_name(), metric_id(), atom()) -> ok. inc(Name, Id, Metric) -> inc(Name, Id, Metric, 1). --spec inc(handler_name(), metric_id(), atom(), integer()) -> ok. +-spec inc(handler_name(), metric_id(), metric_name(), integer()) -> ok. inc(Name, Id, Metric, Val) -> - counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val). + counters:add(get_ref(Name, Id), idx_metric(Name, Id, counter, Metric), Val). + +%% Add a sample to the slide. +%% +%% Slide is short for "sliding window average" type of metric. +%% +%% It allows to monitor an average of some observed values in time, +%% and it's mainly used for performance analysis. For example, it can +%% be used to report run time of operations. +%% +%% Consider an example: +%% +%% ``` +%% emqx_metrics_worker:create_metrics(Name, Id, [{slide, a}]), +%% emqx_metrics_worker:observe(Name, Id, a, 10), +%% emqx_metrics_worker:observe(Name, Id, a, 30), +%% #{a := 20} = emqx_metrics_worker:get_slide(Name, Id, _Window = 1). +%% ''' +%% +%% After recording 2 samples, this metric becomes 20 (the average of 10 and 30). +%% +%% But after 1 second it becomes 0 again, unless new samples are recorded. +%% +-spec observe(handler_name(), metric_id(), atom(), integer()) -> ok. +observe(Name, Id, Metric, Val) -> + #{ref := CRef, slide := Idx} = maps:get(Id, get_pterm(Name)), + Index = maps:get(Metric, Idx), + %% Update sum: + counters:add(CRef, Index, Val), + %% Update number of samples: + counters:add(CRef, Index + 1, 1). + +-spec set_gauge(handler_name(), metric_id(), worker_id(), metric_name(), integer()) -> ok. +set_gauge(Name, Id, WorkerId, Metric, Val) -> + Table = ?GAUGE_TABLE(Name), + try + true = ets:insert(Table, {{Id, Metric, WorkerId}, Val}), + ok + catch + error:badarg -> + ok + end. + +-spec shift_gauge(handler_name(), metric_id(), worker_id(), metric_name(), integer()) -> ok. +shift_gauge(Name, Id, WorkerId, Metric, Val) -> + Table = ?GAUGE_TABLE(Name), + try + _ = ets:update_counter( + Table, + {Id, Metric, WorkerId}, + Val, + {{Id, Metric, WorkerId}, 0} + ), + ok + catch + error:badarg -> + ok + end. + +-spec get_gauge(handler_name(), metric_id(), metric_name()) -> integer(). +get_gauge(Name, Id, Metric) -> + Table = ?GAUGE_TABLE(Name), + MatchSpec = + ets:fun2ms( + fun({{Id0, Metric0, _WorkerId}, Val}) when Id0 =:= Id, Metric0 =:= Metric -> + Val + end + ), + try + lists:sum(ets:select(Table, MatchSpec)) + catch + error:badarg -> + 0 + end. + +-spec get_gauges(handler_name(), metric_id()) -> map(). +get_gauges(Name, Id) -> + Table = ?GAUGE_TABLE(Name), + MatchSpec = + ets:fun2ms( + fun({{Id0, Metric, _WorkerId}, Val}) when Id0 =:= Id -> + {Metric, Val} + end + ), + try + lists:foldr( + fun({Metric, Val}, Acc) -> + maps:update_with(Metric, fun(X) -> X + Val end, Val, Acc) + end, + #{}, + ets:select(Table, MatchSpec) + ) + catch + error:badarg -> + #{} + end. + +-spec delete_gauges(handler_name(), metric_id()) -> ok. +delete_gauges(Name, Id) -> + Table = ?GAUGE_TABLE(Name), + MatchSpec = + ets:fun2ms( + fun({{Id0, _Metric, _WorkerId}, _Val}) when Id0 =:= Id -> + true + end + ), + try + _ = ets:select_delete(Table, MatchSpec), + ok + catch + error:badarg -> + ok + end. start_link(Name) -> gen_server:start_link({local, Name}, ?MODULE, Name, []). @@ -185,6 +362,7 @@ init(Name) -> %% the rate metrics erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), persistent_term:put(?CntrRef(Name), #{}), + _ = ets:new(?GAUGE_TABLE(Name), [named_table, ordered_set, public, {write_concurrency, true}]), {ok, #state{}}. handle_call({get_rate, _Id}, _From, State = #state{rates = undefined}) -> @@ -198,9 +376,9 @@ handle_call({get_rate, Id}, _From, State = #state{rates = Rates}) -> handle_call( {create_metrics, Id, Metrics, RateMetrics}, _From, - State = #state{metric_ids = MIDs, rates = Rates} + State = #state{metric_ids = MIDs, rates = Rates, slides = Slides} ) -> - case RateMetrics -- Metrics of + case RateMetrics -- filter_counters(Metrics) of [] -> RatePerId = maps:from_list([{M, #rate{}} || M <- RateMetrics]), Rate1 = @@ -208,9 +386,11 @@ handle_call( undefined -> #{Id => RatePerId}; _ -> Rates#{Id => RatePerId} end, + Slides1 = Slides#{Id => create_slides(Metrics)}, {reply, create_counters(get_self_name(), Id, Metrics), State#state{ metric_ids = sets:add_element(Id, MIDs), - rates = Rate1 + rates = Rate1, + slides = Slides1 }}; _ -> {reply, {error, not_super_set_of, {RateMetrics, Metrics}}, State} @@ -218,35 +398,54 @@ handle_call( handle_call( {delete_metrics, Id}, _From, - State = #state{metric_ids = MIDs, rates = Rates} + State = #state{metric_ids = MIDs, rates = Rates, slides = Slides} ) -> - {reply, delete_counters(get_self_name(), Id), State#state{ + Name = get_self_name(), + delete_counters(Name, Id), + delete_gauges(Name, Id), + {reply, ok, State#state{ metric_ids = sets:del_element(Id, MIDs), rates = case Rates of undefined -> undefined; _ -> maps:remove(Id, Rates) - end + end, + slides = maps:remove(Id, Slides) }}; handle_call( {reset_metrics, Id}, _From, - State = #state{rates = Rates} + State = #state{rates = Rates, slides = Slides} ) -> + delete_gauges(get_self_name(), Id), + NewRates = + case Rates of + undefined -> + undefined; + _ -> + ResetRate = + maps:map( + fun(_Key, _Value) -> #rate{} end, + maps:get(Id, Rates, #{}) + ), + maps:put(Id, ResetRate, Rates) + end, + SlideSpecs = [{slide, I} || I <- maps:keys(maps:get(Id, Slides, #{}))], + NewSlides = Slides#{Id => create_slides(SlideSpecs)}, {reply, reset_counters(get_self_name(), Id), State#state{ rates = - case Rates of - undefined -> - undefined; - _ -> - ResetRate = - maps:map( - fun(_Key, _Value) -> #rate{} end, - maps:get(Id, Rates, #{}) - ), - maps:put(Id, ResetRate, Rates) - end + NewRates, + slides = NewSlides }}; +handle_call({get_slide, Id}, _From, State = #state{slides = Slides}) -> + SlidesForID = maps:get(Id, Slides, #{}), + {reply, maps:map(fun(Metric, Slide) -> do_get_slide(Id, Metric, Slide) end, SlidesForID), + State}; +handle_call({get_slide, Id, Window}, _From, State = #state{slides = Slides}) -> + SlidesForID = maps:get(Id, Slides, #{}), + {reply, + maps:map(fun(Metric, Slide) -> do_get_slide(Window, Id, Metric, Slide) end, SlidesForID), + State}; handle_call(_Request, _From, State) -> {reply, ok, State}. @@ -256,7 +455,7 @@ handle_cast(_Msg, State) -> handle_info(ticking, State = #state{rates = undefined}) -> erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), {noreply, State}; -handle_info(ticking, State = #state{rates = Rates0}) -> +handle_info(ticking, State = #state{rates = Rates0, slides = Slides0}) -> Rates = maps:map( fun(Id, RatesPerID) -> @@ -269,8 +468,20 @@ handle_info(ticking, State = #state{rates = Rates0}) -> end, Rates0 ), + Slides = + maps:map( + fun(Id, SlidesPerID) -> + maps:map( + fun(Metric, Slide) -> + update_slide(Id, Metric, Slide) + end, + SlidesPerID + ) + end, + Slides0 + ), erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), - {noreply, State#state{rates = Rates}}; + {noreply, State#state{rates = Rates, slides = Slides}}; handle_info(_Info, State) -> {noreply, State}. @@ -301,17 +512,18 @@ create_counters(_Name, _Id, []) -> error({create_counter_error, must_provide_a_list_of_metrics}); create_counters(Name, Id, Metrics) -> %% backup the old counters - OlderCounters = maps:with(Metrics, get_counters(Name, Id)), + OlderCounters = maps:with(filter_counters(Metrics), get_counters(Name, Id)), %% create the new counter - Size = length(Metrics), - Indexes = maps:from_list(lists:zip(Metrics, lists:seq(1, Size))), + {Size, Indexes} = create_metric_indexes(Metrics), Counters = get_pterm(Name), CntrRef = counters:new(Size, [write_concurrency]), persistent_term:put( ?CntrRef(Name), - Counters#{Id => #{ref => CntrRef, indexes => Indexes}} + Counters#{Id => Indexes#{ref => CntrRef}} ), - %% restore the old counters + %% Restore the old counters. Slides are not restored, since they + %% are periodically zeroed anyway. We do lose some samples in the + %% current interval, but that's acceptable for now. lists:foreach( fun({Metric, N}) -> inc(Name, Id, Metric, N) @@ -319,6 +531,16 @@ create_counters(Name, Id, Metrics) -> maps:to_list(OlderCounters) ). +create_metric_indexes(Metrics) -> + create_metric_indexes(Metrics, 1, [], []). + +create_metric_indexes([], Size, Counters, Slides) -> + {Size, #{counter => maps:from_list(Counters), slide => maps:from_list(Slides)}}; +create_metric_indexes([{counter, Id} | Rest], Index, Counters, Slides) -> + create_metric_indexes(Rest, Index + 1, [{Id, Index} | Counters], Slides); +create_metric_indexes([{slide, Id} | Rest], Index, Counters, Slides) -> + create_metric_indexes(Rest, Index + 2, Counters, [{Id, Index} | Slides]). + delete_counters(Name, Id) -> persistent_term:put(?CntrRef(Name), maps:remove(Id, get_pterm(Name))). @@ -328,12 +550,12 @@ get_ref(Name, Id) -> error -> not_found end. -idx_metric(Name, Id, Metric) -> - maps:get(Metric, get_indexes(Name, Id)). +idx_metric(Name, Id, Type, Metric) -> + maps:get(Metric, get_indexes(Name, Type, Id)). -get_indexes(Name, Id) -> +get_indexes(Name, Type, Id) -> case maps:find(Id, get_pterm(Name)) of - {ok, #{indexes := Indexes}} -> Indexes; + {ok, #{Type := Indexes}} -> Indexes; error -> #{} end. @@ -381,6 +603,53 @@ calculate_rate(CurrVal, #rate{ tick = Tick + 1 }. +do_get_slide(Id, Metric, S = #slide{n_samples = NSamples}) -> + #{ + n_samples => NSamples, + current => do_get_slide(2, Id, Metric, S), + last5m => do_get_slide(?SECS_5M, Id, Metric, S) + }. + +do_get_slide(Window, Id, Metric, #slide{datapoints = DP0}) -> + Datapoint = get_slide_datapoint(Id, Metric), + {N, Sum} = get_slide_window(os:system_time(second) - Window, [Datapoint | DP0], 0, 0), + case N > 0 of + true -> Sum div N; + false -> 0 + end. + +get_slide_window(_StartTime, [], N, S) -> + {N, S}; +get_slide_window(StartTime, [#slide_datapoint{time = T} | _], N, S) when T < StartTime -> + {N, S}; +get_slide_window(StartTime, [#slide_datapoint{samples = N, sum = S} | Rest], AccN, AccS) -> + get_slide_window(StartTime, Rest, AccN + N, AccS + S). + +get_slide_datapoint(Id, Metric) -> + Name = get_self_name(), + CRef = get_ref(Name, Id), + Index = idx_metric(Name, Id, slide, Metric), + Total = counters:get(CRef, Index), + N = counters:get(CRef, Index + 1), + #slide_datapoint{ + sum = Total, + samples = N, + time = os:system_time(second) + }. + +update_slide(Id, Metric, Slide0 = #slide{n_samples = NSamples, datapoints = DPs}) -> + Datapoint = get_slide_datapoint(Id, Metric), + %% Reset counters: + Name = get_self_name(), + CRef = get_ref(Name, Id), + Index = idx_metric(Name, Id, slide, Metric), + counters:put(CRef, Index, 0), + counters:put(CRef, Index + 1, 0), + Slide0#slide{ + datapoints = [Datapoint | lists:droplast(DPs)], + n_samples = Datapoint#slide_datapoint.samples + NSamples + }. + format_rates_of_id(RatesPerId) -> maps:map( fun(_Metric, Rates) -> @@ -403,6 +672,27 @@ precision(Float, N) -> Base = math:pow(10, N), round(Float * Base) / Base. +desugar(Metrics) -> + lists:map( + fun + (Atom) when is_atom(Atom) -> + {counter, Atom}; + (Spec = {_, _}) -> + Spec + end, + Metrics + ). + +filter_counters(Metrics) -> + [K || {counter, K} <- Metrics]. + +create_slides(Metrics) -> + EmptyDatapoints = [ + #slide_datapoint{sum = 0, samples = 0, time = 0} + || _ <- lists:seq(1, ?SECS_5M div ?SAMPLING) + ], + maps:from_list([{K, #slide{datapoints = EmptyDatapoints}} || {slide, K} <- Metrics]). + get_self_name() -> {registered_name, Name} = process_info(self(), registered_name), Name. diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 9683b1a8b..483b99587 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -68,7 +68,7 @@ nolink_apply/2 ]). --export([clamp/3]). +-export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). -dialyzer({nowarn_function, [nolink_apply/2]}). @@ -556,6 +556,75 @@ try_to_existing_atom(Convert, Data, Encoding) -> _:Reason -> {error, Reason} end. +is_sensitive_key(token) -> true; +is_sensitive_key("token") -> true; +is_sensitive_key(<<"token">>) -> true; +is_sensitive_key(password) -> true; +is_sensitive_key("password") -> true; +is_sensitive_key(<<"password">>) -> true; +is_sensitive_key(secret) -> true; +is_sensitive_key("secret") -> true; +is_sensitive_key(<<"secret">>) -> true; +is_sensitive_key(_) -> false. + +redact(Term) -> + do_redact(Term, fun is_sensitive_key/1). + +redact(Term, Checker) -> + do_redact(Term, fun(V) -> + is_sensitive_key(V) orelse Checker(V) + end). + +do_redact(L, Checker) when is_list(L) -> + lists:map(fun(E) -> do_redact(E, Checker) end, L); +do_redact(M, Checker) when is_map(M) -> + maps:map( + fun(K, V) -> + do_redact(K, V, Checker) + end, + M + ); +do_redact({Key, Value}, Checker) -> + case Checker(Key) of + true -> + {Key, redact_v(Value)}; + false -> + {do_redact(Key, Checker), do_redact(Value, Checker)} + end; +do_redact(T, Checker) when is_tuple(T) -> + Elements = erlang:tuple_to_list(T), + Redact = do_redact(Elements, Checker), + erlang:list_to_tuple(Redact); +do_redact(Any, _Checker) -> + Any. + +do_redact(K, V, Checker) -> + case Checker(K) of + true -> + redact_v(V); + false -> + do_redact(V, Checker) + end. + +-define(REDACT_VAL, "******"). +redact_v(V) when is_binary(V) -> <>; +redact_v(_V) -> ?REDACT_VAL. + +is_redacted(K, V) -> + do_is_redacted(K, V, fun is_sensitive_key/1). + +is_redacted(K, V, Fun) -> + do_is_redacted(K, V, fun(E) -> + is_sensitive_key(E) orelse Fun(E) + end). + +do_is_redacted(K, ?REDACT_VAL, Fun) -> + Fun(K); +do_is_redacted(K, <>, Fun) -> + Fun(K); +do_is_redacted(_K, _V, _Fun) -> + false. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -568,6 +637,62 @@ ipv6_probe_test() -> ok end. +redact_test_() -> + Case = fun(Type, KeyT) -> + Key = + case Type of + atom -> KeyT; + string -> erlang:atom_to_list(KeyT); + binary -> erlang:atom_to_binary(KeyT) + end, + + ?assert(is_sensitive_key(Key)), + + %% direct + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})), + + %% 1 level nested + ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])), + ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])), + + %% 2 level nested + ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})), + ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})), + ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})), + + %% 3 level nested + ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])), + ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])), + ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}])) + end, + + Types = [atom, string, binary], + Keys = [ + token, + password, + secret + ], + [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types]. + +redact2_test_() -> + Case = fun(Key, Checker) -> + ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)), + ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)), + ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)), + ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker)) + end, + + Checker = fun(E) -> E =:= passcode end, + + Keys = [secret, passcode], + [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. + +case_name(Type, Key) -> + lists:concat([Type, "-", Key]). + -endif. pub_props_to_packet(Properties) -> diff --git a/apps/emqx/src/emqx_mountpoint.erl b/apps/emqx/src/emqx_mountpoint.erl index 5c4c14a42..5b5dac954 100644 --- a/apps/emqx/src/emqx_mountpoint.erl +++ b/apps/emqx/src/emqx_mountpoint.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index fbe4684a7..1806ede1d 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_mqtt_props.erl b/apps/emqx/src/emqx_mqtt_props.erl index e58ed54a4..da811ebd3 100644 --- a/apps/emqx/src/emqx_mqtt_props.erl +++ b/apps/emqx/src/emqx_mqtt_props.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index ae55d4b6e..494e2b33e 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_node_dump.erl b/apps/emqx/src/emqx_node_dump.erl index b6659e257..eacbf98af 100644 --- a/apps/emqx/src/emqx_node_dump.erl +++ b/apps/emqx/src/emqx_node_dump.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_olp.erl b/apps/emqx/src/emqx_olp.erl index 26c5ed6bd..5a4775896 100644 --- a/apps/emqx/src/emqx_olp.erl +++ b/apps/emqx/src/emqx_olp.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 462b80589..a06f56a4c 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index f15898494..d82810d15 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ -module(emqx_packet). +-elvis([{elvis_style, no_spec_with_records, disable}]). + -include("emqx.hrl"). -include("emqx_mqtt.hrl"). @@ -492,7 +494,7 @@ format_variable(undefined, _, _) -> format_variable(Variable, undefined, PayloadEncode) -> format_variable(Variable, PayloadEncode); format_variable(Variable, Payload, PayloadEncode) -> - [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)]. + [format_variable(Variable, PayloadEncode), ",", format_payload(Payload, PayloadEncode)]. format_variable( #mqtt_packet_connect{ diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index 756bcc7b8..0d264f45f 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_pd.erl b/apps/emqx/src/emqx_pd.erl index fb2f848b6..73e75a771 100644 --- a/apps/emqx/src/emqx_pd.erl +++ b/apps/emqx/src/emqx_pd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_pmon.erl b/apps/emqx/src/emqx_pmon.erl index 05007ccef..1deaf3525 100644 --- a/apps/emqx/src/emqx_pmon.erl +++ b/apps/emqx/src/emqx_pmon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 71459d928..1691a533a 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_pool_sup.erl b/apps/emqx/src/emqx_pool_sup.erl index 150e49983..aadd1895a 100644 --- a/apps/emqx/src/emqx_pool_sup.erl +++ b/apps/emqx/src/emqx_pool_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 5b3d53467..9a2589a3a 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 4f40902ab..567488862 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_reason_codes.erl b/apps/emqx/src/emqx_reason_codes.erl index e7e27f565..77a8c1be2 100644 --- a/apps/emqx/src/emqx_reason_codes.erl +++ b/apps/emqx/src/emqx_reason_codes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl index f6a3db5d0..94a0bd9a1 100644 --- a/apps/emqx/src/emqx_release.erl +++ b/apps/emqx/src/emqx_release.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ -export([ edition/0, + edition_vsn_prefix/0, edition_longstr/0, description/0, version/0 @@ -45,11 +46,11 @@ description() -> -spec edition() -> ce | ee. -ifdef(EMQX_RELEASE_EDITION). edition() -> ?EMQX_RELEASE_EDITION. - +edition_vsn_prefix() -> "e". edition_longstr() -> <<"Enterprise">>. -else. edition() -> ce. - +edition_vsn_prefix() -> "v". edition_longstr() -> <<"Opensource">>. -endif. diff --git a/apps/emqx/src/emqx_relup.erl b/apps/emqx/src/emqx_relup.erl index 82ba90b4f..8a9bd9dc3 100644 --- a/apps/emqx/src/emqx_relup.erl +++ b/apps/emqx/src/emqx_relup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 928b0e630..7c9cc61b0 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 3f712bf4a..e2d54b99e 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -144,7 +144,7 @@ handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> %% ignore {noreply, State}; handle_info({mnesia_table_event, Event}, State) -> - ?SLOG(error, #{msg => "unexpected_mnesia_table_event", event => Event}), + ?SLOG(debug, #{msg => "unexpected_mnesia_table_event", event => Event}), {noreply, State}; handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> global:trans( diff --git a/apps/emqx/src/emqx_router_sup.erl b/apps/emqx/src/emqx_router_sup.erl index 2e0f4a3b7..d0e5ea05a 100644 --- a/apps/emqx/src/emqx_router_sup.erl +++ b/apps/emqx/src/emqx_router_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_router_utils.erl b/apps/emqx/src/emqx_router_utils.erl index ccda44fcc..9fab86d37 100644 --- a/apps/emqx/src/emqx_router_utils.erl +++ b/apps/emqx/src/emqx_router_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index 961bfd5d6..e1b5122c4 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ebe27f2a5..4cd78b575 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,8 +40,9 @@ -type comma_separated_atoms() :: [atom()]. -type bar_separated_list() :: list(). -type ip_port() :: tuple() | integer(). --type host_port() :: tuple(). -type cipher() :: map(). +-type port_number() :: 1..65536. +-type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -53,7 +54,6 @@ -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}). -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). --typerefl_from_string({host_port/0, emqx_schema, to_host_port}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). @@ -80,11 +80,19 @@ to_comma_separated_binary/1, to_bar_separated_list/1, to_ip_port/1, - to_host_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1 ]). +-export([ + parse_server/2, + parse_servers/2, + servers_validator/2, + servers_sc/2, + convert_servers/1, + convert_servers/2 +]). + -behaviour(hocon_schema). -reflect_type([ @@ -99,7 +107,6 @@ comma_separated_binary/0, bar_separated_list/0, ip_port/0, - host_port/0, cipher/0, comma_separated_atoms/0 ]). @@ -2172,40 +2179,15 @@ to_bar_separated_list(Str) -> %% - :1883 %% - :::1883 to_ip_port(Str) -> - to_host_port(Str, ip_addr). - -%% @doc support the following format: -%% - 127.0.0.1:1883 -%% - ::1:1883 -%% - [::1]:1883 -%% - :1883 -%% - :::1883 -%% - example.com:80 -to_host_port(Str) -> - to_host_port(Str, hostname). - -%% - example.com:80 -to_host_port(Str, IpOrHost) -> - case split_host_port(Str) of - {"", Port} when IpOrHost =:= ip_addr -> + case split_ip_port(Str) of + {"", Port} -> %% this is a local address {ok, list_to_integer(Port)}; - {"", _Port} -> - %% must specify host part when it's a remote endpoint - {error, bad_host_port}; {MaybeIp, Port} -> PortVal = list_to_integer(Port), case inet:parse_address(MaybeIp) of {ok, IpTuple} -> {ok, {IpTuple, PortVal}}; - _ when IpOrHost =:= hostname -> - %% check is a rfc1035's hostname - case inet_parse:domain(MaybeIp) of - true -> - {ok, {MaybeIp, PortVal}}; - _ -> - {error, bad_hostname} - end; _ -> {error, bad_ip_port} end; @@ -2213,7 +2195,7 @@ to_host_port(Str, IpOrHost) -> {error, bad_ip_port} end. -split_host_port(Str0) -> +split_ip_port(Str0) -> Str = re:replace(Str0, " ", "", [{return, list}, global]), case lists:split(string:rchr(Str, $:), Str) of %% no colon @@ -2376,3 +2358,233 @@ non_empty_string(<<>>) -> {error, empty_string_not_allowed}; non_empty_string("") -> {error, empty_string_not_allowed}; non_empty_string(S) when is_binary(S); is_list(S) -> ok; non_empty_string(_) -> {error, invalid_string}. + +%% @doc Make schema for 'server' or 'servers' field. +%% for each field, there are three passes: +%% 1. converter: Normalize the value. +%% This normalized value is stored in EMQX's raw config. +%% 2. validator: Validate the normalized value. +%% Besides checkin if the value can be empty or undefined +%% it also calls the 3rd pass to see if the provided +%% hosts can be successfully parsed. +%% 3. parsing: Done at runtime in each module which uses this config +servers_sc(Meta0, ParseOpts) -> + %% if this filed has a default value + %% then it is not NOT required + %% NOTE: maps:is_key is not the solution beause #{default => undefined} is legit + HasDefault = (maps:get(default, Meta0, undefined) =/= undefined), + Required = maps:get(required, Meta0, not HasDefault), + Meta = #{ + required => Required, + converter => fun convert_servers/2, + validator => servers_validator(ParseOpts, Required) + }, + sc(string(), maps:merge(Meta, Meta0)). + +%% @hidden Convert a deep map to host:port pairs. +%% This is due to the fact that a host:port string +%% often can be parsed as a HOCON struct. +%% e.g. when a string from environment variable is `host.domain.name:80' +%% without escaped quotes, it's parsed as +%% `#{<<"host">> => #{<<"domain">> => #{<<"name">> => 80}}}' +%% and when it is a comma-separated list of host:port pairs +%% like `h1.foo:80, h2.bar:81' then it is parsed as +%% `#{<<"h1">> => #{<<"foo">> => 80}, <<"h2">> => #{<<"bar">> => 81}}' +%% This function is to format the map back to host:port (pairs) +%% This function also tries to remove spaces around commas in comma-separated, +%% `host:port' list, and format string array to comma-separated. +convert_servers(HoconValue, _HoconOpts) -> + convert_servers(HoconValue). + +convert_servers(undefined) -> + %% should not format 'undefined' as string + %% not to throw exception either + %% (leave it to the 'required => true | false' check) + undefined; +convert_servers(Map) when is_map(Map) -> + try + List = convert_hocon_map_host_port(Map), + iolist_to_binary(string:join(List, ",")) + catch + _:_ -> + throw("bad_host_port") + end; +convert_servers([H | _] = Array) when is_binary(H) orelse is_list(H) -> + %% if the old config was a string array + %% we want to make sure it's converted to a comma-separated + iolist_to_binary([[I, ","] || I <- Array]); +convert_servers(Str) -> + normalize_host_port_str(Str). + +%% remove spaces around comma (,) +normalize_host_port_str(Str) -> + iolist_to_binary(re:replace(Str, "(\s)*,(\s)*", ",")). + +%% @doc Shared validation function for both 'server' and 'servers' string. +%% NOTE: Validator is called after converter. +servers_validator(Opts, Required) -> + fun(Str0) -> + Str = str(Str0), + case Str =:= "" orelse Str =:= "undefined" of + true when Required -> + %% it's a required field + %% but value is set to an empty string (from environment override) + %% or when the filed is not set in config file + %% NOTE: assuming nobody is going to name their server "undefined" + throw("cannot_be_empty"); + true -> + ok; + _ -> + %% it's valid as long as it can be parsed + _ = parse_servers(Str, Opts), + ok + end + end. + +%% @doc Parse `host[:port]' endpoint to a `{Host, Port}' tuple or just `Host' string. +%% `Opt' is a `map()' with below options supported: +%% +%% `default_port': a port number, so users are not forced to configure +%% port number. +%% `no_port': by default it's `false', when set to `true', +%% a `throw' exception is raised if the port is found. +-spec parse_server(undefined | string() | binary(), server_parse_option()) -> + {string(), port_number()}. +parse_server(Str, Opts) -> + case parse_servers(Str, Opts) of + undefined -> + undefined; + [L] -> + L; + [_ | _] = L -> + throw("expecting_one_host_but_got: " ++ integer_to_list(length(L))) + end. + +%% @doc Parse comma separated `host[:port][,host[:port]]' endpoints +%% into a list of `{Host, Port}' tuples or just `Host' string. +-spec parse_servers(undefined | string() | binary(), server_parse_option()) -> + [{string(), port_number()}]. +parse_servers(undefined, _Opts) -> + %% should not parse 'undefined' as string, + %% not to throw exception either, + %% leave it to the 'required => true | false' check + undefined; +parse_servers(Str, Opts) -> + case do_parse_servers(Str, Opts) of + [] -> + %% treat empty as 'undefined' + undefined; + [_ | _] = L -> + L + end. + +do_parse_servers([H | _] = Array, Opts) when is_binary(H) orelse is_list(H) -> + %% the old schema allowed providing a list of strings + %% e.g. ["server1:80", "server2:80"] + lists:map( + fun(HostPort) -> + do_parse_server(str(HostPort), Opts) + end, + Array + ); +do_parse_servers(Str, Opts) when is_binary(Str) orelse is_list(Str) -> + lists:map( + fun(HostPort) -> + do_parse_server(HostPort, Opts) + end, + split_host_port(Str) + ). + +split_host_port(Str) -> + lists:filtermap( + fun(S) -> + case string:strip(S) of + "" -> false; + X -> {true, X} + end + end, + string:tokens(str(Str), ",") + ). + +do_parse_server(Str, Opts) -> + DefaultPort = maps:get(default_port, Opts, undefined), + NotExpectingPort = maps:get(no_port, Opts, false), + case is_integer(DefaultPort) andalso NotExpectingPort of + true -> + %% either provide a default port from schema, + %% or do not allow user to set port number + error("bad_schema"); + false -> + ok + end, + %% do not split with space, there should be no space allowed between host and port + case string:tokens(Str, ":") of + [Hostname, Port] -> + NotExpectingPort andalso throw("not_expecting_port_number"), + {check_hostname(Hostname), parse_port(Port)}; + [Hostname] -> + case is_integer(DefaultPort) of + true -> + {check_hostname(Hostname), DefaultPort}; + false when NotExpectingPort -> + check_hostname(Hostname); + false -> + throw("missing_port_number") + end; + _ -> + throw("bad_host_port") + end. + +check_hostname(Str) -> + %% not intended to use inet_parse:domain here + %% only checking space because it interferes the parsing + case string:tokens(Str, " ") of + [H] -> + case is_port_number(H) of + true -> + throw("expecting_hostname_but_got_a_number"); + false -> + H + end; + _ -> + throw("hostname_has_space") + end. + +convert_hocon_map_host_port(Map) -> + lists:map( + fun({Host, Port}) -> + %% Only when Host:Port string is a valid HOCON object + %% is it possible for the converter to reach here. + %% + %% For example EMQX_FOO__SERVER='1.2.3.4:1234' is parsed as + %% a HOCON string value "1.2.3.4:1234" but not a map because + %% 1 is not a valid HOCON field. + %% + %% EMQX_FOO__SERVER="local.domain.host" (without ':port') + %% is also not a valid HOCON object (because it has no value), + %% hence parsed as string. + true = (Port > 0), + str(Host) ++ ":" ++ integer_to_list(Port) + end, + hocon_maps:flatten(Map, #{}) + ). + +is_port_number(Port) -> + try + _ = parse_port(Port), + true + catch + _:_ -> + false + end. + +parse_port(Port) -> + try + P = list_to_integer(string:strip(Port)), + true = (P > 0), + true = (P =< 65535), + P + catch + _:_ -> + throw("bad_port_number") + end. diff --git a/apps/emqx/src/emqx_secret.erl b/apps/emqx/src/emqx_secret.erl index e0abf1e4c..5340f36ba 100644 --- a/apps/emqx/src/emqx_secret.erl +++ b/apps/emqx/src/emqx_secret.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl index cadfea89f..60596324a 100644 --- a/apps/emqx/src/emqx_sequence.erl +++ b/apps/emqx/src/emqx_sequence.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 6e1060414..2e17190e2 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 2473322bd..94f7fb64d 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_session_router_worker.erl b/apps/emqx/src/emqx_session_router_worker.erl index 4c9397bc1..b55305d3e 100644 --- a/apps/emqx/src/emqx_session_router_worker.erl +++ b/apps/emqx/src/emqx_session_router_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_session_router_worker_sup.erl b/apps/emqx/src/emqx_session_router_worker_sup.erl index e326ad3e2..473f9ccd2 100644 --- a/apps/emqx/src/emqx_session_router_worker_sup.erl +++ b/apps/emqx/src/emqx_session_router_worker_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index e20297703..061f2a42f 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index 7d9854811..ed901d9a9 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_sup.erl b/apps/emqx/src/emqx_sup.erl index c6b802ae7..8c79e7482 100644 --- a/apps/emqx/src/emqx_sup.erl +++ b/apps/emqx/src/emqx_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index 20421235b..81b27d727 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 732cc2321..6ff68820e 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_sys_sup.erl b/apps/emqx/src/emqx_sys_sup.erl index fb50660f8..8c26df020 100644 --- a/apps/emqx/src/emqx_sys_sup.erl +++ b/apps/emqx/src/emqx_sys_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_tables.erl b/apps/emqx/src/emqx_tables.erl index d41d93f63..ffdf7d891 100644 --- a/apps/emqx/src/emqx_tables.erl +++ b/apps/emqx/src/emqx_tables.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index fc3ed9353..eb6091f29 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_tls_psk.erl b/apps/emqx/src/emqx_tls_psk.erl index f350f863e..871c41368 100644 --- a/apps/emqx/src/emqx_tls_psk.erl +++ b/apps/emqx/src/emqx_tls_psk.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 2941a51fe..6434154ed 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index d86a0af33..ea6736038 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.hrl b/apps/emqx/src/emqx_trace/emqx_trace.hrl index a00a37132..096e786dd 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.hrl +++ b/apps/emqx/src/emqx_trace/emqx_trace.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_dl.erl b/apps/emqx/src/emqx_trace/emqx_trace_dl.erl index dd546f793..9392e2302 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_dl.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_dl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index de909ffda..c31bc0355 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index c249d5aee..9c2d2358e 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index 30725be6f..b4ff49bb3 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 14f6ebeed..7223da245 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 731d05844..cf1a9dc08 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 498503495..5447e94e9 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 9b3b34493..817c4b505 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index 27be87808..c2595725b 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session.erl b/apps/emqx/src/persistent_session/emqx_persistent_session.erl index 527206674..c1100cfdb 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session.hrl b/apps/emqx/src/persistent_session/emqx_persistent_session.hrl index 03b823401..eb4224116 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session.hrl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl index 59657470b..34305d7bc 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl index 697c46cfe..1b8beef33 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl index 54ba0adcc..a4c4e5422 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl index d02640cbf..3018df96a 100644 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl +++ b/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/proto/emqx_broker_proto_v1.erl b/apps/emqx/src/proto/emqx_broker_proto_v1.erl index 3e34ba346..1469b4556 100644 --- a/apps/emqx/src/proto/emqx_broker_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_broker_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/proto/emqx_cm_proto_v1.erl b/apps/emqx/src/proto/emqx_cm_proto_v1.erl index 99898bdb2..d81469df4 100644 --- a/apps/emqx/src/proto/emqx_cm_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_cm_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl b/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl index de497f582..875f19852 100644 --- a/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/proto/emqx_proto_v1.erl b/apps/emqx/src/proto/emqx_proto_v1.erl index 3fd2cde9c..c041008ec 100644 --- a/apps/emqx/src/proto/emqx_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/proto/emqx_shared_sub_proto_v1.erl b/apps/emqx/src/proto/emqx_shared_sub_proto_v1.erl index b224bf705..d036947c6 100644 --- a/apps/emqx/src/proto/emqx_shared_sub_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_shared_sub_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 3eb4bf424..dbe8e09a6 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index c079ac125..4bf1e05fd 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index a7596f457..49dd4c240 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 7016a8a00..e2e95d7ad 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_authz_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl index ad06e5a6c..5497422af 100644 --- a/apps/emqx/test/emqx_authz_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_authz_test_mod.erl b/apps/emqx/test/emqx_authz_test_mod.erl index 79c8f865a..3350dd03e 100644 --- a/apps/emqx/test/emqx_authz_test_mod.erl +++ b/apps/emqx/test/emqx_authz_test_mod.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index a0ff5b5cc..ed22a019a 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_batch_SUITE.erl b/apps/emqx/test/emqx_batch_SUITE.erl index e0f6d733c..7522dd8f1 100644 --- a/apps/emqx/test/emqx_batch_SUITE.erl +++ b/apps/emqx/test/emqx_batch_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_boot_SUITE.erl b/apps/emqx/test/emqx_boot_SUITE.erl index 028b1d508..7d0a7b4d3 100644 --- a/apps/emqx/test/emqx_boot_SUITE.erl +++ b/apps/emqx/test/emqx_boot_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_bpapi_SUITE.erl b/apps/emqx/test/emqx_bpapi_SUITE.erl index c1af0046c..5e29071a2 100644 --- a/apps/emqx/test/emqx_bpapi_SUITE.erl +++ b/apps/emqx/test/emqx_bpapi_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 69cede2bc..f218739fc 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index a81d42960..d66f9d974 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_broker_helper_SUITE.erl b/apps/emqx/test/emqx_broker_helper_SUITE.erl index 19053bec1..7c3e1aa91 100644 --- a/apps/emqx/test/emqx_broker_helper_SUITE.erl +++ b/apps/emqx/test/emqx_broker_helper_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 153f0417d..c6610c0e2 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 621e70c63..79c934b47 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 8863088dd..5589f5bd8 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl index 40aa6463c..23bfe5ff3 100644 --- a/apps/emqx/test/emqx_cm_locker_SUITE.erl +++ b/apps/emqx/test/emqx_cm_locker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_cm_registry_SUITE.erl b/apps/emqx/test/emqx_cm_registry_SUITE.erl index c2ae5b56b..ee764a58f 100644 --- a/apps/emqx/test/emqx_cm_registry_SUITE.erl +++ b/apps/emqx/test/emqx_cm_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ac8659735..f440dfc5a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -65,7 +65,8 @@ -export([clear_screen/0]). -export([with_mock/4]). -export([ - on_exit/1 + on_exit/1, + call_janitor/0 ]). %% Toxiproxy API @@ -446,8 +447,11 @@ is_all_tcp_servers_available(Servers) -> is_tcp_server_available(Host, Port) end, case lists:partition(Fun, Servers) of - {_, []} -> true; - {_, Unavail} -> ct:print("Unavailable servers: ~p", [Unavail]) + {_, []} -> + true; + {_, Unavail} -> + ct:print("Unavailable servers: ~p", [Unavail]), + false end. -spec is_tcp_server_available( @@ -933,6 +937,13 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) -> %% Testcase teardown utilities %%------------------------------------------------------------------------------- +%% stop the janitor gracefully to ensure proper cleanup order and less +%% noise in the logs. +call_janitor() -> + Janitor = get_or_spawn_janitor(), + exit(Janitor, normal), + ok. + get_or_spawn_janitor() -> case get({?MODULE, janitor_proc}) of undefined -> diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index ae7706d65..87a35a1e2 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 6277994f1..fe8a5fed8 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 1fe79f74c..8126b35c6 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 344d81f8c..23ddf4008 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_ctl_SUITE.erl b/apps/emqx/test/emqx_ctl_SUITE.erl index d71e5f700..03f7b2148 100644 --- a/apps/emqx/test/emqx_ctl_SUITE.erl +++ b/apps/emqx/test/emqx_ctl_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index f37e20fdc..e27ff67e0 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_frame_SUITE.erl b/apps/emqx/test/emqx_frame_SUITE.erl index 158318b07..2a8d1bc39 100644 --- a/apps/emqx/test/emqx_frame_SUITE.erl +++ b/apps/emqx/test/emqx_frame_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_gc_SUITE.erl b/apps/emqx/test/emqx_gc_SUITE.erl index f853476fe..c4b0922dc 100644 --- a/apps/emqx/test/emqx_gc_SUITE.erl +++ b/apps/emqx/test/emqx_gc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_guid_SUITE.erl b/apps/emqx/test/emqx_guid_SUITE.erl index 27d98b3c9..7c3564bae 100644 --- a/apps/emqx/test/emqx_guid_SUITE.erl +++ b/apps/emqx/test/emqx_guid_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_hooks_SUITE.erl b/apps/emqx/test/emqx_hooks_SUITE.erl index 3c516fcfe..8b631b5b1 100644 --- a/apps/emqx/test/emqx_hooks_SUITE.erl +++ b/apps/emqx/test/emqx_hooks_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_inflight_SUITE.erl b/apps/emqx/test/emqx_inflight_SUITE.erl index 90e2fb398..2c0949b88 100644 --- a/apps/emqx/test/emqx_inflight_SUITE.erl +++ b/apps/emqx/test/emqx_inflight_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx/test/emqx_json_SUITE.erl index 8ab803acf..a0bf48e4e 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx/test/emqx_json_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_keepalive_SUITE.erl b/apps/emqx/test/emqx_keepalive_SUITE.erl index 93f68bfca..dce55409e 100644 --- a/apps/emqx/test/emqx_keepalive_SUITE.erl +++ b/apps/emqx/test/emqx_keepalive_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 5eb216be5..6a7cd2791 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -133,6 +133,8 @@ t_start_stop_listeners(_) -> t_restart_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(), + %% flakyness: eaddrinuse + timer:sleep(timer:seconds(2)), ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). diff --git a/apps/emqx/test/emqx_logger_SUITE.erl b/apps/emqx/test/emqx_logger_SUITE.erl index e7f6c08bc..c8ff63c75 100644 --- a/apps/emqx/test/emqx_logger_SUITE.erl +++ b/apps/emqx/test/emqx_logger_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_map_lib_tests.erl b/apps/emqx/test/emqx_map_lib_tests.erl index f95e75d7b..894811d7c 100644 --- a/apps/emqx/test/emqx_map_lib_tests.erl +++ b/apps/emqx/test/emqx_map_lib_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_message_SUITE.erl b/apps/emqx/test/emqx_message_SUITE.erl index 56313c1ee..f404ff15d 100644 --- a/apps/emqx/test/emqx_message_SUITE.erl +++ b/apps/emqx/test/emqx_message_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_metrics_SUITE.erl b/apps/emqx/test/emqx_metrics_SUITE.erl index 10bbeda27..45a14a6f7 100644 --- a/apps/emqx/test/emqx_metrics_SUITE.erl +++ b/apps/emqx/test/emqx_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_metrics_worker_SUITE.erl b/apps/emqx/test/emqx_metrics_worker_SUITE.erl index 326b0be1e..194c9cc99 100644 --- a/apps/emqx/test/emqx_metrics_worker_SUITE.erl +++ b/apps/emqx/test/emqx_metrics_worker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -46,8 +46,9 @@ end_per_testcase(_, _Config) -> ok. t_get_metrics(_) -> - Metrics = [a, b, c], - ok = emqx_metrics_worker:create_metrics(?NAME, <<"testid">>, Metrics), + Metrics = [a, b, c, {slide, d}], + Id = <<"testid">>, + ok = emqx_metrics_worker:create_metrics(?NAME, Id, Metrics), %% all the metrics are set to zero at start ?assertMatch( #{ @@ -56,18 +57,24 @@ t_get_metrics(_) -> b := #{current := 0.0, max := 0.0, last5m := 0.0}, c := #{current := 0.0, max := 0.0, last5m := 0.0} }, + gauges := #{}, counters := #{ a := 0, b := 0, c := 0 } }, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>) + emqx_metrics_worker:get_metrics(?NAME, Id) ), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, a), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, b), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), + ok = emqx_metrics_worker:inc(?NAME, Id, a), + ok = emqx_metrics_worker:inc(?NAME, Id, b), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, inflight, 5), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, inflight, 7), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id2, queuing, 9), + ok = emqx_metrics_worker:observe(?NAME, Id, d, 10), + ok = emqx_metrics_worker:observe(?NAME, Id, d, 30), ct:sleep(1500), ?LET( #{ @@ -76,27 +83,79 @@ t_get_metrics(_) -> b := #{current := CurrB, max := MaxB, last5m := _}, c := #{current := CurrC, max := MaxC, last5m := _} }, + gauges := #{ + inflight := Inflight, + queuing := Queuing + }, counters := #{ a := 1, b := 1, c := 2 + } = Counters, + slides := #{ + d := #{n_samples := 2, last5m := 20, current := _} } }, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>), + emqx_metrics_worker:get_metrics(?NAME, Id), { ?assert(CurrA > 0), ?assert(CurrB > 0), ?assert(CurrC > 0), ?assert(MaxA > 0), ?assert(MaxB > 0), - ?assert(MaxC > 0) + ?assert(MaxC > 0), + ?assert(Inflight == 12), + ?assert(Queuing == 9), + ?assertNot(maps:is_key(d, Counters)) } ), - ok = emqx_metrics_worker:clear_metrics(?NAME, <<"testid">>). + ok = emqx_metrics_worker:clear_metrics(?NAME, Id). + +t_clear_metrics(_Config) -> + Metrics = [a, b, c], + Id = <<"testid">>, + ok = emqx_metrics_worker:create_metrics(?NAME, Id, Metrics), + ?assertMatch( + #{ + rate := #{ + a := #{current := 0.0, max := 0.0, last5m := 0.0}, + b := #{current := 0.0, max := 0.0, last5m := 0.0}, + c := #{current := 0.0, max := 0.0, last5m := 0.0} + }, + gauges := #{}, + slides := #{}, + counters := #{ + a := 0, + b := 0, + c := 0 + } + }, + emqx_metrics_worker:get_metrics(?NAME, Id) + ), + ok = emqx_metrics_worker:inc(?NAME, Id, a), + ok = emqx_metrics_worker:inc(?NAME, Id, b), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, inflight, 5), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, inflight, 7), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id2, queuing, 9), + ct:sleep(1500), + ok = emqx_metrics_worker:clear_metrics(?NAME, Id), + ?assertEqual( + #{ + counters => #{}, + gauges => #{}, + rate => #{current => 0.0, last5m => 0.0, max => 0.0}, + slides => #{} + }, + emqx_metrics_worker:get_metrics(?NAME, Id) + ), + ok. t_reset_metrics(_) -> - Metrics = [a, b, c], - ok = emqx_metrics_worker:create_metrics(?NAME, <<"testid">>, Metrics), + Metrics = [a, b, c, {slide, d}], + Id = <<"testid">>, + ok = emqx_metrics_worker:create_metrics(?NAME, Id, Metrics), %% all the metrics are set to zero at start ?assertMatch( #{ @@ -105,20 +164,32 @@ t_reset_metrics(_) -> b := #{current := 0.0, max := 0.0, last5m := 0.0}, c := #{current := 0.0, max := 0.0, last5m := 0.0} }, + gauges := #{}, counters := #{ a := 0, b := 0, c := 0 + }, + slides := #{ + d := #{n_samples := 0, last5m := 0, current := 0} } }, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>) + emqx_metrics_worker:get_metrics(?NAME, Id) ), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, a), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, b), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), + ok = emqx_metrics_worker:inc(?NAME, Id, a), + ok = emqx_metrics_worker:inc(?NAME, Id, b), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, inflight, 5), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, inflight, 7), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id2, queuing, 9), + ok = emqx_metrics_worker:observe(?NAME, Id, d, 100), + ok = emqx_metrics_worker:observe(?NAME, Id, d, 200), ct:sleep(1500), - ok = emqx_metrics_worker:reset_metrics(?NAME, <<"testid">>), + ?assertMatch( + #{d := #{n_samples := 2}}, emqx_metrics_worker:get_slide(?NAME, <<"testid">>) + ), + ok = emqx_metrics_worker:reset_metrics(?NAME, Id), ?LET( #{ rate := #{ @@ -126,68 +197,86 @@ t_reset_metrics(_) -> b := #{current := CurrB, max := MaxB, last5m := _}, c := #{current := CurrC, max := MaxC, last5m := _} }, + gauges := Gauges, counters := #{ a := 0, b := 0, c := 0 + }, + slides := #{ + d := #{n_samples := 0, last5m := 0, current := 0} } }, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>), + emqx_metrics_worker:get_metrics(?NAME, Id), { ?assert(CurrA == 0), ?assert(CurrB == 0), ?assert(CurrC == 0), ?assert(MaxA == 0), ?assert(MaxB == 0), - ?assert(MaxC == 0) + ?assert(MaxC == 0), + ?assertEqual(#{}, Gauges) } ), - ok = emqx_metrics_worker:clear_metrics(?NAME, <<"testid">>). + ok = emqx_metrics_worker:clear_metrics(?NAME, Id). t_get_metrics_2(_) -> - Metrics = [a, b, c], + Metrics = [a, b, c, {slide, d}], + Id = <<"testid">>, ok = emqx_metrics_worker:create_metrics( ?NAME, - <<"testid">>, + Id, Metrics, [a] ), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, a), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, b), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), + ok = emqx_metrics_worker:inc(?NAME, Id, a), + ok = emqx_metrics_worker:inc(?NAME, Id, b), + ok = emqx_metrics_worker:inc(?NAME, Id, c), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, inflight, 5), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, inflight, 7), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id2, queuing, 9), ?assertMatch( #{ rate := Rate = #{ a := #{current := _, max := _, last5m := _} }, + gauges := #{}, counters := #{ a := 1, b := 1, c := 1 } } when map_size(Rate) =:= 1, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>) + emqx_metrics_worker:get_metrics(?NAME, Id) ), - ok = emqx_metrics_worker:clear_metrics(?NAME, <<"testid">>). + ok = emqx_metrics_worker:clear_metrics(?NAME, Id). t_recreate_metrics(_) -> - ok = emqx_metrics_worker:create_metrics(?NAME, <<"testid">>, [a]), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, a), + Id = <<"testid">>, + ok = emqx_metrics_worker:create_metrics(?NAME, Id, [a]), + ok = emqx_metrics_worker:inc(?NAME, Id, a), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, inflight, 5), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, inflight, 7), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id2, queuing, 9), ?assertMatch( #{ rate := R = #{ a := #{current := _, max := _, last5m := _} }, + gauges := #{ + inflight := 12, + queuing := 9 + }, counters := C = #{ a := 1 } } when map_size(R) == 1 andalso map_size(C) == 1, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>) + emqx_metrics_worker:get_metrics(?NAME, Id) ), %% we create the metrics again, to add some counters - ok = emqx_metrics_worker:create_metrics(?NAME, <<"testid">>, [a, b, c]), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, b), - ok = emqx_metrics_worker:inc(?NAME, <<"testid">>, c), + ok = emqx_metrics_worker:create_metrics(?NAME, Id, [a, b, c]), + ok = emqx_metrics_worker:inc(?NAME, Id, b), + ok = emqx_metrics_worker:inc(?NAME, Id, c), ?assertMatch( #{ rate := R = #{ @@ -195,13 +284,17 @@ t_recreate_metrics(_) -> b := #{current := _, max := _, last5m := _}, c := #{current := _, max := _, last5m := _} }, + gauges := #{ + inflight := 12, + queuing := 9 + }, counters := C = #{ a := 1, b := 1, c := 1 } } when map_size(R) == 3 andalso map_size(C) == 3, - emqx_metrics_worker:get_metrics(?NAME, <<"testid">>) + emqx_metrics_worker:get_metrics(?NAME, Id) ), - ok = emqx_metrics_worker:clear_metrics(?NAME, <<"testid">>). + ok = emqx_metrics_worker:clear_metrics(?NAME, Id). t_inc_matched(_) -> Metrics = ['rules.matched'], @@ -238,3 +331,102 @@ t_rate(_) -> ct:sleep(3000), ok = emqx_metrics_worker:clear_metrics(?NAME, <<"rule1">>), ok = emqx_metrics_worker:clear_metrics(?NAME, <<"rule:2">>). + +t_get_gauge(_Config) -> + Metric = 'queueing', + %% unknown handler name (inexistent table) + ?assertEqual(0, emqx_metrics_worker:get_gauge(unknown_name, unknown_id, Metric)), + %% unknown resource id + ?assertEqual(0, emqx_metrics_worker:get_gauge(?NAME, unknown_id, Metric)), + + Id = <<"some id">>, + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, Metric, 2), + + ?assertEqual(2, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + ?assertEqual(0, emqx_metrics_worker:get_gauge(?NAME, unknown, Metric)), + + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, Metric, 3), + ?assertEqual(5, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, Metric, 1), + ?assertEqual(4, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + + ?assertEqual(0, emqx_metrics_worker:get_gauge(?NAME, Id, another_metric)), + + ok. + +t_get_gauges(_Config) -> + %% unknown handler name (inexistent table) + ?assertEqual(#{}, emqx_metrics_worker:get_gauges(unknown_name, unknown_id)), + %% unknown resource id + ?assertEqual(#{}, emqx_metrics_worker:get_gauges(?NAME, unknown_id)), + + Metric = 'queuing', + Id = <<"some id">>, + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, Metric, 2), + + ?assertEqual(#{queuing => 2}, emqx_metrics_worker:get_gauges(?NAME, Id)), + ?assertEqual(#{}, emqx_metrics_worker:get_gauges(?NAME, unknown)), + + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, Metric, 3), + ?assertEqual(#{queuing => 5}, emqx_metrics_worker:get_gauges(?NAME, Id)), + + AnotherMetric = 'inflight', + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, Metric, 1), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, AnotherMetric, 10), + ?assertEqual(#{queuing => 4, inflight => 10}, emqx_metrics_worker:get_gauges(?NAME, Id)), + + ok. + +t_delete_gauge(_Config) -> + %% unknown handler name (inexistent table) + ?assertEqual(ok, emqx_metrics_worker:delete_gauges(unknown_name, unknown_id)), + %% unknown resource id + ?assertEqual(ok, emqx_metrics_worker:delete_gauges(?NAME, unknown_id)), + + Metric = 'queuing', + AnotherMetric = 'inflight', + Id = <<"some id">>, + AnotherId = <<"another id">>, + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, Metric, 2), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id1, Metric, 3), + ok = emqx_metrics_worker:set_gauge(?NAME, Id, worker_id0, AnotherMetric, 10), + ok = emqx_metrics_worker:set_gauge(?NAME, AnotherId, worker_id1, AnotherMetric, 10), + ?assertEqual(#{queuing => 5, inflight => 10}, emqx_metrics_worker:get_gauges(?NAME, Id)), + + ?assertEqual(ok, emqx_metrics_worker:delete_gauges(?NAME, Id)), + + ?assertEqual(#{}, emqx_metrics_worker:get_gauges(?NAME, Id)), + ?assertEqual(#{inflight => 10}, emqx_metrics_worker:get_gauges(?NAME, AnotherId)), + + ok. + +t_shift_gauge(_Config) -> + Metric = 'queueing', + Id = <<"some id">>, + AnotherId = <<"another id">>, + + %% unknown handler name (inexistent table) + ?assertEqual( + ok, emqx_metrics_worker:shift_gauge(unknown_name, unknown_id, worker_id0, Metric, 2) + ), + ?assertEqual(0, emqx_metrics_worker:get_gauge(unknown_name, unknown_id, Metric)), + + %% empty resource id + ?assertEqual(ok, emqx_metrics_worker:shift_gauge(?NAME, Id, worker_id0, Metric, 2)), + ?assertEqual(ok, emqx_metrics_worker:shift_gauge(?NAME, AnotherId, worker_id0, Metric, 2)), + ?assertEqual(2, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + ?assertEqual(2, emqx_metrics_worker:get_gauge(?NAME, AnotherId, Metric)), + + ?assertEqual(ok, emqx_metrics_worker:shift_gauge(?NAME, Id, worker_id0, Metric, 3)), + ?assertEqual(5, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + + ?assertEqual(ok, emqx_metrics_worker:shift_gauge(?NAME, Id, worker_id1, Metric, 10)), + ?assertEqual(15, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + + ?assertEqual(ok, emqx_metrics_worker:shift_gauge(?NAME, Id, worker_id1, Metric, -4)), + ?assertEqual(11, emqx_metrics_worker:get_gauge(?NAME, Id, Metric)), + + ?assertEqual(2, emqx_metrics_worker:get_gauge(?NAME, AnotherId, Metric)), + + ok. diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl index e3227d94c..c2bd751fa 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx/test/emqx_misc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl index 63c5fb888..6d065d521 100644 --- a/apps/emqx/test/emqx_mountpoint_SUITE.erl +++ b/apps/emqx/test/emqx_mountpoint_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl index 7032e553c..287d7fdba 100644 --- a/apps/emqx/test/emqx_mqtt_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index efe215276..2ee4b5ffd 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mqtt_props_SUITE.erl b/apps/emqx/test/emqx_mqtt_props_SUITE.erl index 0b9b90a67..b91058855 100644 --- a/apps/emqx/test/emqx_mqtt_props_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_props_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index cb71cef95..7e97c5bf4 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_mqueue_SUITE.erl b/apps/emqx/test/emqx_mqueue_SUITE.erl index c87c42266..ecf4caf90 100644 --- a/apps/emqx/test/emqx_mqueue_SUITE.erl +++ b/apps/emqx/test/emqx_mqueue_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_olp_SUITE.erl b/apps/emqx/test/emqx_olp_SUITE.erl index 7a0a2d21d..b1cfbc0f1 100644 --- a/apps/emqx/test/emqx_olp_SUITE.erl +++ b/apps/emqx/test/emqx_olp_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index c558669af..8729bbdb6 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_packet_SUITE.erl b/apps/emqx/test/emqx_packet_SUITE.erl index ba38c1f08..809fc0d3c 100644 --- a/apps/emqx/test/emqx_packet_SUITE.erl +++ b/apps/emqx/test/emqx_packet_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_passwd_SUITE.erl b/apps/emqx/test/emqx_passwd_SUITE.erl index 7558e2b8f..ddd5ba495 100644 --- a/apps/emqx/test/emqx_passwd_SUITE.erl +++ b/apps/emqx/test/emqx_passwd_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_pd_SUITE.erl b/apps/emqx/test/emqx_pd_SUITE.erl index 3c53e9012..c53bad624 100644 --- a/apps/emqx/test/emqx_pd_SUITE.erl +++ b/apps/emqx/test/emqx_pd_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index c53e4c58b..cc583c632 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_pmon_SUITE.erl b/apps/emqx/test/emqx_pmon_SUITE.erl index 9ab0e6231..31fc406ea 100644 --- a/apps/emqx/test/emqx_pmon_SUITE.erl +++ b/apps/emqx/test/emqx_pmon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_pool_SUITE.erl b/apps/emqx/test/emqx_pool_SUITE.erl index 10e83ee23..070a95240 100644 --- a/apps/emqx/test/emqx_pool_SUITE.erl +++ b/apps/emqx/test/emqx_pool_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_pqueue_SUITE.erl b/apps/emqx/test/emqx_pqueue_SUITE.erl index 4e901ed54..e31477388 100644 --- a/apps/emqx/test/emqx_pqueue_SUITE.erl +++ b/apps/emqx/test/emqx_pqueue_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 0be10f476..2f0f9d494 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 7efcbaa18..f3b97d517 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_reason_codes_SUITE.erl b/apps/emqx/test/emqx_reason_codes_SUITE.erl index 1b258cfe3..de3673800 100644 --- a/apps/emqx/test/emqx_reason_codes_SUITE.erl +++ b/apps/emqx/test/emqx_reason_codes_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_request_handler.erl b/apps/emqx/test/emqx_request_handler.erl index 7f9407258..e317476ca 100644 --- a/apps/emqx/test/emqx_request_handler.erl +++ b/apps/emqx/test/emqx_request_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_request_responser_SUITE.erl b/apps/emqx/test/emqx_request_responser_SUITE.erl index 41ae7f57d..2685b4ac3 100644 --- a/apps/emqx/test/emqx_request_responser_SUITE.erl +++ b/apps/emqx/test/emqx_request_responser_SUITE.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_request_sender.erl b/apps/emqx/test/emqx_request_sender.erl index 170b15d10..41983e2b5 100644 --- a/apps/emqx/test/emqx_request_sender.erl +++ b/apps/emqx/test/emqx_request_sender.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index e5392923c..298a33fe8 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 20c94087a..9fc3bd97b 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_run_sh.erl b/apps/emqx/test/emqx_run_sh.erl index dfb6b9800..566bd985a 100644 --- a/apps/emqx/test/emqx_run_sh.erl +++ b/apps/emqx/test/emqx_run_sh.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index fba70e303..b249dea92 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() -> #{ kind := validation_error, reason := unknown_fields, - unknown := <<"gc_after_handshake">> + unknown := "gc_after_handshake" } ]}, validate(Sc, #{<<"gc_after_handshake">> => true}) @@ -178,27 +178,290 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() -> to_ip_port_test_() -> Ip = fun emqx_schema:to_ip_port/1, - Host = fun(Str) -> - case Ip(Str) of - {ok, {_, _} = Res} -> - %% assert - {ok, Res} = emqx_schema:to_host_port(Str); - _ -> - emqx_schema:to_host_port(Str) - end - end, [ ?_assertEqual({ok, 80}, Ip("80")), - ?_assertEqual({error, bad_host_port}, Host("80")), ?_assertEqual({ok, 80}, Ip(":80")), - ?_assertEqual({error, bad_host_port}, Host(":80")), ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")), - ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")), - ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")), ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")), ?_assertEqual({error, bad_ip_port}, Ip("$:1900")), - ?_assertEqual({error, bad_hostname}, Host("$:1900")), ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")), ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")), ?_assertMatch({ok, {_, 1883}}, Ip(":::1883")) ]. + +-define(T(CASE, EXPR), {CASE, fun() -> EXPR end}). + +parse_server_test_() -> + DefaultPort = ?LINE, + DefaultOpts = #{default_port => DefaultPort}, + Parse2 = fun(Value0, Opts) -> + Value = emqx_schema:convert_servers(Value0), + Validator = emqx_schema:servers_validator(Opts, _Required = true), + try + Result = emqx_schema:parse_servers(Value, Opts), + ?assertEqual(ok, Validator(Value)), + Result + catch + throw:Throw -> + %% assert validator throws the same exception + ?assertThrow(Throw, Validator(Value)), + %% and then let the test code validate the exception + throw(Throw) + end + end, + Parse = fun(Value) -> Parse2(Value, DefaultOpts) end, + HoconParse = fun(Str0) -> + {ok, Map} = hocon:binary(Str0), + Str = emqx_schema:convert_servers(Map), + Parse(Str) + end, + [ + ?T( + "single server, binary, no port", + ?assertEqual( + [{"localhost", DefaultPort}], + Parse(<<"localhost">>) + ) + ), + ?T( + "single server, string, no port", + ?assertEqual( + [{"localhost", DefaultPort}], + Parse("localhost") + ) + ), + ?T( + "single server, list(string), no port", + ?assertEqual( + [{"localhost", DefaultPort}], + Parse(["localhost"]) + ) + ), + ?T( + "single server, list(binary), no port", + ?assertEqual( + [{"localhost", DefaultPort}], + Parse([<<"localhost">>]) + ) + ), + ?T( + "single server, binary, with port", + ?assertEqual( + [{"localhost", 9999}], + Parse(<<"localhost:9999">>) + ) + ), + ?T( + "single server, list(string), with port", + ?assertEqual( + [{"localhost", 9999}], + Parse(["localhost:9999"]) + ) + ), + ?T( + "single server, string, with port", + ?assertEqual( + [{"localhost", 9999}], + Parse("localhost:9999") + ) + ), + ?T( + "single server, list(binary), with port", + ?assertEqual( + [{"localhost", 9999}], + Parse([<<"localhost:9999">>]) + ) + ), + ?T( + "multiple servers, string, no port", + ?assertEqual( + [{"host1", DefaultPort}, {"host2", DefaultPort}], + Parse("host1, host2") + ) + ), + ?T( + "multiple servers, binary, no port", + ?assertEqual( + [{"host1", DefaultPort}, {"host2", DefaultPort}], + Parse(<<"host1, host2,,,">>) + ) + ), + ?T( + "multiple servers, list(string), no port", + ?assertEqual( + [{"host1", DefaultPort}, {"host2", DefaultPort}], + Parse(["host1", "host2"]) + ) + ), + ?T( + "multiple servers, list(binary), no port", + ?assertEqual( + [{"host1", DefaultPort}, {"host2", DefaultPort}], + Parse([<<"host1">>, <<"host2">>]) + ) + ), + ?T( + "multiple servers, string, with port", + ?assertEqual( + [{"host1", 1234}, {"host2", 2345}], + Parse("host1:1234, host2:2345") + ) + ), + ?T( + "multiple servers, binary, with port", + ?assertEqual( + [{"host1", 1234}, {"host2", 2345}], + Parse(<<"host1:1234, host2:2345, ">>) + ) + ), + ?T( + "multiple servers, list(string), with port", + ?assertEqual( + [{"host1", 1234}, {"host2", 2345}], + Parse([" host1:1234 ", "host2:2345"]) + ) + ), + ?T( + "multiple servers, list(binary), with port", + ?assertEqual( + [{"host1", 1234}, {"host2", 2345}], + Parse([<<"host1:1234">>, <<"host2:2345">>]) + ) + ), + ?T( + "unexpected multiple servers", + ?assertThrow( + "expecting_one_host_but_got: 2", + emqx_schema:parse_server(<<"host1:1234, host2:1234">>, #{default_port => 1}) + ) + ), + ?T( + "multiple servers without ports invalid string list", + ?assertThrow( + "hostname_has_space", + Parse2(["host1 host2"], #{no_port => true}) + ) + ), + ?T( + "multiple servers without ports invalid binary list", + ?assertThrow( + "hostname_has_space", + Parse2([<<"host1 host2">>], #{no_port => true}) + ) + ), + ?T( + "multiple servers wihtout port, mixed list(binary|string)", + ?assertEqual( + ["host1", "host2"], + Parse2([<<"host1">>, "host2"], #{no_port => true}) + ) + ), + ?T( + "no default port, missing port number in config", + ?assertThrow( + "missing_port_number", + emqx_schema:parse_server(<<"a">>, #{}) + ) + ), + ?T( + "empty binary string", + ?assertEqual( + undefined, + emqx_schema:parse_server(<<>>, #{no_port => true}) + ) + ), + ?T( + "empty array", + ?assertEqual( + undefined, + emqx_schema:parse_servers([], #{no_port => true}) + ) + ), + ?T( + "empty binary array", + ?assertThrow( + "bad_host_port", + emqx_schema:parse_servers([<<>>], #{no_port => true}) + ) + ), + ?T( + "HOCON value undefined", + ?assertEqual( + undefined, + emqx_schema:parse_server(undefined, #{no_port => true}) + ) + ), + ?T( + "single server map", + ?assertEqual( + [{"host1.domain", 1234}], + HoconParse("host1.domain:1234") + ) + ), + ?T( + "multiple servers map", + ?assertEqual( + [{"host1.domain", 1234}, {"host2.domain", 2345}, {"host3.domain", 3456}], + HoconParse("host1.domain:1234,host2.domain:2345,host3.domain:3456") + ) + ), + ?T( + "no port expected valid port", + ?assertThrow( + "not_expecting_port_number", + emqx_schema:parse_server("localhost:80", #{no_port => true}) + ) + ), + ?T( + "no port expected invalid port", + ?assertThrow( + "not_expecting_port_number", + emqx_schema:parse_server("localhost:notaport", #{no_port => true}) + ) + ), + + ?T( + "bad hostname", + ?assertThrow( + "expecting_hostname_but_got_a_number", + emqx_schema:parse_server(":80", #{default_port => 80}) + ) + ), + ?T( + "bad port", + ?assertThrow( + "bad_port_number", + emqx_schema:parse_server("host:33x", #{default_port => 33}) + ) + ), + ?T( + "bad host with port", + ?assertThrow( + "bad_host_port", + emqx_schema:parse_server("host:name:80", #{default_port => 80}) + ) + ), + ?T( + "bad schema", + ?assertError( + "bad_schema", + emqx_schema:parse_server("whatever", #{default_port => 10, no_port => true}) + ) + ) + ]. + +servers_validator_test() -> + Required = emqx_schema:servers_validator(#{}, true), + NotRequired = emqx_schema:servers_validator(#{}, false), + ?assertThrow("cannot_be_empty", Required("")), + ?assertThrow("cannot_be_empty", Required(<<>>)), + ?assertThrow("cannot_be_empty", Required(undefined)), + ?assertEqual(ok, NotRequired("")), + ?assertEqual(ok, NotRequired(<<>>)), + ?assertEqual(ok, NotRequired(undefined)), + ok. + +converter_invalid_input_test() -> + ?assertEqual(undefined, emqx_schema:convert_servers(undefined)), + %% 'foo: bar' is a valid HOCON value, but 'bar' is not a port number + ?assertThrow("bad_host_port", emqx_schema:convert_servers(#{foo => bar})). diff --git a/apps/emqx/test/emqx_sequence_SUITE.erl b/apps/emqx/test/emqx_sequence_SUITE.erl index 27740c012..b26d5dd8c 100644 --- a/apps/emqx/test/emqx_sequence_SUITE.erl +++ b/apps/emqx/test/emqx_sequence_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl index 91bf4659b..ecc9794d1 100644 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ b/apps/emqx/test/emqx_session_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 3455c41f9..506e97e4d 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_static_checks.erl b/apps/emqx/test/emqx_static_checks.erl index d1f27a3d6..ea221b2b8 100644 --- a/apps/emqx/test/emqx_static_checks.erl +++ b/apps/emqx/test/emqx_static_checks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_stats_SUITE.erl b/apps/emqx/test/emqx_stats_SUITE.erl index e26fd5d1d..a13f94f32 100644 --- a/apps/emqx/test/emqx_stats_SUITE.erl +++ b/apps/emqx/test/emqx_stats_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_sup_SUITE.erl b/apps/emqx/test/emqx_sup_SUITE.erl index 706a9c1f5..d2780469e 100644 --- a/apps/emqx/test/emqx_sup_SUITE.erl +++ b/apps/emqx/test/emqx_sup_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_sys_SUITE.erl b/apps/emqx/test/emqx_sys_SUITE.erl index 6af42ca85..5e4965f7f 100644 --- a/apps/emqx/test/emqx_sys_SUITE.erl +++ b/apps/emqx/test/emqx_sys_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_sys_mon_SUITE.erl b/apps/emqx/test/emqx_sys_mon_SUITE.erl index 98d3e7019..805a06328 100644 --- a/apps/emqx/test/emqx_sys_mon_SUITE.erl +++ b/apps/emqx/test/emqx_sys_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx/test/emqx_tables_SUITE.erl index dcab2ac73..ad53e7139 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx/test/emqx_tables_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl index df17e434a..36ba6c6a0 100644 --- a/apps/emqx/test/emqx_takeover_SUITE.erl +++ b/apps/emqx/test/emqx_takeover_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index b7d2c3507..07d09aca1 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -49,8 +49,7 @@ push_on_exit_callback(Server, Callback) when is_function(Callback, 0) -> init(Parent) -> process_flag(trap_exit, true), - Ref = monitor(process, Parent), - {ok, #{callbacks => [], owner => {Ref, Parent}}}. + {ok, #{callbacks => [], owner => Parent}}. terminate(_Reason, #{callbacks := Callbacks}) -> lists:foreach(fun(Fun) -> Fun() end, Callbacks). @@ -63,7 +62,7 @@ handle_call(_Req, _From, State) -> handle_cast(_Req, State) -> {noreply, State}. -handle_info({'DOWN', Ref, process, Parent, _Reason}, State = #{owner := {Ref, Parent}}) -> +handle_info({'EXIT', Parent, _Reason}, State = #{owner := Parent}) -> {stop, normal, State}; handle_info(_Msg, State) -> {noreply, State}. diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 647e7c3e2..5a81daf6a 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index ff0ec92a0..cbd7e5a6d 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index 38459c16a..c66808132 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index ff4815c30..089575a1d 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_trie_SUITE.erl b/apps/emqx/test/emqx_trie_SUITE.erl index d05e463f4..06696b9ed 100644 --- a/apps/emqx/test/emqx_trie_SUITE.erl +++ b/apps/emqx/test/emqx_trie_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_vm_SUITE.erl b/apps/emqx/test/emqx_vm_SUITE.erl index c5e50e2bb..f9809361b 100644 --- a/apps/emqx/test/emqx_vm_SUITE.erl +++ b/apps/emqx/test/emqx_vm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_vm_mon_SUITE.erl b/apps/emqx/test/emqx_vm_mon_SUITE.erl index aa0a78532..140a00010 100644 --- a/apps/emqx/test/emqx_vm_mon_SUITE.erl +++ b/apps/emqx/test/emqx_vm_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index bb0a6cb05..787491c4b 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_base62.erl b/apps/emqx/test/props/prop_emqx_base62.erl index 822fcb8e4..cf93d08be 100644 --- a/apps/emqx/test/props/prop_emqx_base62.erl +++ b/apps/emqx/test/props/prop_emqx_base62.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_frame.erl b/apps/emqx/test/props/prop_emqx_frame.erl index df50408a8..65f3ab246 100644 --- a/apps/emqx/test/props/prop_emqx_frame.erl +++ b/apps/emqx/test/props/prop_emqx_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx/test/props/prop_emqx_json.erl index 2320176f5..2bc079634 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx/test/props/prop_emqx_json.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_psk.erl b/apps/emqx/test/props/prop_emqx_psk.erl index 3900b43cd..f0659f532 100644 --- a/apps/emqx/test/props/prop_emqx_psk.erl +++ b/apps/emqx/test/props/prop_emqx_psk.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_reason_codes.erl b/apps/emqx/test/props/prop_emqx_reason_codes.erl index 213a05144..67b1165c4 100644 --- a/apps/emqx/test/props/prop_emqx_reason_codes.erl +++ b/apps/emqx/test/props/prop_emqx_reason_codes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_rpc.erl b/apps/emqx/test/props/prop_emqx_rpc.erl index 933bef5cc..e544ae082 100644 --- a/apps/emqx/test/props/prop_emqx_rpc.erl +++ b/apps/emqx/test/props/prop_emqx_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/props/prop_emqx_sys.erl b/apps/emqx/test/props/prop_emqx_sys.erl index c554cbcdf..5e6c56341 100644 --- a/apps/emqx/test/props/prop_emqx_sys.erl +++ b/apps/emqx/test/props/prop_emqx_sys.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index d59eea1af..30482553c 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index c7f9d6fdf..8c8e2efd9 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 2a5695cef..452a7bb90 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 5adf067b4..99a141822 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_password_hashing.erl b/apps/emqx_authn/src/emqx_authn_password_hashing.erl index b55351859..b3e90e2cd 100644 --- a/apps/emqx_authn/src/emqx_authn_password_hashing.erl +++ b/apps/emqx_authn/src/emqx_authn_password_hashing.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index f2a4085d2..88d8955c5 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl index d88acfacc..635bd7323 100644 --- a/apps/emqx_authn/src/emqx_authn_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl index dbd4c3656..bab25bb78 100644 --- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index d920d2719..1352e3daf 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index ddc1bb464..ba13bd069 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/proto/emqx_authn_proto_v1.erl b/apps/emqx_authn/src/proto/emqx_authn_proto_v1.erl index 4f23e0eb8..cb0f2641c 100644 --- a/apps/emqx_authn/src/proto/emqx_authn_proto_v1.erl +++ b/apps/emqx_authn/src/proto/emqx_authn_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 7527edcb3..0a9aaa825 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl index 37ff59379..5ee923859 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index 480950143..1fdd7cef7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 8c43332ca..5709a1fe7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 415d23c25..e915744e1 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 1351ae0dd..3fac0ed7d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index e95302ad4..68913443f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 2962308ab..1cadf9c56 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 69d01f7e8..0c8fedfb5 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index 4c07489c8..3017c3346 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 466aaddbe..1a867b0be 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl index aa4ab6cf8..59865ab41 100644 --- a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index b063bd833..9a3c7c833 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_http_test_server.erl b/apps/emqx_authn/test/emqx_authn_http_test_server.erl index 5f0b3e93b..0cb31b405 100644 --- a/apps/emqx_authn/test/emqx_authn_http_test_server.erl +++ b/apps/emqx_authn/test/emqx_authn_http_test_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index 7aa40f909..7d51ff425 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 547f173d5..7a51d2bbb 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index d9fa225bb..8191fe2e9 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index 0016274ea..07933031e 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl index 1e9981d11..3725bc051 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl index 734a22cc7..baac9554a 100644 --- a/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl +++ b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 2f84b7b90..7681b6ee3 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -100,7 +100,6 @@ t_create_invalid(_Config) -> InvalidConfigs = [ - maps:without([<<"server">>], AuthConfig), AuthConfig#{<<"server">> => <<"unknownhost:3333">>}, AuthConfig#{<<"password">> => <<"wrongpass">>}, AuthConfig#{<<"database">> => <<"wrongdatabase">>} @@ -541,7 +540,7 @@ mysql_config() -> username => <<"root">>, password => <<"public">>, pool_size => 8, - server => {?MYSQL_HOST, ?MYSQL_DEFAULT_PORT}, + server => <>, ssl => #{enable => false} }. diff --git a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl index 653be8daa..5d5a3f7ac 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl index 57b99b508..83b923d0e 100644 --- a/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index a0fbefb01..66529525b 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,11 @@ -define(PATH, [authentication]). all() -> - [{group, require_seeds}, t_create_invalid]. + [ + {group, require_seeds}, + t_update_with_invalid_config, + t_update_with_bad_config_value + ]. groups() -> [{require_seeds, [], [t_create, t_authenticate, t_update, t_destroy, t_is_superuser]}]. @@ -96,12 +100,36 @@ t_create(_Config) -> {ok, [#{provider := emqx_authn_pgsql}]} = emqx_authentication:list_authenticators(?GLOBAL), emqx_authn_test_lib:delete_config(?ResourceID). -t_create_invalid(_Config) -> +%% invalid config which does not pass the schema check should result in an error +t_update_with_invalid_config(_Config) -> + AuthConfig = raw_pgsql_auth_config(), + BadConfig = maps:without([<<"server">>], AuthConfig), + ?assertMatch( + {error, + {bad_authenticator_config, #{ + reason := + {emqx_authn_pgsql, [ + #{ + kind := validation_error, + path := "authentication.server", + reason := required_field, + value := undefined + } + ]} + }}}, + emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, BadConfig} + ) + ), + ok. + +%% bad config values may cause connection failure, but should still be able to update +t_update_with_bad_config_value(_Config) -> AuthConfig = raw_pgsql_auth_config(), InvalidConfigs = [ - maps:without([<<"server">>], AuthConfig), AuthConfig#{<<"server">> => <<"unknownhost:3333">>}, AuthConfig#{<<"password">> => <<"wrongpass">>}, AuthConfig#{<<"database">> => <<"wrongdatabase">>} @@ -591,7 +619,7 @@ pgsql_config() -> username => <<"root">>, password => <<"public">>, pool_size => 8, - server => {?PGSQL_HOST, ?PGSQL_DEFAULT_PORT}, + server => pgsql_server(), ssl => #{enable => false} }. diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl index 59d37ba96..ae0a01572 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 7f4726dda..1df1faa4c 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,7 +31,12 @@ -define(ResourceID, <<"password_based:redis">>). all() -> - [{group, require_seeds}, t_create, t_create_invalid]. + [ + {group, require_seeds}, + t_create, + t_create_with_config_values_wont_work, + t_create_invalid_config + ]. groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. @@ -97,7 +102,7 @@ t_create(_Config) -> {ok, [#{provider := emqx_authn_redis}]} = emqx_authentication:list_authenticators(?GLOBAL). -t_create_invalid(_Config) -> +t_create_with_config_values_wont_work(_Config) -> AuthConfig = raw_redis_auth_config(), InvalidConfigs = [ @@ -131,7 +136,6 @@ t_create_invalid(_Config) -> InvalidConfigs1 = [ - maps:without([<<"server">>], AuthConfig), AuthConfig#{<<"server">> => <<"unknownhost:3333">>}, AuthConfig#{<<"password">> => <<"wrongpass">>}, AuthConfig#{<<"database">> => <<"5678">>} @@ -152,6 +156,22 @@ t_create_invalid(_Config) -> InvalidConfigs1 ). +t_create_invalid_config(_Config) -> + Config0 = raw_redis_auth_config(), + Config = maps:without([<<"server">>], Config0), + ?assertMatch( + {error, + {bad_authenticator_config, #{ + reason := {emqx_authn_redis, [#{kind := validation_error}]} + }}}, + emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, Config}) + ), + ?assertMatch([], emqx_config:get_raw([authentication])), + ?assertEqual( + {error, {not_found, {chain, ?GLOBAL}}}, + emqx_authentication:list_authenticators(?GLOBAL) + ). + t_authenticate(_Config) -> ok = lists:foreach( fun(Sample) -> @@ -591,7 +611,7 @@ redis_config() -> pool_size => 8, redis_type => single, password => "public", - server => {?REDIS_HOST, ?REDIS_DEFAULT_PORT}, + server => <>, ssl => #{enable => false} }. diff --git a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl index 601b58c3c..8df54ebce 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_authn_test_lib.erl b/apps/emqx_authn/test/emqx_authn_test_lib.erl index 7a770be5a..1c0eeb359 100644 --- a/apps/emqx_authn/test/emqx_authn_test_lib.erl +++ b/apps/emqx_authn/test/emqx_authn_test_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl index a0b5ce980..e1a2586cd 100644 --- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 5c7602658..b7afd5e84 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index c876fbf16..f5b9f9da6 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 52f03fa90..682ad7f2e 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ -include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -ifdef(TEST). -compile(export_all). @@ -358,6 +359,7 @@ authorize_non_superuser( emqx_metrics:inc(?METRIC_DENY), {stop, #{result => deny, from => AuthzSource}}; nomatch -> + ?tp(authz_non_superuser, #{result => nomatch}), ?SLOG(info, #{ msg => "authorization_failed_nomatch", username => Username, @@ -388,6 +390,12 @@ do_authorize( nomatch -> emqx_metrics_worker:inc(authz_metrics, Type, nomatch), do_authorize(Client, PubSub, Topic, Tail); + %% {matched, allow | deny | ignore} + {matched, ignore} -> + do_authorize(Client, PubSub, Topic, Tail); + ignore -> + do_authorize(Client, PubSub, Topic, Tail); + %% {matched, allow | deny} Matched -> {Matched, Type} catch diff --git a/apps/emqx_authz/src/emqx_authz_api_cache.erl b/apps/emqx_authz/src/emqx_authz_api_cache.erl index bfcab823c..2be23cc13 100644 --- a/apps/emqx_authz/src/emqx_authz_api_cache.erl +++ b/apps/emqx_authz/src/emqx_authz_api_cache.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index 090f9d1e2..6a747496c 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index b971faa77..44ec0d28a 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl index 905dbe223..72a2db35c 100644 --- a/apps/emqx_authz/src/emqx_authz_api_settings.erl +++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 3e67d4749..f5570f1f1 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_app.erl b/apps/emqx_authz/src/emqx_authz_app.erl index 3ff883f7e..9e906850b 100644 --- a/apps/emqx_authz/src/emqx_authz_app.erl +++ b/apps/emqx_authz/src/emqx_authz_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_client_info.erl b/apps/emqx_authz/src/emqx_authz_client_info.erl index 547ebdf99..3454d56fe 100644 --- a/apps/emqx_authz/src/emqx_authz_client_info.erl +++ b/apps/emqx_authz/src/emqx_authz_client_info.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index fe3150048..9aa2d506f 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 246d9c294..4479d1483 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(emqx_authz). @@ -104,6 +105,7 @@ authorize( log_nomtach_msg(Status, Headers, Body), nomatch; {error, Reason} -> + ?tp(authz_http_request_failure, #{error => Reason}), ?SLOG(error, #{ msg => "http_server_query_failed", resource => ResourceID, diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index 4a5257ecb..6b9c367d1 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 753416ab9..840ce8fb9 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index df3e5c74d..bd81a7398 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index e4e8d13ba..ae58efee1 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 4298510ee..52469ad43 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index c5ad0b657..4aa0983e6 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 8948dd8ac..a684ae6ba 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,14 +47,8 @@ %% Hocon Schema %%-------------------------------------------------------------------- -namespace() -> authz. - -%% @doc authorization schema is not exported -%% but directly used by emqx_schema -roots() -> []. - -fields("authorization") -> - Types = [ +type_names() -> + [ file, http_get, http_post, @@ -67,12 +61,26 @@ fields("authorization") -> redis_single, redis_sentinel, redis_cluster - ], - Unions = [?R_REF(Type) || Type <- Types], + ]. + +namespace() -> authz. + +%% @doc authorization schema is not exported +%% but directly used by emqx_schema +roots() -> []. + +fields("authorization") -> + Types = [?R_REF(Type) || Type <- type_names()], + UnionMemberSelector = + fun + (all_union_members) -> Types; + %% must return list + ({value, Value}) -> [select_union_member(Value)] + end, [ {sources, ?HOCON( - ?ARRAY(?UNION(Unions)), + ?ARRAY(?UNION(UnionMemberSelector)), #{ default => [], desc => ?DESC(sources) @@ -408,9 +416,75 @@ common_rate_field() -> ]. method(Method) -> - ?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}). + ?HOCON(Method, #{required => true, desc => ?DESC(method)}). array(Ref) -> array(Ref, Ref). array(Ref, DescId) -> ?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}). + +select_union_member(#{<<"type">> := <<"mongodb">>} = Value) -> + MongoType = maps:get(<<"mongo_type">>, Value, undefined), + case MongoType of + <<"single">> -> + ?R_REF(mongo_single); + <<"rs">> -> + ?R_REF(mongo_rs); + <<"sharded">> -> + ?R_REF(mongo_sharded); + Else -> + throw(#{ + reason => "unknown_mongo_type", + expected => "single | rs | sharded", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"redis">>} = Value) -> + RedisType = maps:get(<<"redis_type">>, Value, undefined), + case RedisType of + <<"single">> -> + ?R_REF(redis_single); + <<"cluster">> -> + ?R_REF(redis_cluster); + <<"sentinel">> -> + ?R_REF(redis_sentinel); + Else -> + throw(#{ + reason => "unknown_redis_type", + expected => "single | cluster | sentinel", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"http">>} = Value) -> + RedisType = maps:get(<<"method">>, Value, undefined), + case RedisType of + <<"get">> -> + ?R_REF(http_get); + <<"post">> -> + ?R_REF(http_post); + Else -> + throw(#{ + reason => "unknown_http_method", + expected => "get | post", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"built_in_database">>}) -> + ?R_REF(mnesia); +select_union_member(#{<<"type">> := Type}) -> + select_union_member_loop(Type, type_names()); +select_union_member(_) -> + throw("missing_type_field"). + +select_union_member_loop(TypeValue, []) -> + throw(#{ + reason => "unknown_authz_type", + got => TypeValue + }); +select_union_member_loop(TypeValue, [Type | Types]) -> + case TypeValue =:= atom_to_binary(Type) of + true -> + ?R_REF(Type); + false -> + select_union_member_loop(TypeValue, Types) + end. diff --git a/apps/emqx_authz/src/emqx_authz_sup.erl b/apps/emqx_authz/src/emqx_authz_sup.erl index 47a639201..b5841693b 100644 --- a/apps/emqx_authz/src/emqx_authz_sup.erl +++ b/apps/emqx_authz/src/emqx_authz_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index d1302d84f..df77673a2 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/src/proto/emqx_authz_proto_v1.erl b/apps/emqx_authz/src/proto/emqx_authz_proto_v1.erl index 24e820c95..1671b39a3 100644 --- a/apps/emqx_authz/src/proto/emqx_authz_proto_v1.erl +++ b/apps/emqx_authz/src/proto/emqx_authz_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 36841cb1a..b3ce04f43 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 7ddab7321..24b8fe25e 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index f851afe1c..62c747433 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 186f04740..275b04e40 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index a8ae38444..34638d0aa 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 164271c6d..5b5d2618c 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 6ad966344..e91da9829 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(HTTP_PORT, 33333). -define(HTTP_PATH, "/authz/[...]"). @@ -64,7 +65,14 @@ init_per_testcase(_Case, Config) -> Config. end_per_testcase(_Case, _Config) -> - ok = emqx_authz_http_test_server:stop(). + try + ok = emqx_authz_http_test_server:stop() + catch + exit:noproc -> + ok + end, + snabbkaffe:stop(), + ok. %%------------------------------------------------------------------------------ %% Tests @@ -148,7 +156,39 @@ t_response_handling(_Config) -> ?assertEqual( deny, emqx_access_control:authorize(ClientInfo, publish, <<"t">>) - ). + ), + + %% the server cannot be reached; should skip to the next + %% authorizer in the chain. + ok = emqx_authz_http_test_server:stop(), + + ?check_trace( + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + ), + fun(Trace) -> + ?assertMatch( + [ + #{ + ?snk_kind := authz_http_request_failure, + error := {recoverable_error, econnrefused} + } + ], + ?of_kind(authz_http_request_failure, Trace) + ), + ?assert( + ?strict_causality( + #{?snk_kind := authz_http_request_failure}, + #{?snk_kind := authz_non_superuser, result := nomatch}, + Trace + ) + ), + ok + end + ), + + ok. t_query_params(_Config) -> ok = setup_handler_and_config( diff --git a/apps/emqx_authz/test/emqx_authz_http_test_server.erl b/apps/emqx_authz/test/emqx_authz_http_test_server.erl index ea67f363b..31b181729 100644 --- a/apps/emqx_authz/test/emqx_authz_http_test_server.erl +++ b/apps/emqx_authz/test/emqx_authz_http_test_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl index 16600a0ac..56e893c5b 100644 --- a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 8c0f282e8..d7b31b5b5 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index 61c37b8a1..c685e8237 100644 --- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index e1acfd771..38c997f17 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -317,8 +317,7 @@ raw_mysql_authz_config() -> "SELECT permission, action, topic " "FROM acl WHERE username = ${username}" >>, - - <<"server">> => mysql_server() + <<"server">> => <> }. q(Sql) -> @@ -385,9 +384,6 @@ setup_config(SpecialParams) -> SpecialParams ). -mysql_server() -> - iolist_to_binary(io_lib:format("~s", [?MYSQL_HOST])). - mysql_config() -> #{ auto_reconnect => true, @@ -395,7 +391,7 @@ mysql_config() -> username => <<"root">>, password => <<"public">>, pool_size => 8, - server => {?MYSQL_HOST, ?MYSQL_DEFAULT_PORT}, + server => <>, ssl => #{enable => false} }. diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 7ed19716f..fbe17f92e 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -322,8 +322,7 @@ raw_pgsql_authz_config() -> "SELECT permission, action, topic " "FROM acl WHERE username = ${username}" >>, - - <<"server">> => pgsql_server() + <<"server">> => <> }. q(Sql) -> @@ -393,9 +392,6 @@ setup_config(SpecialParams) -> SpecialParams ). -pgsql_server() -> - iolist_to_binary(io_lib:format("~s", [?PGSQL_HOST])). - pgsql_config() -> #{ auto_reconnect => true, @@ -403,7 +399,7 @@ pgsql_config() -> username => <<"root">>, password => <<"public">>, pool_size => 8, - server => {?PGSQL_HOST, ?PGSQL_DEFAULT_PORT}, + server => <>, ssl => #{enable => false} }. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 54d7a406c..ae734f20d 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -163,7 +163,7 @@ t_lookups(_Config) -> %% should still succeed to create even if the config will not work, %% because it's not a part of the schema check -t_create_with_config_values_wont_works(_Config) -> +t_create_with_config_values_wont_work(_Config) -> AuthzConfig = raw_redis_authz_config(), InvalidConfigs = @@ -182,11 +182,15 @@ t_create_with_config_values_wont_works(_Config) -> ). %% creating without a require filed should return error -t_create_invalid_schema(_Config) -> +t_create_invalid_config(_Config) -> AuthzConfig = raw_redis_authz_config(), C = maps:without([<<"server">>], AuthzConfig), ?assertMatch( - {error, {emqx_conf_schema, _}}, + {error, #{ + kind := validation_error, + path := "authorization.sources.1", + discarded_errors_count := 0 + }}, emqx_authz:update(?CMD_REPLACE, [C]) ). @@ -255,12 +259,9 @@ raw_redis_authz_config() -> <<"cmd">> => <<"HGETALL mqtt_user:${username}">>, <<"database">> => <<"1">>, <<"password">> => <<"public">>, - <<"server">> => redis_server() + <<"server">> => <> }. -redis_server() -> - iolist_to_binary(io_lib:format("~s", [?REDIS_HOST])). - q(Command) -> emqx_resource:query( ?REDIS_RESOURCE, @@ -274,7 +275,7 @@ redis_config() -> pool_size => 8, redis_type => single, password => "public", - server => {?REDIS_HOST, ?REDIS_DEFAULT_PORT}, + server => <>, ssl => #{enable => false} }. diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl index eb16c4e50..77f8617ee 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_authz/test/emqx_authz_schema_tests.erl b/apps/emqx_authz/test/emqx_authz_schema_tests.erl new file mode 100644 index 000000000..f7b2e3c10 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_schema_tests.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_schema_tests). + +-include_lib("eunit/include/eunit.hrl"). + +bad_authz_type_test() -> + Txt = "[{type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_authz_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +bad_mongodb_type_test() -> + Txt = "[{type: mongodb, mongo_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_mongodb_type_test() -> + Txt = "[{type: mongodb}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_redis_type_test() -> + Txt = "[{type: redis, redis_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_redis_type_test() -> + Txt = "[{type: redis}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_http_method_test() -> + Txt = "[{type: http, method: getx}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := <<"getx">> + } + ], + check(Txt) + ). + +missing_http_method_test() -> + Txt = "[{type: http, methodx: get}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := undefined + } + ], + check(Txt) + ). + +check(Txt0) -> + Txt = ["sources: ", Txt0], + {ok, RawConf} = hocon:binary(Txt), + try + hocon_tconf:check_plain(schema(), RawConf, #{}) + catch + throw:{_Schema, Errors} -> + throw(Errors) + end. + +schema() -> + #{roots => emqx_authz_schema:fields("authorization")}. diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl index a2cc39b13..db604d764 100644 --- a/apps/emqx_authz/test/emqx_authz_test_lib.erl +++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/LICENSE b/apps/emqx_auto_subscribe/LICENSE deleted file mode 100644 index 818096217..000000000 --- a/apps/emqx_auto_subscribe/LICENSE +++ /dev/null @@ -1,190 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/apps/emqx_auto_subscribe/include/emqx_auto_subscribe.hrl b/apps/emqx_auto_subscribe/include/emqx_auto_subscribe.hrl deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index 1c5627a1f..a273face1 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ - {description, "An OTP application"}, - {vsn, "0.1.2"}, + {description, "Auto subscribe Application"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl index 44d3d04c5..7453eabdb 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,8 +51,21 @@ max_limit() -> list() -> format(emqx_conf:get([auto_subscribe, topics], [])). -update(Topics) -> - update_(Topics). +update(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> + case + emqx_conf:update( + [auto_subscribe, topics], + Topics, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewTopics}} -> + {ok, NewTopics}; + {error, Reason} -> + {error, Reason} + end; +update(_Topics) -> + {error, quota_exceeded}. post_config_update(_KeyPath, _Req, NewTopics, _OldConf, _AppEnvs) -> Config = emqx_conf:get([auto_subscribe], #{}), @@ -95,22 +108,6 @@ format(Rule = #{topic := Topic}) when is_map(Rule) -> nl => maps:get(nl, Rule, 0) }. -update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> - case - emqx_conf:update( - [auto_subscribe, topics], - Topics, - #{rawconf_with_defaults => true, override_to => cluster} - ) - of - {ok, #{raw_config := NewTopics}} -> - {ok, NewTopics}; - {error, Reason} -> - {error, Reason} - end; -update_(_Topics) -> - {error, quota_exceeded}. - update_hook() -> update_hook(emqx_conf:get([auto_subscribe], #{})). diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl index 4d48bfced..678c8e9b7 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). api_spec() -> - emqx_dashboard_swagger:spec(?MODULE). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> ["/mqtt/auto_subscribe"]. @@ -46,15 +46,15 @@ schema("/mqtt/auto_subscribe") -> description => ?DESC(list_auto_subscribe_api), tags => [<<"Auto Subscribe">>], responses => #{ - 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe") + 200 => topics() } }, put => #{ description => ?DESC(update_auto_subscribe_api), tags => [<<"Auto Subscribe">>], - 'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), + 'requestBody' => topics(), responses => #{ - 200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"), + 200 => topics(), 409 => emqx_dashboard_swagger:error_codes( [?EXCEED_LIMIT], ?DESC(update_auto_subscribe_api_response409) @@ -63,14 +63,17 @@ schema("/mqtt/auto_subscribe") -> } }. +topics() -> + Fields = emqx_auto_subscribe_schema:fields("auto_subscribe"), + {topics, Topics} = lists:keyfind(topics, 1, Fields), + Topics. + %%%============================================================================================== %% api apply auto_subscribe(get, _) -> {200, emqx_auto_subscribe:list()}; -auto_subscribe(put, #{body := #{}}) -> - {400, #{code => ?BAD_REQUEST, message => <<"Request body required">>}}; -auto_subscribe(put, #{body := Params}) -> - case emqx_auto_subscribe:update(Params) of +auto_subscribe(put, #{body := Topics}) when is_list(Topics) -> + case emqx_auto_subscribe:update(Topics) of {error, quota_exceeded} -> Message = list_to_binary( io_lib:format( diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl index 7bac3d9fa..460cbff62 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl index f86139059..ec17895fd 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 94e4aa799..a01e17c1f 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_sup.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_sup.erl index 3257afa83..c192acc4f 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_sup.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl index e50bf0ef4..5b24f106e 100644 --- a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl +++ b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_internal.erl b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_internal.erl index 5d413eb41..975195c00 100644 --- a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_internal.erl +++ b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_internal.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 4e11cfd3d..959f8ec1b 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -151,6 +151,32 @@ t_update(_) -> ResponseMap = emqx_json:decode(Response, [return_maps]), ?assertEqual(1, erlang:length(ResponseMap)), + BadBody1 = #{topic => ?TOPIC_S}, + ?assertMatch( + {error, {"HTTP/1.1", 400, "Bad Request"}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, BadBody1) + ), + BadBody2 = [#{topic => ?TOPIC_S, qos => 3}], + ?assertMatch( + {error, {"HTTP/1.1", 400, "Bad Request"}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, BadBody2) + ), + BadBody3 = [#{topic => ?TOPIC_S, rh => 10}], + ?assertMatch( + {error, {"HTTP/1.1", 400, "Bad Request"}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, BadBody3) + ), + BadBody4 = [#{topic => ?TOPIC_S, rap => -1}], + ?assertMatch( + {error, {"HTTP/1.1", 400, "Bad Request"}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, BadBody4) + ), + BadBody5 = [#{topic => ?TOPIC_S, nl => -1}], + ?assertMatch( + {error, {"HTTP/1.1", 400, "Bad Request"}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, BadBody5) + ), + {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}), {ok, _} = emqtt:connect(Client), timer:sleep(100), diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf index c465ef242..d575f09bc 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf @@ -78,17 +78,6 @@ emqx_bridge_schema { } } - metric_batching { - desc { - en: """Count of messages that are currently accumulated in memory waiting for sending in one batch.""" - zh: """当前积压在内存里，等待批量发送的消息个数""" - } - label: { - en: "Batched" - zh: "等待批量发送" - } - } - metric_dropped { desc { en: """Count of messages dropped.""" @@ -120,16 +109,6 @@ emqx_bridge_schema { zh: "队列已满被丢弃" } } - metric_dropped_queue_not_enabled { - desc { - en: """Count of messages dropped due to the queue is not enabled.""" - zh: """因为队列未启用被丢弃的消息个数。""" - } - label: { - en: "Dropped Queue Disabled" - zh: "队列未启用被丢弃" - } - } metric_dropped_resource_not_found { desc { en: """Count of messages dropped due to the resource is not found.""" @@ -193,9 +172,9 @@ emqx_bridge_schema { } } - metric_sent_inflight { + metric_inflight { desc { - en: """Count of messages that were sent asynchronously but ACKs are not received.""" + en: """Count of messages that were sent asynchronously but ACKs are not yet received.""" zh: """已异步地发送但没有收到 ACK 的消息个数。""" } label: { diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf index d9d2d0c40..f58b59aad 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf @@ -93,11 +93,20 @@ HTTP 请求的标头。
desc { en: """ The body of the HTTP request.
+If not provided, the body will be a JSON object of all the available fields.
+There, 'all the available fields' means the context of a MQTT message when +this webhook is triggered by receiving a MQTT message (the `local_topic` is set), +or the context of the event when this webhook is triggered by a rule (i.e. this +webhook is used as an action of a rule).
Template with variables is allowed. """ zh: """ HTTP 请求的正文。
-允许使用带有变量的模板。""" +如果没有设置该字段，请求正文将是包含所有可用字段的 JSON object。
+如果该 webhook 是由于收到 MQTT 消息触发的，'所有可用字段' 将是 MQTT 消息的 +上下文信息；如果该 webhook 是由于规则触发的，'所有可用字段' 则为触发事件的上下文信息。
+允许使用带有变量的模板。 +""" } label: { en: "HTTP Body" diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl index ab0e895fa..d8229cc77 100644 --- a/apps/emqx_bridge/include/emqx_bridge.hrl +++ b/apps/emqx_bridge/include/emqx_bridge.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -16,16 +16,14 @@ -define(EMPTY_METRICS, ?METRICS( - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) ). -define(METRICS( - Batched, Dropped, DroppedOther, DroppedQueueFull, - DroppedQueueNotEnabled, DroppedResourceNotFound, DroppedResourceStopped, Matched, @@ -40,11 +38,9 @@ Rcvd ), #{ - 'batching' => Batched, 'dropped' => Dropped, 'dropped.other' => DroppedOther, 'dropped.queue_full' => DroppedQueueFull, - 'dropped.queue_not_enabled' => DroppedQueueNotEnabled, 'dropped.resource_not_found' => DroppedResourceNotFound, 'dropped.resource_stopped' => DroppedResourceStopped, 'matched' => Matched, @@ -61,11 +57,9 @@ ). -define(metrics( - Batched, Dropped, DroppedOther, DroppedQueueFull, - DroppedQueueNotEnabled, DroppedResourceNotFound, DroppedResourceStopped, Matched, @@ -80,11 +74,9 @@ Rcvd ), #{ - 'batching' := Batched, 'dropped' := Dropped, 'dropped.other' := DroppedOther, 'dropped.queue_full' := DroppedQueueFull, - 'dropped.queue_not_enabled' := DroppedQueueNotEnabled, 'dropped.resource_not_found' := DroppedResourceNotFound, 'dropped.resource_stopped' := DroppedResourceStopped, 'matched' := Matched, diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 86ab01a97..89fb7adaf 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 30387eb8a..c86087014 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -363,10 +363,13 @@ get_matched_egress_bridges(Topic) -> get_matched_bridge_id(_BType, #{enable := false}, _Topic, _BName, Acc) -> Acc; -get_matched_bridge_id(BType, #{local_topic := Filter}, Topic, BName, Acc) when - ?EGRESS_DIR_BRIDGES(BType) --> - do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc); +get_matched_bridge_id(BType, Conf, Topic, BName, Acc) when ?EGRESS_DIR_BRIDGES(BType) -> + case maps:get(local_topic, Conf, undefined) of + undefined -> + Acc; + Filter -> + do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) + end; get_matched_bridge_id(mqtt, #{egress := #{local := #{topic := Filter}}}, Topic, BName, Acc) -> do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc); get_matched_bridge_id(kafka, #{producer := #{mqtt := #{topic := Filter}}}, Topic, BName, Acc) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 6b5e307d8..cf39ebf14 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -207,7 +207,6 @@ info_example_basic(webhook) -> auto_restart_interval => 15000, query_mode => async, async_inflight_window => 100, - enable_queue => false, max_queue_bytes => 100 * 1024 * 1024 } }; @@ -233,7 +232,6 @@ mqtt_main_example() -> health_check_interval => <<"15s">>, auto_restart_interval => <<"60s">>, query_mode => sync, - enable_queue => false, max_queue_bytes => 100 * 1024 * 1024 }, ssl => #{ @@ -330,6 +328,7 @@ schema("/bridges/:id") -> responses => #{ 204 => <<"Bridge deleted">>, 400 => error_schema(['INVALID_ID'], "Update bridge failed"), + 404 => error_schema('NOT_FOUND', "Bridge not found"), 403 => error_schema('FORBIDDEN_REQUEST', "Forbidden operation"), 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") } @@ -409,11 +408,13 @@ schema("/nodes/:node/bridges/:id/operation/:operation") -> '/bridges/:id'(get, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); '/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> - Conf = filter_out_request_body(Conf0), + Conf1 = filter_out_request_body(Conf0), ?TRY_PARSE_ID( Id, case emqx_bridge:lookup(BridgeType, BridgeName) of {ok, _} -> + RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), + Conf = deobfuscate(Conf1, RawConf), case ensure_bridge_created(BridgeType, BridgeName, Conf) of ok -> lookup_from_all_nodes(BridgeType, BridgeName, 200); @@ -433,19 +434,24 @@ schema("/nodes/:node/bridges/:id/operation/:operation") -> end, ?TRY_PARSE_ID( Id, - case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of + case emqx_bridge:lookup(BridgeType, BridgeName) of {ok, _} -> - 204; - {error, {rules_deps_on_this_bridge, RuleIds}} -> - {403, - error_msg( - 'FORBIDDEN_REQUEST', - {<<"There're some rules dependent on this bridge">>, RuleIds} - )}; - {error, timeout} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; - {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} + case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of + {ok, _} -> + 204; + {error, {rules_deps_on_this_bridge, RuleIds}} -> + {403, + error_msg( + 'FORBIDDEN_REQUEST', + {<<"There're some rules dependent on this bridge">>, RuleIds} + )}; + {error, timeout} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + {error, Reason} -> + {500, error_msg('INTERNAL_ERROR', Reason)} + end; + {error, not_found} -> + {404, error_msg('NOT_FOUND', <<"Bridge not found">>)} end ). @@ -604,12 +610,12 @@ format_bridge_info([FirstBridge | _] = Bridges) -> Res = maps:remove(node, FirstBridge), NodeStatus = collect_status(Bridges), NodeMetrics = collect_metrics(Bridges), - Res#{ + redact(Res#{ status => aggregate_status(NodeStatus), node_status => NodeStatus, metrics => aggregate_metrics(NodeMetrics), node_metrics => NodeMetrics - }. + }). collect_status(Bridges) -> [maps:with([node, status], B) || B <- Bridges]. @@ -632,11 +638,11 @@ aggregate_metrics(AllMetrics) -> fun( #{ metrics := ?metrics( - M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15, M16, M17 + M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15 ) }, ?metrics( - N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15, N16, N17 + N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15 ) ) -> ?METRICS( @@ -654,9 +660,7 @@ aggregate_metrics(AllMetrics) -> M12 + N12, M13 + N13, M14 + N14, - M15 + N15, - M16 + N16, - M17 + N17 + M15 + N15 ) end, InitMetrics, @@ -676,41 +680,38 @@ format_resp( Node ) -> RawConfFull = fill_defaults(Type, RawConf), - RawConfFull#{ + redact(RawConfFull#{ type => Type, name => maps:get(<<"name">>, RawConf, BridgeName), node => Node, status => Status, metrics => format_metrics(Metrics) - }. + }). format_metrics(#{ counters := #{ - 'batching' := Batched, 'dropped' := Dropped, 'dropped.other' := DroppedOther, 'dropped.queue_full' := DroppedQueueFull, - 'dropped.queue_not_enabled' := DroppedQueueNotEnabled, 'dropped.resource_not_found' := DroppedResourceNotFound, 'dropped.resource_stopped' := DroppedResourceStopped, 'matched' := Matched, - 'queuing' := Queued, 'retried' := Retried, 'failed' := SentFailed, - 'inflight' := SentInflight, 'success' := SentSucc, 'received' := Rcvd }, + gauges := Gauges, rate := #{ matched := #{current := Rate, last5m := Rate5m, max := RateMax} } }) -> + Queued = maps:get('queuing', Gauges, 0), + SentInflight = maps:get('inflight', Gauges, 0), ?METRICS( - Batched, Dropped, DroppedOther, DroppedQueueFull, - DroppedQueueNotEnabled, DroppedResourceNotFound, DroppedResourceStopped, Matched, @@ -805,3 +806,27 @@ call_operation(Node, OperFunc, BridgeType, BridgeName) -> {error, _} -> {400, error_msg('INVALID_NODE', <<"invalid node">>)} end. + +redact(Term) -> + emqx_misc:redact(Term). + +deobfuscate(NewConf, OldConf) -> + maps:fold( + fun(K, V, Acc) -> + case maps:find(K, OldConf) of + error -> + Acc#{K => V}; + {ok, OldV} when is_map(V), is_map(OldV) -> + Acc#{K => deobfuscate(V, OldV)}; + {ok, OldV} -> + case emqx_misc:is_redacted(K, V) of + true -> + Acc#{K => OldV}; + _ -> + Acc#{K => V} + end + end + end, + #{}, + NewConf + ). diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 077d37d4a..e10034bae 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ef5dd6093..18ce354f1 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -132,13 +132,14 @@ create(BridgeId, Conf) -> create(Type, Name, Conf) -> create(Type, Name, Conf, #{}). -create(Type, Name, Conf, Opts) -> +create(Type, Name, Conf, Opts0) -> ?SLOG(info, #{ msg => "create bridge", type => Type, name => Name, config => Conf }), + Opts = override_start_after_created(Conf, Opts0), {ok, _Data} = emqx_resource:create_local( resource_id(Type, Name), <<"emqx_bridge">>, @@ -146,7 +147,7 @@ create(Type, Name, Conf, Opts) -> parse_confs(bin(Type), Name, Conf), Opts ), - maybe_disable_bridge(Type, Name, Conf). + ok. update(BridgeId, {OldConf, Conf}) -> {BridgeType, BridgeName} = parse_bridge_id(BridgeId), @@ -155,7 +156,7 @@ update(BridgeId, {OldConf, Conf}) -> update(Type, Name, {OldConf, Conf}) -> update(Type, Name, {OldConf, Conf}, #{}). -update(Type, Name, {OldConf, Conf}, Opts) -> +update(Type, Name, {OldConf, Conf}, Opts0) -> %% TODO: sometimes its not necessary to restart the bridge connection. %% %% - if the connection related configs like `servers` is updated, we should restart/start @@ -164,6 +165,7 @@ update(Type, Name, {OldConf, Conf}, Opts) -> %% the `method` or `headers` of a WebHook is changed, then the bridge can be updated %% without restarting the bridge. %% + Opts = override_start_after_created(Conf, Opts0), case emqx_map_lib:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ @@ -174,10 +176,10 @@ update(Type, Name, {OldConf, Conf}, Opts) -> }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> - maybe_disable_bridge(Type, Name, Conf); + ok; {error, not_found} -> ?SLOG(warning, #{ - msg => "updating_a_non-exist_bridge_need_create_a_new_one", + msg => "updating_a_non_existing_bridge", type => Type, name => Name, config => Conf @@ -242,12 +244,6 @@ remove(Type, Name, _Conf, _Opts) -> {error, Reason} -> {error, Reason} end. -maybe_disable_bridge(Type, Name, Conf) -> - case maps:get(enable, Conf, true) of - false -> stop(Type, Name); - true -> ok - end. - maybe_clear_certs(TmpPath, #{ssl := SslConf} = Conf) -> %% don't remove the cert files if they are in use case is_tmp_path_conf(TmpPath, SslConf) of @@ -274,7 +270,6 @@ parse_confs( #{ url := Url, method := Method, - body := Body, headers := Headers, request_timeout := ReqTimeout, max_retries := Retry @@ -288,7 +283,7 @@ parse_confs( #{ path => Path, method => Method, - body => Body, + body => maps:get(body, Conf, undefined), headers => Headers, request_timeout => ReqTimeout, max_retries => Retry @@ -322,3 +317,8 @@ str(Str) when is_list(Str) -> Str. bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). + +override_start_after_created(Config, Opts) -> + Enabled = maps:get(enable, Config, true), + StartAfterCreated = Enabled andalso maps:get(start_after_created, Opts, Enabled), + Opts#{start_after_created => StartAfterCreated}. diff --git a/apps/emqx_bridge/src/emqx_bridge_sup.erl b/apps/emqx_bridge/src/emqx_bridge_sup.erl index ba8181f83..a5e72a8c6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_sup.erl +++ b/apps/emqx_bridge/src/emqx_bridge_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl index 576d2247f..52790ca42 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl index 41601ee1e..7bd83d139 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -82,7 +82,6 @@ default_resource_opts() -> #{ <<"async_inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, - <<"enable_queue">> => false, <<"health_check_interval">> => <<"15s">>, <<"max_queue_bytes">> => <<"1GB">>, <<"query_mode">> => <<"sync">>, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl index 6d2baaaa8..4665a3bc5 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_bridge_mqtt_schema). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 756a8347d..1ad5d7aba 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_bridge_schema). -include_lib("typerefl/include/types.hrl"). @@ -107,18 +122,17 @@ fields(bridges) -> #{ desc => ?DESC("bridges_mqtt"), required => false, - converter => fun emqx_bridge_mqtt_config:upgrade_pre_ee/1 + converter => fun(X, _HoconOpts) -> + emqx_bridge_mqtt_config:upgrade_pre_ee(X) + end } )} ] ++ ee_fields_bridges(); fields("metrics") -> [ - {"batching", mk(integer(), #{desc => ?DESC("metric_batching")})}, {"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})}, {"dropped.other", mk(integer(), #{desc => ?DESC("metric_dropped_other")})}, {"dropped.queue_full", mk(integer(), #{desc => ?DESC("metric_dropped_queue_full")})}, - {"dropped.queue_not_enabled", - mk(integer(), #{desc => ?DESC("metric_dropped_queue_not_enabled")})}, {"dropped.resource_not_found", mk(integer(), #{desc => ?DESC("metric_dropped_resource_not_found")})}, {"dropped.resource_stopped", @@ -127,7 +141,7 @@ fields("metrics") -> {"queuing", mk(integer(), #{desc => ?DESC("metric_queuing")})}, {"retried", mk(integer(), #{desc => ?DESC("metric_retried")})}, {"failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})}, - {"inflight", mk(integer(), #{desc => ?DESC("metric_sent_inflight")})}, + {"inflight", mk(integer(), #{desc => ?DESC("metric_inflight")})}, {"success", mk(integer(), #{desc => ?DESC("metric_sent_success")})}, {"rate", mk(float(), #{desc => ?DESC("metric_rate")})}, {"rate_max", mk(float(), #{desc => ?DESC("metric_rate_max")})}, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index d270fc91e..0495911e7 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_bridge_webhook_schema). -include_lib("typerefl/include/types.hrl"). @@ -100,7 +115,7 @@ request_config() -> mk( binary(), #{ - default => <<"${payload}">>, + default => undefined, desc => ?DESC("config_body") } )}, diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index 99d5af447..ed4807d12 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index c0a58abcc..557eced13 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -303,6 +303,64 @@ t_http_crud_apis(Config) -> }, jsx:decode(ErrMsg2) ), + %% Deleting a non-existing bridge should result in an error + {ok, 404, ErrMsg3} = request(delete, uri(["bridges", BridgeID]), []), + ?assertMatch( + #{ + <<"code">> := _, + <<"message">> := <<"Bridge not found">> + }, + jsx:decode(ErrMsg3) + ), + ok. + +t_http_bridges_local_topic(Config) -> + Port = ?config(port, Config), + %% assert we there's no bridges at first + {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + + %% then we add a webhook bridge, using POST + %% POST /bridges/ will create a bridge + URL1 = ?URL(Port, "path1"), + Name1 = <<"t_http_bridges_with_local_topic1">>, + Name2 = <<"t_http_bridges_without_local_topic1">>, + %% create one http bridge with local_topic + {ok, 201, _} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name1) + ), + %% and we create another one without local_topic + {ok, 201, _} = request( + post, + uri(["bridges"]), + maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name2)) + ), + BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name1), + BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name2), + %% Send an message to emqx and the message should be forwarded to the HTTP server. + %% This is to verify we can have 2 bridges with and without local_topic fields + %% at the same time. + Body = <<"my msg">>, + emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), + ?assert( + receive + {http_server, received, #{ + method := <<"POST">>, + path := <<"/path1">>, + body := Body + }} -> + true; + Msg -> + ct:pal("error: http got unexpected request: ~p", [Msg]), + false + after 100 -> + false + end + ), + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID1]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID2]), []), ok. t_check_dependent_actions_on_delete(Config) -> @@ -514,6 +572,39 @@ t_reset_bridges(Config) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). +t_with_redact_update(_Config) -> + Name = <<"redact_update">>, + Type = <<"mqtt">>, + Password = <<"123456">>, + Template = #{ + <<"type">> => Type, + <<"name">> => Name, + <<"server">> => <<"127.0.0.1:1883">>, + <<"username">> => <<"test">>, + <<"password">> => Password, + <<"ingress">> => + #{<<"remote">> => #{<<"topic">> => <<"t/#">>}} + }, + + {ok, 201, _} = request( + post, + uri(["bridges"]), + Template + ), + + %% update with redacted config + Conf = emqx_misc:redact(Template), + BridgeID = emqx_bridge_resource:bridge_id(Type, Name), + {ok, 200, _ResBin} = request( + put, + uri(["bridges", BridgeID]), + Conf + ), + RawConf = emqx:get_raw_config([bridges, Type, Name]), + Value = maps:get(<<"password">>, RawConf), + ?assertEqual(Password, Value), + ok. + request(Method, Url, Body) -> request(<<"bridge_admin">>, Method, Url, Body). diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index 1bf156ed4..ae4fc4692 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ -compile(export_all). -import(emqx_dashboard_api_test_helpers, [request/4, uri/1]). +-import(emqx_common_test_helpers, [on_exit/1]). -include("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -124,6 +125,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(_, _Config) -> clear_resources(), + emqx_common_test_helpers:call_janitor(), ok. clear_resources() -> @@ -302,7 +304,7 @@ t_egress_custom_clientid_prefix(_Config) -> receive {deliver, RemoteTopic, #message{from = From}} -> Size = byte_size(ResourceID), - ?assertMatch(<<"my-custom-prefix:", ResouceID:Size/binary, _/binary>>, From), + ?assertMatch(<<"my-custom-prefix:", _ResouceID:Size/binary, _/binary>>, From), ok after 1000 -> ct:fail("should have published message") @@ -660,7 +662,6 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> <<"reconnect_interval">> => <<"1s">>, <<"resource_opts">> => #{ <<"worker_pool_size">> => 2, - <<"enable_queue">> => true, <<"query_mode">> => <<"sync">>, %% to make it check the healthy quickly <<"health_check_interval">> => <<"0.5s">> @@ -672,6 +673,12 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> <<"name">> := ?BRIDGE_NAME_EGRESS } = jsx:decode(Bridge), BridgeIDEgress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS), + on_exit(fun() -> + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), + {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + ok + end), %% we now test if the bridge works as expected LocalTopic = <>, RemoteTopic = <>, @@ -733,15 +740,20 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> %% verify the metrics of the bridge, the message should be queued {ok, 200, BridgeStr1} = request(get, uri(["bridges", BridgeIDEgress]), []), + Decoded1 = jsx:decode(BridgeStr1), + ?assertMatch( + Status when (Status == <<"connected">> orelse Status == <<"connecting">>), + maps:get(<<"status">>, Decoded1) + ), %% matched >= 3 because of possible retries. ?assertMatch( #{ - <<"status">> := Status, - <<"metrics">> := #{ - <<"matched">> := Matched, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2 - } - } when Matched >= 3 andalso (Status == <<"connected">> orelse Status == <<"connecting">>), - jsx:decode(BridgeStr1) + <<"matched">> := Matched, + <<"success">> := 1, + <<"failed">> := 0, + <<"queuing">> := 2 + } when Matched >= 3, + maps:get(<<"metrics">>, Decoded1) ), %% start the listener 1883 to make the bridge reconnected @@ -766,9 +778,6 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> %% also verify the 2 messages have been sent to the remote broker assert_mqtt_msg_received(RemoteTopic, Payload1), assert_mqtt_msg_received(RemoteTopic, Payload2), - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), ok. assert_mqtt_msg_received(Topic, Payload) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl index fa3fff7d9..90723252d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index a87471989..0297fddf7 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -ifndef(EMQX_CONF_HRL). -define(EMQX_CONF_HRL, true). diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 4b3886798..fe701049c 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -512,7 +512,7 @@ do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> wait_for_all_nodes_commit(TnxId, Delay, Remain) -> Lagging = lagging_nodes(TnxId), - Stopped = stopped_nodes(), + Stopped = Lagging -- mria_mnesia:running_nodes(), case Lagging -- Stopped of [] when Stopped =:= [] -> ok; @@ -537,9 +537,10 @@ wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain) -> [] -> ok; Lagging -> - case stopped_nodes() of + Stopped = Lagging -- mria_mnesia:running_nodes(), + case Stopped of [] -> {peers_lagging, Lagging}; - Stopped -> {stopped_nodes, Stopped} + _ -> {stopped_nodes, Stopped} end end end. @@ -558,9 +559,6 @@ commit_status_trans(Operator, TnxId) -> Result = '$2', mnesia:select(?CLUSTER_COMMIT, [{MatchHead, [Guard], [Result]}]). -stopped_nodes() -> - ekka_cluster:info(stopped_nodes). - get_retry_ms() -> emqx_conf:get([node, cluster_call, retry_interval], timer:minutes(1)). diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl b/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl index 7f7c7f77f..c3d946a91 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 6d7e9ef9c..3d18083f8 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 25aa82d76..8b471a137 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 20e1444ba..f2e4f6f56 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 71a96ccf3..5c2fd9e18 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 80bb676c8..a7b388964 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -60,7 +60,8 @@ emqx_exhook_schema, emqx_psk_schema, emqx_limiter_schema, - emqx_slow_subs_schema + emqx_slow_subs_schema, + emqx_mgmt_api_key_schema ]). %% root config should not have a namespace diff --git a/apps/emqx_conf/src/emqx_conf_sup.erl b/apps/emqx_conf/src/emqx_conf_sup.erl index 10398d851..d4411af4b 100644 --- a/apps/emqx_conf/src/emqx_conf_sup.erl +++ b/apps/emqx_conf/src/emqx_conf_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 97e14b7c4..84687e314 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index acb62eb94..dd8d2fedd 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 51d1a1cde..4c449f580 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,11 +48,14 @@ init_per_suite(Config) -> meck:new(emqx_alarm, [non_strict, passthrough, no_link]), meck:expect(emqx_alarm, activate, 3, ok), meck:expect(emqx_alarm, deactivate, 3, ok), + meck:new(mria_mnesia, [non_strict, passthrough, no_link]), + meck:expect(mria_mnesia, running_nodes, 0, [?NODE1, {node(), ?NODE2}, {node(), ?NODE3}]), Config. end_per_suite(_Config) -> ekka:stop(), mria:stop(), + meck:unload(mria_mnesia), mria_mnesia:delete_schema(), meck:unload(emqx_alarm), ok. diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index fcf65046d..dab4c4919 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 628f69a8b..3653b9d19 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_conf_schema_tests). diff --git a/apps/emqx_conf/test/emqx_global_gc_SUITE.erl b/apps/emqx_conf/test/emqx_global_gc_SUITE.erl index 9e4a5ffa6..36639f078 100644 --- a/apps/emqx_conf/test/emqx_global_gc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_global_gc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/i18n/emqx_connector_ldap.conf b/apps/emqx_connector/i18n/emqx_connector_ldap.conf new file mode 100644 index 000000000..0bcb4869e --- /dev/null +++ b/apps/emqx_connector/i18n/emqx_connector_ldap.conf @@ -0,0 +1,37 @@ +emqx_connector_ldap { + + bind_dn { + desc { + en: """LDAP's Binding Distinguished Name (DN)""" + zh: """LDAP 绑定的 DN 的值""" + } + label: { + en: "Bind DN" + zh: "Bind DN" + } + } + + port { + desc { + en: """LDAP Port""" + zh: """LDAP 端口""" + } + label: { + en: "Port" + zh: "端口" + } + } + + + timeout { + desc { + en: """LDAP's query timeout""" + zh: """LDAP 查询超时时间""" + } + label: { + en: "timeout" + zh: "超时时间" + } + } + +} diff --git a/apps/emqx_connector/i18n/emqx_connector_mongo.conf b/apps/emqx_connector/i18n/emqx_connector_mongo.conf index e43f7bc33..619a8e3b4 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mongo.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mongo.conf @@ -2,34 +2,34 @@ emqx_connector_mongo { single_mongo_type { desc { - en: "Standalone instance." - zh: "Standalone模式。" + en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode." + zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。 " } label: { en: "Standalone instance" - zh: "Standalone模式" + zh: "Standalone 模式" } } rs_mongo_type { desc { - en: "Replica set." - zh: "Replica set模式。" + en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode." + zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。" } label: { en: "Replica set" - zh: "Replica set模式" + zh: "Replica set 模式" } } sharded_mongo_type { desc { - en: "Sharded cluster." - zh: "Sharded cluster模式。" + en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode." + zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。" } label: { en: "Sharded cluster" - zh: "Sharded cluster模式" + zh: "Sharded cluster 模式" } } @@ -106,15 +106,103 @@ The MongoDB default port 27017 is used if `[:Port]` is not specified. } } - duration { - desc { - en: "Time interval, such as timeout or TTL." - zh: "时间间隔，例如超时或 TTL。" - } - label: { - en: "Time Interval" - zh: "时间间隔" - } + overflow_ttl { + desc { + en: "Period of time before workers that exceed the configured pool size (\"overflow\") to be terminated." + zh: "当池内工人太多时，等待多久清除多余工人。" + } + label { + en: "Overflow TTL" + zh: "溢出TTL" + } + } + + overflow_check_period { + desc { + en: "Period for checking if there are more workers than configured (\"overflow\")." + zh: "检查是否有超过配置的工人的周期（\"溢出\"）。" + } + label { + en: "Overflow Check Period" + zh: "溢出检查周期" + } + } + + local_threshold { + desc { + en: "The size of the latency window for selecting among multiple suitable MongoDB instances." + zh: "在多个合适的MongoDB实例中进行选择的延迟窗口的大小。" + } + label { + en: "Local Threshold" + zh: "本地阈值" + } + } + + connect_timeout { + desc { + en: "The duration to attempt a connection before timing out." + zh: "超时重连的等待时间。" + } + label { + en: "Connect Timeout" + zh: "连接超时" + } + } + + socket_timeout { + desc { + en: "The duration to attempt to send or to receive on a socket before the attempt times out." + zh: "在尝试超时之前，在套接字上尝试发送或接收的持续时间。" + } + label { + en: "Socket Timeout" + zh: "套接字操作超时" + } + } + + server_selection_timeout { + desc { + en: "Specifies how long to block for server selection before throwing an exception." + zh: "指定在抛出异常之前为服务器选择阻断多长时间。" + } + label { + en: "Server Selection Timeout" + zh: "服务器选择超时" + } + } + + wait_queue_timeout { + desc { + en: "The maximum duration that a worker can wait for a connection to become available." + zh: "工作者等待连接可用的最长时间。" + } + label { + en: "Wait Queue Timeout" + zh: "等待队列超时" + } + } + + heartbeat_period { + desc { + en: "Controls when the driver checks the state of the MongoDB deployment. Specify the interval between checks, counted from the end of the previous check until the beginning of the next one." + zh: "控制驱动程序何时检查MongoDB部署的状态。指定检查的间隔时间，从上一次检查结束到下一次检查开始计算。" + } + label { + en: "Heartbeat period" + zh: "心跳期" + } + } + + min_heartbeat_period { + desc { + en: "Controls the minimum amount of time to wait between heartbeats." + zh: "心跳间的最小间隙" + } + label { + en: "Minimum Heartbeat Period" + zh: "最小心跳周期" + } } max_overflow { diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/apps/emqx_connector/i18n/emqx_connector_redis.conf index 228d0805a..f42f38f30 100644 --- a/apps/emqx_connector/i18n/emqx_connector_redis.conf +++ b/apps/emqx_connector/i18n/emqx_connector_redis.conf @@ -2,8 +2,8 @@ emqx_connector_redis { single { desc { - en: "Single mode" - zh: "单机模式。" + en: "Single mode. Must be set to 'single' when Redis server is running in single mode." + zh: "单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。" } label: { en: "Single Mode" @@ -13,8 +13,8 @@ emqx_connector_redis { cluster { desc { - en: "Cluster mode" - zh: "集群模式。" + en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode." + zh: "集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。" } label: { en: "Cluster Mode" @@ -24,8 +24,8 @@ emqx_connector_redis { sentinel { desc { - en: "Sentinel mode" - zh: "哨兵模式。" + en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode." + zh: "哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。" } label: { en: "Sentinel Mode" diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 52c9929a5..96b6ba4d6 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/include/emqx_connector_tables.hrl b/apps/emqx_connector/include/emqx_connector_tables.hrl index 86b1b0549..354df0962 100644 --- a/apps/emqx_connector/include/emqx_connector_tables.hrl +++ b/apps/emqx_connector/include/emqx_connector_tables.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 98490a91c..b84d87e38 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -12,9 +12,9 @@ {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version - {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}}, + {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.18"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version - {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}}, + {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.5"}}}, %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git %% (which has overflow_ttl feature added). %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07). diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index e73b43751..65ef49c6b 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ - {description, "An OTP application"}, - {vsn, "0.1.10"}, + {description, "EMQX Data Integration Connectors"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 62167dc18..2f31e838b 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index ed5897a59..a04850746 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -431,14 +431,13 @@ preprocess_request( #{ method := Method, path := Path, - body := Body, headers := Headers } = Req ) -> #{ method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)), path => emqx_plugin_libs_rule:preproc_tmpl(Path), - body => emqx_plugin_libs_rule:preproc_tmpl(Body), + body => maybe_preproc_tmpl(body, Req), headers => preproc_headers(Headers), request_timeout => maps:get(request_timeout, Req, 30000), max_retries => maps:get(max_retries, Req, 2) @@ -469,6 +468,12 @@ preproc_headers(Headers) when is_list(Headers) -> Headers ). +maybe_preproc_tmpl(Key, Conf) -> + case maps:get(Key, Conf, undefined) of + undefined -> undefined; + Val -> emqx_plugin_libs_rule:preproc_tmpl(Val) + end. + process_request( #{ method := MethodTks, @@ -487,7 +492,7 @@ process_request( request_timeout => ReqTimeout }. -process_request_body([], Msg) -> +process_request_body(undefined, Msg) -> emqx_json:encode(Msg); process_request_body(BodyTks, Msg) -> emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector/src/emqx_connector_jwt.erl index c5cd54cb9..1e695faad 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ -include_lib("emqx_connector/include/emqx_connector_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% API -export([ lookup_jwt/1, - lookup_jwt/2 + lookup_jwt/2, + delete_jwt/2 ]). -type jwt() :: binary(). @@ -44,3 +46,14 @@ lookup_jwt(TId, ResourceId) -> error:badarg -> {error, not_found} end. + +-spec delete_jwt(ets:table(), resource_id()) -> ok. +delete_jwt(TId, ResourceId) -> + try + ets:delete(TId, {ResourceId, jwt}), + ?tp(connector_jwt_deleted, #{}), + ok + catch + error:badarg -> + ok + end. diff --git a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl b/apps/emqx_connector/src/emqx_connector_jwt_sup.erl index ac1d22b71..4076cc725 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -78,7 +78,7 @@ jwt_worker_child_spec(Id, Config) -> restart => transient, type => worker, significant => false, - shutdown => brutal_kill, + shutdown => 5_000, modules => [emqx_connector_jwt_worker] }. diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index cb975ca63..e51b9bbee 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ %% API -export([ start_link/1, - ensure_jwt/1 + ensure_jwt/1, + force_refresh/1 ]). %% gen_server API @@ -32,7 +33,8 @@ handle_cast/2, handle_info/2, format_status/1, - format_status/2 + format_status/2, + terminate/2 ]). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -52,7 +54,7 @@ }. -type jwt() :: binary(). -type state() :: #{ - refresh_timer := undefined | timer:tref(), + refresh_timer := undefined | timer:tref() | reference(), resource_id := resource_id(), expiration := timer:time(), table := ets:table(), @@ -94,6 +96,11 @@ ensure_jwt(Worker) -> gen_server:cast(Worker, {ensure_jwt, Ref}), Ref. +-spec force_refresh(pid()) -> ok. +force_refresh(Worker) -> + _ = erlang:send(Worker, {timeout, force_refresh, ?refresh_jwt}), + ok. + %%----------------------------------------------------------------------------------------- %% gen_server API %%----------------------------------------------------------------------------------------- @@ -102,6 +109,7 @@ ensure_jwt(Worker) -> {ok, state(), {continue, {make_key, binary()}}} | {stop, {error, term()}}. init(#{private_key := PrivateKeyPEM} = Config) -> + process_flag(trap_exit, true), State0 = maps:without([private_key], Config), State = State0#{ jwk => undefined, @@ -148,7 +156,7 @@ handle_cast({ensure_jwt, From}, State0 = #{jwt := JWT}) -> handle_cast(_Req, State) -> {noreply, State}. -handle_info({timeout, TRef, ?refresh_jwt}, State0 = #{refresh_timer := TRef}) -> +handle_info({timeout, _TRef, ?refresh_jwt}, State0) -> State = generate_and_store_jwt(State0), {noreply, State}; handle_info(_Msg, State) -> @@ -161,6 +169,11 @@ format_status(_Opt, [_PDict, State0]) -> State = censor_secrets(State0), [{data, [{"State", State}]}]. +terminate(_Reason, State) -> + #{resource_id := ResourceId, table := TId} = State, + emqx_connector_jwt:delete_jwt(TId, ResourceId), + ok. + %%----------------------------------------------------------------------------------------- %% Helper fns %%----------------------------------------------------------------------------------------- @@ -211,15 +224,14 @@ store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> -spec ensure_timer(state()) -> state(). ensure_timer( State = #{ - refresh_timer := undefined, + refresh_timer := OldTimer, expiration := ExpirationMS0 } ) -> + cancel_timer(OldTimer), ExpirationMS = max(5_000, ExpirationMS0 - 5_000), TRef = erlang:start_timer(ExpirationMS, self(), ?refresh_jwt), - State#{refresh_timer => TRef}; -ensure_timer(State) -> - State. + State#{refresh_timer => TRef}. -spec censor_secrets(state()) -> map(). censor_secrets(State = #{jwt := JWT, jwk := JWK}) -> @@ -232,3 +244,10 @@ censor_secret(undefined) -> undefined; censor_secret(_Secret) -> "******". + +-spec cancel_timer(undefined | timer:tref() | reference()) -> ok. +cancel_timer(undefined) -> + ok; +cancel_timer(TRef) -> + _ = erlang:cancel_timer(TRef), + ok. diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index d53c0e41b..1cb65034d 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,11 @@ -export([connect/1]). -export([search/4]). + +%% port is not expected from configuration because +%% all servers expected to use the same port number +-define(LDAP_HOST_OPTIONS, #{no_port => true}). + %%===================================================================== roots() -> ldap_fields() ++ emqx_connector_schema_lib:ssl_fields(). @@ -63,12 +68,7 @@ on_start( connector => InstId, config => Config }), - Servers = [ - begin - proplists:get_value(host, S) - end - || S <- Servers0 - ], + Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = case maps:get(enable, SSL) of true -> @@ -86,8 +86,7 @@ on_start( {bind_password, BindPassword}, {timeout, Timeout}, {pool_size, PoolSize}, - {auto_reconnect, reconn_interval(AutoReconn)}, - {servers, Servers} + {auto_reconnect, reconn_interval(AutoReconn)} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of @@ -166,7 +165,7 @@ connect(Opts) -> ldap_fields() -> [ - {servers, fun servers/1}, + {servers, servers()}, {port, fun port/1}, {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {bind_dn, fun bind_dn/1}, @@ -175,11 +174,8 @@ ldap_fields() -> {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. -servers(type) -> list(); -servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; -servers(converter) -> fun to_servers_raw/1; -servers(required) -> true; -servers(_) -> undefined. +servers() -> + emqx_schema:servers_sc(#{}, ?LDAP_HOST_OPTIONS). bind_dn(type) -> binary(); bind_dn(default) -> 0; @@ -191,24 +187,3 @@ port(_) -> undefined. duration(type) -> emqx_schema:duration_ms(); duration(_) -> undefined. - -to_servers_raw(Servers) -> - {ok, - lists:map( - fun(Server) -> - case string:tokens(Server, ": ") of - [Ip] -> - [{host, Ip}]; - [Ip, Port] -> - [{host, Ip}, {port, list_to_integer(Port)}] - end - end, - string:tokens(str(Servers), ", ") - )}. - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. diff --git a/apps/emqx_connector/src/emqx_connector_lib.erl b/apps/emqx_connector/src/emqx_connector_lib.erl new file mode 100644 index 000000000..d554ca5ab --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_lib.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_lib). + +-export([resolve_dns/2]). + +%% @doc Mostly for meck. +resolve_dns(DNS, Type) -> + inet_res:lookup(DNS, in, Type). diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 678a4f847..0bcc39208 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -39,18 +39,16 @@ -export([mongo_query/5, mongo_insert/3, check_worker_health/1]). +%% for testing +-export([maybe_resolve_srv_and_txt_records/1]). + -define(HEALTH_CHECK_TIMEOUT, 30000). %% mongo servers don't need parse -define(MONGO_HOST_OPTIONS, #{ - host_type => hostname, default_port => ?MONGO_DEFAULT_PORT }). --ifdef(TEST). --export([to_servers_raw/1]). --endif. - %%===================================================================== roots() -> [ @@ -70,10 +68,9 @@ fields(single) -> {mongo_type, #{ type => single, default => single, - required => true, desc => ?DESC("single_mongo_type") }}, - {server, fun server/1}, + {server, server()}, {w_mode, fun w_mode/1} ] ++ mongo_fields(); fields(rs) -> @@ -81,10 +78,9 @@ fields(rs) -> {mongo_type, #{ type => rs, default => rs, - required => true, desc => ?DESC("rs_mongo_type") }}, - {servers, fun servers/1}, + {servers, servers()}, {w_mode, fun w_mode/1}, {r_mode, fun r_mode/1}, {replica_set_name, fun replica_set_name/1} @@ -94,25 +90,24 @@ fields(sharded) -> {mongo_type, #{ type => sharded, default => sharded, - required => true, desc => ?DESC("sharded_mongo_type") }}, - {servers, fun servers/1}, + {servers, servers()}, {w_mode, fun w_mode/1} ] ++ mongo_fields(); fields(topology) -> [ {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {max_overflow, fun max_overflow/1}, - {overflow_ttl, fun duration/1}, - {overflow_check_period, fun duration/1}, - {local_threshold_ms, fun duration/1}, - {connect_timeout_ms, fun duration/1}, - {socket_timeout_ms, fun duration/1}, - {server_selection_timeout_ms, fun duration/1}, - {wait_queue_timeout_ms, fun duration/1}, - {heartbeat_frequency_ms, fun duration/1}, - {min_heartbeat_frequency_ms, fun duration/1} + {overflow_ttl, duration("overflow_ttl")}, + {overflow_check_period, duration("overflow_check_period")}, + {local_threshold_ms, duration("local_threshold")}, + {connect_timeout_ms, duration("connect_timeout")}, + {socket_timeout_ms, duration("socket_timeout")}, + {server_selection_timeout_ms, duration("server_selection_timeout")}, + {wait_queue_timeout_ms, duration("wait_queue_timeout")}, + {heartbeat_frequency_ms, duration("heartbeat_period")}, + {min_heartbeat_frequency_ms, duration("min_heartbeat_period")} ]. desc(single) -> @@ -161,7 +156,7 @@ on_start( sharded -> "starting_mongodb_sharded_connector" end, ?SLOG(info, #{msg => Msg, connector => InstId, config => Config}), - NConfig = #{hosts := Hosts} = may_parse_srv_and_txt_records(Config), + NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of true -> @@ -387,19 +382,13 @@ init_worker_options([], Acc) -> %% =================================================================== %% Schema funcs -server(type) -> emqx_schema:host_port(); -server(required) -> true; -server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; -server(converter) -> fun to_server_raw/1; -server(desc) -> ?DESC("server"); -server(_) -> undefined. +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?MONGO_HOST_OPTIONS). -servers(type) -> list(); -servers(required) -> true; -servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; -servers(converter) -> fun to_servers_raw/1; -servers(desc) -> ?DESC("servers"); -servers(_) -> undefined. +servers() -> + Meta = #{desc => ?DESC("servers")}, + emqx_schema:servers_sc(Meta, ?MONGO_HOST_OPTIONS). w_mode(type) -> hoconsc:enum([unsafe, safe]); w_mode(desc) -> ?DESC("w_mode"); @@ -411,10 +400,12 @@ r_mode(desc) -> ?DESC("r_mode"); r_mode(default) -> master; r_mode(_) -> undefined. -duration(type) -> emqx_schema:duration_ms(); -duration(desc) -> ?DESC("duration"); -duration(required) -> false; -duration(_) -> undefined. +duration(Desc) -> + #{ + type => emqx_schema:duration_ms(), + required => false, + desc => ?DESC(Desc) + }. max_overflow(type) -> non_neg_integer(); max_overflow(desc) -> ?DESC("max_overflow"); @@ -434,163 +425,109 @@ srv_record(_) -> undefined. %% =================================================================== %% Internal funcs -may_parse_srv_and_txt_records(#{server := Server} = Config) -> +maybe_resolve_srv_and_txt_records(#{server := Server} = Config) -> NConfig = maps:remove(server, Config), - may_parse_srv_and_txt_records_(NConfig#{servers => [Server]}); -may_parse_srv_and_txt_records(Config) -> - may_parse_srv_and_txt_records_(Config). + maybe_resolve_srv_and_txt_records1(Server, NConfig); +maybe_resolve_srv_and_txt_records(#{servers := Servers} = Config) -> + NConfig = maps:remove(servers, Config), + maybe_resolve_srv_and_txt_records1(Servers, NConfig). -may_parse_srv_and_txt_records_( +maybe_resolve_srv_and_txt_records1( + Servers0, #{ mongo_type := Type, - srv_record := false, - servers := Servers + srv_record := false } = Config ) -> case Type =:= rs andalso maps:is_key(replica_set_name, Config) =:= false of true -> - error({missing_parameter, replica_set_name}); + throw(#{ + reason => "missing_parameter", + param => replica_set_name + }); false -> - Config#{hosts => servers_to_bin(lists:flatten(Servers))} + Servers = parse_servers(Servers0), + Config#{hosts => format_hosts(Servers)} end; -may_parse_srv_and_txt_records_( +maybe_resolve_srv_and_txt_records1( + Servers, #{ mongo_type := Type, - srv_record := true, - servers := Servers + srv_record := true } = Config ) -> - Hosts = parse_srv_records(Type, Servers), - ExtraOpts = parse_txt_records(Type, Servers), + %% when srv is in use, it's typically only one DNS resolution needed, + %% however, by the schema definition, it's allowed to configure more than one. + %% here we keep only the fist + [{DNS, _IgnorePort} | _] = parse_servers(Servers), + DnsRecords = resolve_srv_records(DNS), + Hosts = format_hosts(DnsRecords), + ?tp(info, resolved_srv_records, #{dns => DNS, resolved_hosts => Hosts}), + ExtraOpts = resolve_txt_records(Type, DNS), + ?tp(info, resolved_txt_records, #{dns => DNS, resolved_options => ExtraOpts}), maps:merge(Config#{hosts => Hosts}, ExtraOpts). -parse_srv_records(Type, Servers) -> - Fun = fun(AccIn, {IpOrHost, _Port}) -> - case - inet_res:lookup( - "_mongodb._tcp." ++ - ip_or_host_to_string(IpOrHost), - in, - srv - ) - of - [] -> - error(service_not_found); - Services -> - [ - [server_to_bin({Host, Port}) || {_, _, Port, Host} <- Services] - | AccIn - ] - end - end, - Res = lists:foldl(Fun, [], Servers), - case Type of - single -> lists:nth(1, Res); - _ -> Res +resolve_srv_records(DNS0) -> + DNS = "_mongodb._tcp." ++ DNS0, + DnsData = emqx_connector_lib:resolve_dns(DNS, srv), + case [{Host, Port} || {_, _, Port, Host} <- DnsData] of + [] -> + throw(#{ + reason => "failed_to_resolve_srv_record", + dns => DNS + }); + L -> + L end. -parse_txt_records(Type, Servers) -> - Fields = - case Type of - rs -> ["authSource", "replicaSet"]; - _ -> ["authSource"] - end, - Fun = fun(AccIn, {IpOrHost, _Port}) -> - case inet_res:lookup(IpOrHost, in, txt) of - [] -> - #{}; - [[QueryString]] -> - case uri_string:dissect_query(QueryString) of - {error, _, _} -> - error({invalid_txt_record, invalid_query_string}); - Options -> - maps:merge(AccIn, take_and_convert(Fields, Options)) - end; - _ -> - error({invalid_txt_record, multiple_records}) - end - end, - lists:foldl(Fun, #{}, Servers). +resolve_txt_records(Type, DNS) -> + case emqx_connector_lib:resolve_dns(DNS, txt) of + [] -> + #{}; + [[QueryString]] = L -> + %% e.g. "authSource=admin&replicaSet=atlas-wrnled-shard-0" + case uri_string:dissect_query(QueryString) of + {error, _, _} -> + throw(#{ + reason => "bad_txt_record_resolution", + resolved => L + }); + Options -> + convert_options(Type, normalize_options(Options)) + end; + L -> + throw(#{ + reason => "multiple_txt_records", + resolved => L + }) + end. -take_and_convert(Fields, Options) -> - take_and_convert(Fields, Options, #{}). +normalize_options([]) -> + []; +normalize_options([{Name, Value} | Options]) -> + [{string:lowercase(Name), Value} | normalize_options(Options)]. -take_and_convert([], [_ | _], _Acc) -> - error({invalid_txt_record, invalid_option}); -take_and_convert([], [], Acc) -> - Acc; -take_and_convert([Field | More], Options, Acc) -> - case lists:keytake(Field, 1, Options) of - {value, {"authSource", V}, NOptions} -> - take_and_convert(More, NOptions, Acc#{auth_source => list_to_binary(V)}); - {value, {"replicaSet", V}, NOptions} -> - take_and_convert(More, NOptions, Acc#{replica_set_name => list_to_binary(V)}); - {value, _, _} -> - error({invalid_txt_record, invalid_option}); +convert_options(rs, Options) -> + M1 = maybe_add_option(auth_source, "authSource", Options), + M2 = maybe_add_option(replica_set_name, "replicaSet", Options), + maps:merge(M1, M2); +convert_options(_, Options) -> + maybe_add_option(auth_source, "authSource", Options). + +maybe_add_option(ConfigKey, OptName0, Options) -> + OptName = string:lowercase(OptName0), + case lists:keyfind(OptName, 1, Options) of + {_, OptValue} -> + #{ConfigKey => iolist_to_binary(OptValue)}; false -> - take_and_convert(More, Options, Acc) + #{} end. --spec ip_or_host_to_string(binary() | string() | tuple()) -> - string(). -ip_or_host_to_string(Ip) when is_tuple(Ip) -> - inet:ntoa(Ip); -ip_or_host_to_string(Host) -> - str(Host). +format_host({Host, Port}) -> + iolist_to_binary([Host, ":", integer_to_list(Port)]). -servers_to_bin([Server | Rest]) -> - [server_to_bin(Server) | servers_to_bin(Rest)]; -servers_to_bin([]) -> - []. +format_hosts(Hosts) -> + lists:map(fun format_host/1, Hosts). -server_to_bin({IpOrHost, Port}) -> - iolist_to_binary(ip_or_host_to_string(IpOrHost) ++ ":" ++ integer_to_list(Port)). - -%% =================================================================== -%% typereflt funcs - --spec to_server_raw(string()) -> - {string(), pos_integer()}. -to_server_raw(Server) -> - emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS). - --spec to_servers_raw(string()) -> - [{string(), pos_integer()}]. -to_servers_raw(Servers) -> - lists:map( - fun(Server) -> - emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS) - end, - split_servers(Servers) - ). - -split_servers(L) when is_list(L) -> - PossibleTypes = [ - list(binary()), - list(string()), - string() - ], - TypeChecks = lists:map(fun(T) -> typerefl:typecheck(T, L) end, PossibleTypes), - case TypeChecks of - [ok, _, _] -> - %% list(binary()) - lists:map(fun binary_to_list/1, L); - [_, ok, _] -> - %% list(string()) - L; - [_, _, ok] -> - %% string() - string:tokens(L, ", "); - [_, _, _] -> - %% invalid input - throw("List of servers must contain only strings") - end; -split_servers(B) when is_binary(B) -> - string:tokens(str(B), ", "). - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. +parse_servers(HoconValue) -> + emqx_schema:parse_servers(HoconValue, ?MONGO_HOST_OPTIONS). diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index f22563960..522f15ccf 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -136,7 +136,7 @@ drop_bridge(Name) -> %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. on_message_received(Msg, HookPoint, ResId) -> - emqx_resource:inc_received(ResId), + emqx_resource_metrics:received_inc(ResId), emqx:run_hook(HookPoint, [Msg]). %% =================================================================== diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 634968b09..6c0ff7210 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -43,7 +43,6 @@ -export([do_get_status/1]). -define(MYSQL_HOST_OPTIONS, #{ - host_type => inet_addr, default_port => ?MYSQL_DEFAULT_PORT }). @@ -66,17 +65,14 @@ roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. fields(config) -> - [{server, fun server/1}] ++ + [{server, server()}] ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:host_port(); -server(required) -> true; -server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; -server(converter) -> fun to_server/1; -server(desc) -> ?DESC("server"); -server(_) -> undefined. +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?MYSQL_HOST_OPTIONS). %% =================================================================== callback_mode() -> always_sync. @@ -85,7 +81,7 @@ callback_mode() -> always_sync. on_start( InstId, #{ - server := {Host, Port}, + server := Server, database := DB, username := User, password := Password, @@ -94,6 +90,7 @@ on_start( ssl := SSL } = Config ) -> + {Host, Port} = emqx_schema:parse_server(Server, ?MYSQL_HOST_OPTIONS), ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, @@ -239,11 +236,6 @@ reconn_interval(false) -> false. connect(Options) -> mysql:start_link(Options). --spec to_server(string()) -> - {inet:ip_address() | inet:hostname(), pos_integer()}. -to_server(Str) -> - emqx_connector_schema_lib:parse_server(Str, ?MYSQL_HOST_OPTIONS). - init_prepare(State = #{prepare_statement := Prepares, poolname := PoolName}) -> case maps:size(Prepares) of 0 -> @@ -416,20 +408,34 @@ on_sql_query( LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, ?TRACE("QUERY", "mysql_connector_received", LogMeta), Worker = ecpool:get_client(PoolName), - {ok, Conn} = ecpool_worker:client(Worker), - ?tp( - mysql_connector_send_query, - #{sql_func => SQLFunc, sql_or_key => SQLOrKey, data => Data} - ), + case ecpool_worker:client(Worker) of + {ok, Conn} -> + ?tp( + mysql_connector_send_query, + #{sql_func => SQLFunc, sql_or_key => SQLOrKey, data => Data} + ), + do_sql_query(SQLFunc, Conn, SQLOrKey, Data, Timeout, LogMeta); + {error, disconnected} -> + ?SLOG( + error, + LogMeta#{ + msg => "mysql_connector_do_sql_query_failed", + reason => worker_is_disconnected + } + ), + {error, {recoverable_error, disconnected}} + end. + +do_sql_query(SQLFunc, Conn, SQLOrKey, Data, Timeout, LogMeta) -> try mysql:SQLFunc(Conn, SQLOrKey, Data, Timeout) of - {error, disconnected} = Result -> + {error, disconnected} -> ?SLOG( error, LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected} ), %% kill the poll worker to trigger reconnection _ = exit(Conn, restart), - Result; + {error, {recoverable_error, disconnected}}; {error, not_prepared} = Error -> ?tp( mysql_connector_prepare_query_failed, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 71dd2bbeb..4b565a614 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("epgsql/include/epgsql.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([roots/0, fields/1]). @@ -31,6 +32,7 @@ on_start/2, on_stop/2, on_query/3, + on_batch_query/3, on_get_status/2 ]). @@ -38,41 +40,51 @@ -export([ query/3, - prepared_query/3 + prepared_query/3, + execute_batch/3 ]). -export([do_get_status/1]). -define(PGSQL_HOST_OPTIONS, #{ - host_type => inet_addr, default_port => ?PGSQL_DEFAULT_PORT }). +-type prepares() :: #{atom() => binary()}. +-type params_tokens() :: #{atom() => list()}. + +-type state() :: + #{ + poolname := atom(), + auto_reconnect := boolean(), + prepare_sql := prepares(), + params_tokens := params_tokens(), + prepare_statement := epgsql:statement() + }. + %%===================================================================== roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. fields(config) -> - [{server, fun server/1}] ++ + [{server, server()}] ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:host_port(); -server(required) -> true; -server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; -server(converter) -> fun to_server/1; -server(desc) -> ?DESC("server"); -server(_) -> undefined. +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?PGSQL_HOST_OPTIONS). %% =================================================================== callback_mode() -> always_sync. +-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ - server := {Host, Port}, + server := Server, database := DB, username := User, password := Password, @@ -81,6 +93,7 @@ on_start( ssl := SSL } = Config ) -> + {Host, Port} = emqx_schema:parse_server(Server, ?PGSQL_HOST_OPTIONS), ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, @@ -90,7 +103,7 @@ on_start( case maps:get(enable, SSL) of true -> [ - {ssl, true}, + {ssl, required}, {ssl_opts, emqx_tls_lib:to_client_opts(SSL)} ]; false -> @@ -103,13 +116,21 @@ on_start( {password, emqx_secret:wrap(Password)}, {database, DB}, {auto_reconnect, reconn_interval(AutoReconn)}, - {pool_size, PoolSize}, - {prepare_statement, maps:to_list(maps:get(prepare_statement, Config, #{}))} + {pool_size, PoolSize} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), + Prepares = parse_prepare_sql(Config), + InitState = #{poolname => PoolName, auto_reconnect => AutoReconn, prepare_statement => #{}}, + State = maps:merge(InitState, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of - ok -> {ok, #{poolname => PoolName, auto_reconnect => AutoReconn}}; - {error, Reason} -> {error, Reason} + ok -> + {ok, init_prepare(State)}; + {error, Reason} -> + ?tp( + pgsql_connector_start_failed, + #{error => Reason} + ), + {error, Reason} end. on_stop(InstId, #{poolname := PoolName}) -> @@ -119,37 +140,145 @@ on_stop(InstId, #{poolname := PoolName}) -> }), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {Type, NameOrSQL}, #{poolname := _PoolName} = State) -> - on_query(InstId, {Type, NameOrSQL, []}, State); -on_query(InstId, {Type, NameOrSQL, Params}, #{poolname := PoolName} = State) -> +on_query(InstId, {TypeOrKey, NameOrSQL}, #{poolname := _PoolName} = State) -> + on_query(InstId, {TypeOrKey, NameOrSQL, []}, State); +on_query( + InstId, + {TypeOrKey, NameOrSQL, Params}, + #{poolname := PoolName} = State +) -> ?SLOG(debug, #{ msg => "postgresql connector received sql query", connector => InstId, + type => TypeOrKey, sql => NameOrSQL, state => State }), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Params]}, no_handover) of + Type = pgsql_query_type(TypeOrKey), + {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), + on_sql_query(InstId, PoolName, Type, NameOrSQL2, Data). + +pgsql_query_type(sql) -> + query; +pgsql_query_type(query) -> + query; +pgsql_query_type(prepared_query) -> + prepared_query; +%% for bridge +pgsql_query_type(_) -> + pgsql_query_type(prepared_query). + +on_batch_query( + InstId, + BatchReq, + #{poolname := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State +) -> + case BatchReq of + [{Key, _} = Request | _] -> + BinKey = to_bin(Key), + case maps:get(BinKey, Tokens, undefined) of + undefined -> + Log = #{ + connector => InstId, + first_request => Request, + state => State, + msg => "batch prepare not implemented" + }, + ?SLOG(error, Log), + {error, batch_prepare_not_implemented}; + TokenList -> + {_, Datas} = lists:unzip(BatchReq), + Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas], + St = maps:get(BinKey, Sts), + {_Column, Results} = on_sql_query(InstId, PoolName, execute_batch, St, Datas2), + %% this local function only suits for the result of batch insert + TransResult = fun + Trans([{ok, Count} | T], Acc) -> + Trans(T, Acc + Count); + Trans([{error, _} = Error | _], _Acc) -> + Error; + Trans([], Acc) -> + {ok, Acc} + end, + + TransResult(Results, 0) + end; + _ -> + Log = #{ + connector => InstId, + request => BatchReq, + state => State, + msg => "invalid request" + }, + ?SLOG(error, Log), + {error, invalid_request} + end. + +proc_sql_params(query, SQLOrKey, Params, _State) -> + {SQLOrKey, Params}; +proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> + {SQLOrKey, Params}; +proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> + Key = to_bin(TypeOrKey), + case maps:get(Key, ParamsTokens, undefined) of + undefined -> + {SQLOrData, Params}; + Tokens -> + {Key, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)} + end. + +on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> + Result = ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, no_handover), + case Result of {error, Reason} -> ?SLOG(error, #{ msg => "postgresql connector do sql query failed", connector => InstId, + type => Type, sql => NameOrSQL, reason => Reason }); _ -> + ?tp( + pgsql_connector_query_return, + #{result => Result} + ), ok end, Result. -on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn}) -> +on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) -> case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of - true -> connected; - false -> conn_status(AutoReconn) + true -> + case do_check_prepares(State) of + ok -> + connected; + {ok, NState} -> + %% return new state with prepared statements + {connected, NState}; + false -> + %% do not log error, it is logged in prepare_sql_to_conn + conn_status(AutoReconn) + end; + false -> + conn_status(AutoReconn) end. do_get_status(Conn) -> ok == element(1, epgsql:squery(Conn, "SELECT count(1) AS T")). +do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> + ok; +do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepares}}) -> + %% retry to prepare + case prepare_sql(Prepares, PoolName) of + {ok, Sts} -> + %% remove the error + {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}; + _Error -> + false + end. + %% =================================================================== conn_status(_AutoReconn = true) -> connecting; conn_status(_AutoReconn = false) -> disconnected. @@ -161,13 +290,9 @@ connect(Opts) -> Host = proplists:get_value(host, Opts), Username = proplists:get_value(username, Opts), Password = emqx_secret:unwrap(proplists:get_value(password, Opts)), - PrepareStatement = proplists:get_value(prepare_statement, Opts), case epgsql:connect(Host, Username, Password, conn_opts(Opts)) of - {ok, Conn} -> - case parse(Conn, PrepareStatement) of - ok -> {ok, Conn}; - {error, Reason} -> {error, Reason} - end; + {ok, _Conn} = Ok -> + Ok; {error, Reason} -> {error, Reason} end. @@ -178,15 +303,8 @@ query(Conn, SQL, Params) -> prepared_query(Conn, Name, Params) -> epgsql:prepared_query2(Conn, Name, Params). -parse(_Conn, []) -> - ok; -parse(Conn, [{Name, Query} | More]) -> - case epgsql:parse2(Conn, Name, Query, []) of - {ok, _Statement} -> - parse(Conn, More); - Other -> - Other - end. +execute_batch(Conn, Statement, Params) -> + epgsql:execute_batch(Conn, Statement, Params). conn_opts(Opts) -> conn_opts(Opts, []). @@ -210,10 +328,90 @@ conn_opts([Opt = {ssl_opts, _} | Opts], Acc) -> conn_opts([_Opt | Opts], Acc) -> conn_opts(Opts, Acc). -%% =================================================================== -%% typereflt funcs +parse_prepare_sql(Config) -> + SQL = + case maps:get(prepare_statement, Config, undefined) of + undefined -> + case maps:get(sql, Config, undefined) of + undefined -> #{}; + Template -> #{<<"send_message">> => Template} + end; + Any -> + Any + end, + parse_prepare_sql(maps:to_list(SQL), #{}, #{}). --spec to_server(string()) -> - {inet:ip_address() | inet:hostname(), pos_integer()}. -to_server(Str) -> - emqx_connector_schema_lib:parse_server(Str, ?PGSQL_HOST_OPTIONS). +parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> + {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, '$n'), + parse_prepare_sql( + T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} + ); +parse_prepare_sql([], Prepares, Tokens) -> + #{ + prepare_sql => Prepares, + params_tokens => Tokens + }. + +init_prepare(State = #{prepare_sql := Prepares, poolname := PoolName}) -> + case maps:size(Prepares) of + 0 -> + State; + _ -> + case prepare_sql(Prepares, PoolName) of + {ok, Sts} -> + State#{prepare_statement := Sts}; + Error -> + LogMeta = #{ + msg => <<"PostgreSQL init prepare statement failed">>, error => Error + }, + ?SLOG(error, LogMeta), + %% mark the prepare_sqlas failed + State#{prepare_sql => {error, Prepares}} + end + end. + +prepare_sql(Prepares, PoolName) when is_map(Prepares) -> + prepare_sql(maps:to_list(Prepares), PoolName); +prepare_sql(Prepares, PoolName) -> + case do_prepare_sql(Prepares, PoolName) of + {ok, _Sts} = Ok -> + %% prepare for reconnect + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + Ok; + Error -> + Error + end. + +do_prepare_sql(Prepares, PoolName) -> + do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + +do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> + {ok, Conn} = ecpool_worker:client(Worker), + case prepare_sql_to_conn(Conn, Prepares) of + {ok, Sts} -> + do_prepare_sql(T, Prepares, PoolName, Sts); + Error -> + Error + end; +do_prepare_sql([], _Prepares, _PoolName, LastSts) -> + {ok, LastSts}. + +prepare_sql_to_conn(Conn, Prepares) -> + prepare_sql_to_conn(Conn, Prepares, #{}). + +prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; +prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> + LogMeta = #{msg => "PostgreSQL Prepare Statement", name => Key, prepare_sql => SQL}, + ?SLOG(info, LogMeta), + case epgsql:parse2(Conn, Key, SQL, []) of + {ok, Statement} -> + prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); + {error, Error} = Other -> + ?SLOG(error, LogMeta#{msg => "PostgreSQL parse failed", error => Error}), + Other + end. + +to_bin(Bin) when is_binary(Bin) -> + Bin; +to_bin(Atom) when is_atom(Atom) -> + erlang:atom_to_binary(Atom). diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 5a77ba6ab..726af2d9b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,7 +41,6 @@ %% redis host don't need parse -define(REDIS_HOST_OPTIONS, #{ - host_type => hostname, default_port => ?REDIS_DEFAULT_PORT }). @@ -61,11 +60,11 @@ roots() -> fields(single) -> [ - {server, fun server/1}, + {server, server()}, {redis_type, #{ type => single, default => single, - required => true, + required => false, desc => ?DESC("single") }} ] ++ @@ -73,11 +72,11 @@ fields(single) -> emqx_connector_schema_lib:ssl_fields(); fields(cluster) -> [ - {servers, fun servers/1}, + {servers, servers()}, {redis_type, #{ type => cluster, default => cluster, - required => true, + required => false, desc => ?DESC("cluster") }} ] ++ @@ -85,11 +84,11 @@ fields(cluster) -> emqx_connector_schema_lib:ssl_fields(); fields(sentinel) -> [ - {servers, fun servers/1}, + {servers, servers()}, {redis_type, #{ type => sentinel, default => sentinel, - required => true, + required => false, desc => ?DESC("sentinel") }}, {sentinel, #{ @@ -101,21 +100,16 @@ fields(sentinel) -> redis_fields() ++ emqx_connector_schema_lib:ssl_fields(). -server(type) -> emqx_schema:host_port(); -server(required) -> true; -server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; -server(converter) -> fun to_server_raw/1; -server(desc) -> ?DESC("server"); -server(_) -> undefined. +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?REDIS_HOST_OPTIONS). -servers(type) -> list(); -servers(required) -> true; -servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; -servers(converter) -> fun to_servers_raw/1; -servers(desc) -> ?DESC("servers"); -servers(_) -> undefined. +servers() -> + Meta = #{desc => ?DESC("servers")}, + emqx_schema:servers_sc(Meta, ?REDIS_HOST_OPTIONS). %% =================================================================== + callback_mode() -> always_sync. on_start( @@ -132,11 +126,13 @@ on_start( connector => InstId, config => Config }), - Servers = + ConfKey = case Type of - single -> [{servers, [maps:get(server, Config)]}]; - _ -> [{servers, maps:get(servers, Config)}] + single -> server; + _ -> servers end, + Servers0 = maps:get(ConfKey, Config), + Servers = [{servers, emqx_schema:parse_servers(Servers0, ?REDIS_HOST_OPTIONS)}], Database = case Type of cluster -> []; @@ -299,25 +295,3 @@ redis_fields() -> }}, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. - --spec to_server_raw(string()) -> - {string(), pos_integer()}. -to_server_raw(Server) -> - emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS). - --spec to_servers_raw(string()) -> - [{string(), pos_integer()}]. -to_servers_raw(Servers) -> - lists:map( - fun(Server) -> - emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS) - end, - string:tokens(str(Servers), ", ") - ). - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 7bcfb6d21..2364aeeaa 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,10 +25,6 @@ prepare_statement_fields/0 ]). --export([ - parse_server/2 -]). - -export([ pool_size/1, database/1, @@ -111,53 +107,3 @@ auto_reconnect(type) -> boolean(); auto_reconnect(desc) -> ?DESC("auto_reconnect"); auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. - -parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) -> - case string:tokens(str(Str), ": ") of - [Ip, Port] -> - {parse_ip(Ip), parse_port(Port)}; - [Ip] -> - {parse_ip(Ip), DefaultPort}; - _ -> - throw("Bad server schema") - end; -parse_server(Str, #{host_type := hostname, default_port := DefaultPort}) -> - case string:tokens(str(Str), ": ") of - [Hostname, Port] -> - {Hostname, parse_port(Port)}; - [Hostname] -> - {Hostname, DefaultPort}; - _ -> - throw("Bad server schema") - end; -parse_server(_, _) -> - throw("Invalid Host"). - -parse_ip(Str) -> - case inet:parse_address(Str) of - {ok, R} -> - R; - _ -> - %% check is a rfc1035's hostname - case inet_parse:domain(Str) of - true -> - Str; - _ -> - throw("Bad IP or Host") - end - end. - -parse_port(Port) -> - try - list_to_integer(Port) - catch - _:_ -> - throw("Bad port number") - end. - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 9c0133ac9..54dc0e022 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index f6333bbcd..13516813f 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/src/emqx_connector_utils.erl b/apps/emqx_connector/src/emqx_connector_utils.erl index 94b12921d..6000f6be5 100644 --- a/apps/emqx_connector/src/emqx_connector_utils.erl +++ b/apps/emqx_connector/src/emqx_connector_utils.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_connector_utils). -export([split_insert_sql/1]). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index f1ecbf68c..870f9acfc 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,15 +51,15 @@ start(Config) -> Parent = self(), - {Host, Port} = maps:get(server, Config), + ServerStr = iolist_to_binary(maps:get(server, Config)), + {Server, Port} = emqx_connector_mqtt_schema:parse_server(ServerStr), Mountpoint = maps:get(receive_mountpoint, Config, undefined), Subscriptions = maps:get(subscriptions, Config, undefined), Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Subscriptions), - ServerStr = ip_port_to_server_str(Host, Port), Handlers = make_hdlr(Parent, Vars, #{server => ServerStr}), Config1 = Config#{ msg_handler => Handlers, - host => Host, + host => Server, port => Port, force_ping => true, proto_ver => maps:get(proto_ver, Config, v4) @@ -234,11 +234,3 @@ printable_maps(Headers) -> #{}, Headers ). - -ip_port_to_server_str(Host, Port) -> - HostStr = - case inet:ntoa(Host) of - {error, einval} -> Host; - IPStr -> IPStr - end, - list_to_binary(io_lib:format("~s:~w", [HostStr, Port])). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 0d03465d3..bdd516db6 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 1c9f66d21..5e833aa99 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,13 +25,16 @@ namespace/0, roots/0, fields/1, - desc/1 + desc/1, + parse_server/1 ]). -import(emqx_schema, [mk_duration/2]). -import(hoconsc, [mk/2, ref/2]). +-define(MQTT_HOST_OPTS, #{default_port => 1883}). + namespace() -> "connector-mqtt". roots() -> @@ -67,14 +70,7 @@ fields("server_configs") -> desc => ?DESC("mode") } )}, - {server, - mk( - emqx_schema:host_port(), - #{ - required => true, - desc => ?DESC("server") - } - )}, + {server, emqx_schema:servers_sc(#{desc => ?DESC("server")}, ?MQTT_HOST_OPTS)}, {clientid_prefix, mk(binary(), #{required => false, desc => ?DESC("clientid_prefix")})}, {reconnect_interval, mk_duration( @@ -299,3 +295,6 @@ desc(_) -> qos() -> hoconsc:union([emqx_schema:qos(), binary()]). + +parse_server(Str) -> + emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index fe359437c..ba2162993 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl index 87ce70d59..3105c95ea 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -67,3 +67,13 @@ t_lookup_jwt_missing(_Config) -> ResourceId = <<"resource id">>, ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(ResourceId)), ok. + +t_delete_jwt(_Config) -> + TId = ?JWT_TABLE, + JWT = <<"some jwt">>, + ResourceId = <<"resource id">>, + true = insert_jwt(TId, ResourceId, JWT), + {ok, _} = emqx_connector_jwt:lookup_jwt(ResourceId), + ?assertEqual(ok, emqx_connector_jwt:delete_jwt(TId, ResourceId)), + ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)), + ok. diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl index 74075917e..eb104801c 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -81,7 +81,7 @@ t_create_success(_Config) -> receive {Ref, token_created} -> ok - after 1_000 -> + after 5_000 -> ct:fail( "should have confirmed token creation; msgs: ~0p", [process_info(self(), messages)] @@ -127,8 +127,12 @@ t_unknown_error(_Config) -> 1_000 ), fun(Trace) -> + %% there seems to be some occasions when empty_key is + %% returned instead. ?assertMatch( - [#{error := {invalid_private_key, some_strange_error}}], + [#{error := Error}] when + Error =:= {invalid_private_key, some_strange_error} orelse + Error =:= empty_key, ?of_kind(connector_jwt_worker_startup_error, Trace) ), ok @@ -186,14 +190,30 @@ t_refresh(_Config) -> {ok, SecondJWT} = emqx_connector_jwt:lookup_jwt(Table, ResourceId), ?assertNot(is_expired(SecondJWT)), ?assert(is_expired(FirstJWT)), - {FirstJWT, SecondJWT} + %% check yet another refresh to ensure the timer was properly + %% reset. + ?block_until( + #{ + ?snk_kind := connector_jwt_worker_refresh, + jwt := JWT1 + } when + JWT1 =/= SecondJWT andalso + JWT1 =/= FirstJWT, + 15_000 + ), + {ok, ThirdJWT} = emqx_connector_jwt:lookup_jwt(Table, ResourceId), + ?assertNot(is_expired(ThirdJWT)), + ?assert(is_expired(SecondJWT)), + {FirstJWT, SecondJWT, ThirdJWT} end, - fun({FirstJWT, SecondJWT}, Trace) -> + fun({FirstJWT, SecondJWT, ThirdJWT}, Trace) -> ?assertMatch( - [_, _ | _], + [_, _, _ | _], ?of_kind(connector_jwt_worker_token_stored, Trace) ), ?assertNotEqual(FirstJWT, SecondJWT), + ?assertNotEqual(SecondJWT, ThirdJWT), + ?assertNotEqual(FirstJWT, ThirdJWT), ok end ), @@ -289,7 +309,7 @@ t_lookup_badarg(_Config) -> t_start_supervised_worker(_Config) -> {ok, _} = emqx_connector_jwt_sup:start_link(), - Config = #{resource_id := ResourceId} = generate_config(), + Config = #{resource_id := ResourceId, table := TId} = generate_config(), {ok, Pid} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config), Ref = emqx_connector_jwt_worker:ensure_jwt(Pid), receive @@ -300,6 +320,7 @@ t_start_supervised_worker(_Config) -> end, MRef = monitor(process, Pid), ?assert(is_process_alive(Pid)), + ?assertMatch({ok, _}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)), ok = emqx_connector_jwt_sup:ensure_worker_deleted(ResourceId), receive {'DOWN', MRef, process, Pid, _} -> @@ -307,6 +328,11 @@ t_start_supervised_worker(_Config) -> after 1_000 -> ct:fail("timeout") end, + %% ensure it cleans up its own tokens to avoid leakage when + %% probing/testing rule resources. + ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)), + %% ensure the specs are removed from the supervision tree. + ?assertEqual([], supervisor:which_children(emqx_connector_jwt_sup)), ok. t_start_supervised_worker_already_started(_Config) -> @@ -322,9 +348,9 @@ t_start_supervised_worker_already_present(_Config) -> Config = #{resource_id := ResourceId} = generate_config(), {ok, Pid0} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config), Ref = monitor(process, Pid0), - exit(Pid0, {shutdown, normal}), + exit(Pid0, kill), receive - {'DOWN', Ref, process, Pid0, {shutdown, normal}} -> ok + {'DOWN', Ref, process, Pid0, killed} -> ok after 1_000 -> error(worker_didnt_stop) end, {ok, Pid1} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config), diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl index 5473463ec..2be30466c 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl @@ -1,5 +1,5 @@ % %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. % %% % %% Licensed under the Apache License, Version 2.0 (the "License"); % %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_mongo_tests.erl b/apps/emqx_connector/test/emqx_connector_mongo_tests.erl index 7978ed289..3cfe52b98 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,151 +18,135 @@ -include_lib("eunit/include/eunit.hrl"). --define(DEFAULT_MONGO_PORT, 27017). +srv_record_test() -> + with_dns_mock( + fun normal_dns_resolution_mock/2, + fun() -> + Single = single_config(), + Rs = simple_rs_config(), + Hosts = [ + <<"cluster0-shard-00-02.zkemc.mongodb.net:27017">>, + <<"cluster0-shard-00-01.zkemc.mongodb.net:27017">>, + <<"cluster0-shard-00-00.zkemc.mongodb.net:27017">> + ], + ?assertMatch( + #{ + hosts := Hosts, + auth_source := <<"admin">> + }, + resolve(Single) + ), + ?assertMatch( + #{ + hosts := Hosts, + auth_source := <<"admin">>, + replica_set_name := <<"atlas-wrnled-shard-0">> + }, + resolve(Rs) + ), + ok + end + ). -%%------------------------------------------------------------------------------ -%% Helper fns -%%------------------------------------------------------------------------------ +empty_srv_record_test() -> + with_dns_mock( + bad_srv_record_mock(_DnsResolution = []), + fun() -> + ?assertThrow(#{reason := "failed_to_resolve_srv_record"}, resolve(simple_rs_config())) + end + ). -%%------------------------------------------------------------------------------ -%% Test cases -%%------------------------------------------------------------------------------ +empty_txt_record_test() -> + with_dns_mock( + bad_txt_record_mock(_DnsResolution = []), + fun() -> + Config = resolve(single_config()), + ?assertNot(maps:is_key(auth_source, Config)), + ?assertNot(maps:is_key(replica_set_name, Config)), + ok + end + ). -to_servers_raw_test_() -> +multiple_txt_records_test() -> + with_dns_mock( + bad_txt_record_mock(_DnsResolution = [1, 2]), + fun() -> + ?assertThrow(#{reason := "multiple_txt_records"}, resolve(simple_rs_config())) + end + ). + +bad_query_string_test() -> + with_dns_mock( + bad_txt_record_mock(_DnsResolution = [["%-111"]]), + fun() -> + ?assertThrow(#{reason := "bad_txt_record_resolution"}, resolve(simple_rs_config())) + end + ). + +resolve(Config) -> + emqx_connector_mongo:maybe_resolve_srv_and_txt_records(Config). + +checked_config(Hocon) -> + {ok, Config} = hocon:binary(Hocon), + hocon_tconf:check_plain( + emqx_connector_mongo, + #{<<"config">> => Config}, + #{atom_key => true} + ). + +simple_rs_config() -> + #{config := Rs} = checked_config( + "mongo_type = rs\n" + "servers = \"cluster0.zkemc.mongodb.net:27017\"\n" + "srv_record = true\n" + "database = foobar\n" + "replica_set_name = configured_replicaset_name\n" + ), + Rs. + +single_config() -> + #{config := Single} = checked_config( + "mongo_type = single\n" + "server = \"cluster0.zkemc.mongodb.net:27017,cluster0.zkemc.mongodb.net:27017\"\n" + "srv_record = true\n" + "database = foobar\n" + ), + Single. + +normal_srv_resolution() -> [ - {"single server, binary, no port", - ?_test( - ?assertEqual( - [{"localhost", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw(<<"localhost">>) - ) - )}, - {"single server, string, no port", - ?_test( - ?assertEqual( - [{"localhost", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw("localhost") - ) - )}, - {"single server, list(binary), no port", - ?_test( - ?assertEqual( - [{"localhost", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw([<<"localhost">>]) - ) - )}, - {"single server, list(string), no port", - ?_test( - ?assertEqual( - [{"localhost", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw(["localhost"]) - ) - )}, - %%%%%%%%% - {"single server, binary, with port", - ?_test( - ?assertEqual( - [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw(<<"localhost:9999">>) - ) - )}, - {"single server, string, with port", - ?_test( - ?assertEqual( - [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw("localhost:9999") - ) - )}, - {"single server, list(binary), with port", - ?_test( - ?assertEqual( - [{"localhost", 9999}], - emqx_connector_mongo:to_servers_raw([<<"localhost:9999">>]) - ) - )}, - {"single server, list(string), with port", - ?_test( - ?assertEqual( - [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw(["localhost:9999"]) - ) - )}, - %%%%%%%%% - {"multiple servers, string, no port", - ?_test( - ?assertEqual( - [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw("host1, host2") - ) - )}, - {"multiple servers, binary, no port", - ?_test( - ?assertEqual( - [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw(<<"host1, host2">>) - ) - )}, - {"multiple servers, list(string), no port", - ?_test( - ?assertEqual( - [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw(["host1", "host2"]) - ) - )}, - {"multiple servers, list(binary), no port", - ?_test( - ?assertEqual( - [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}], - emqx_connector_mongo:to_servers_raw([<<"host1">>, <<"host2">>]) - ) - )}, - %%%%%%%%% - {"multiple servers, string, with port", - ?_test( - ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], - emqx_connector_mongo:to_servers_raw("host1:1234, host2:2345") - ) - )}, - {"multiple servers, binary, with port", - ?_test( - ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], - emqx_connector_mongo:to_servers_raw(<<"host1:1234, host2:2345">>) - ) - )}, - {"multiple servers, list(string), with port", - ?_test( - ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], - emqx_connector_mongo:to_servers_raw(["host1:1234", "host2:2345"]) - ) - )}, - {"multiple servers, list(binary), with port", - ?_test( - ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], - emqx_connector_mongo:to_servers_raw([<<"host1:1234">>, <<"host2:2345">>]) - ) - )}, - %%%%%%%% - {"multiple servers, invalid list(string)", - ?_test( - ?assertThrow( - _, - emqx_connector_mongo:to_servers_raw(["host1, host2"]) - ) - )}, - {"multiple servers, invalid list(binary)", - ?_test( - ?assertThrow( - _, - emqx_connector_mongo:to_servers_raw([<<"host1, host2">>]) - ) - )}, - %% TODO: handle this case?? - {"multiple servers, mixed list(binary|string)", - ?_test( - ?assertThrow( - _, - emqx_connector_mongo:to_servers_raw([<<"host1">>, "host2"]) - ) - )} + {0, 0, 27017, "cluster0-shard-00-02.zkemc.mongodb.net"}, + {0, 0, 27017, "cluster0-shard-00-01.zkemc.mongodb.net"}, + {0, 0, 27017, "cluster0-shard-00-00.zkemc.mongodb.net"} ]. + +normal_txt_resolution() -> + [["authSource=admin&replicaSet=atlas-wrnled-shard-0"]]. + +normal_dns_resolution_mock("_mongodb._tcp.cluster0.zkemc.mongodb.net", srv) -> + normal_srv_resolution(); +normal_dns_resolution_mock("cluster0.zkemc.mongodb.net", txt) -> + normal_txt_resolution(). + +bad_srv_record_mock(DnsResolution) -> + fun("_mongodb._tcp.cluster0.zkemc.mongodb.net", srv) -> + DnsResolution + end. + +bad_txt_record_mock(DnsResolution) -> + fun + ("_mongodb._tcp.cluster0.zkemc.mongodb.net", srv) -> + normal_srv_resolution(); + ("cluster0.zkemc.mongodb.net", txt) -> + DnsResolution + end. + +with_dns_mock(MockFn, TestFn) -> + meck:new(emqx_connector_lib, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_connector_lib, resolve_dns, MockFn), + try + TestFn() + after + meck:unload(emqx_connector_lib) + end, + ok. diff --git a/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl index 2bb9abd84..88c8b5218 100644 --- a/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,8 +50,8 @@ send_and_ack_test() -> try Max = 1, Batch = lists:seq(1, Max), - {ok, Conn} = emqx_connector_mqtt_mod:start(#{server => {{127, 0, 0, 1}, 1883}}), - % %% return last packet id as batch reference + {ok, Conn} = emqx_connector_mqtt_mod:start(#{server => "127.0.0.1:1883"}), + %% return last packet id as batch reference {ok, _AckRef} = emqx_connector_mqtt_mod:send(Conn, Batch), ok = emqx_connector_mqtt_mod:stop(Conn) diff --git a/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl index 3f0374d26..49bff7bbc 100644 --- a/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl index 3a41cc0b1..dc5826766 100644 --- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl @@ -1,5 +1,5 @@ % %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. % %% % %% Licensed under the Apache License, Version 2.0 (the "License"); % %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl index 10293a241..2f77ca38d 100644 --- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl @@ -1,5 +1,5 @@ % %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. % %% % %% Licensed under the Apache License, Version 2.0 (the "License"); % %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index e67dced2f..a1d8fe9d5 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -1,5 +1,5 @@ % %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. % %% % %% Licensed under the Apache License, Version 2.0 (the "License"); % %% you may not use this file except in compliance with the License. @@ -36,22 +36,16 @@ groups() -> []. init_per_suite(Config) -> - case - emqx_common_test_helpers:is_all_tcp_servers_available( - [ - {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT}, - {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT} - ] - ) - of - true -> - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - Config; - false -> - {skip, no_redis} - end. + Checks = + case os:getenv("IS_CI") of + "yes" -> 10; + _ -> 1 + end, + ok = wait_for_redis(Checks), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_resource]), @@ -63,9 +57,27 @@ init_per_testcase(_, Config) -> end_per_testcase(_, _Config) -> ok. -% %%------------------------------------------------------------------------------ -% %% Testcases -% %%------------------------------------------------------------------------------ +wait_for_redis(0) -> + throw(timeout); +wait_for_redis(Checks) -> + case + emqx_common_test_helpers:is_all_tcp_servers_available( + [ + {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT}, + {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT} + ] + ) + of + true -> + ok; + false -> + timer:sleep(1000), + wait_for_redis(Checks - 1) + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ t_single_lifecycle(_Config) -> perform_lifecycle_check( diff --git a/apps/emqx_connector/test/emqx_connector_test_helpers.erl b/apps/emqx_connector/test/emqx_connector_test_helpers.erl index ea3380e85..8a01f8991 100644 --- a/apps/emqx_connector/test/emqx_connector_test_helpers.erl +++ b/apps/emqx_connector/test/emqx_connector_test_helpers.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl index 3d5a4490a..b68ebcbba 100644 --- a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl +++ b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index e6758d0de..872cfdf26 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -199,23 +199,12 @@ its own from which a browser should permit loading resources.""" } bootstrap_users_file { desc { - en: "Initialize users file." - zh: "初始化用户文件" + en: "Deprecated, use api_key.bootstrap_file" + zh: "已废弃，请使用 api_key.bootstrap_file" } label { - en: """Is used to add an administrative user to Dashboard when emqx is first launched, - the format is: - ``` - username1:password1 - username2:password2 - ``` -""" - zh: """用于在首次启动 emqx 时，为 Dashboard 添加管理用户，其格式为： - ``` - username1:password1 - username2:password2 - ``` -""" + en: """Deprecated""" + zh: """已废弃""" } } } diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index a4af8ae1b..fe727ef7f 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 56bd64c74..2698d5534 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index e032eb28b..f15467658 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index b3b9c1023..e36c2628b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,8 +51,7 @@ -export([ add_default_user/0, - default_username/0, - add_bootstrap_users/0 + default_username/0 ]). -type emqx_admin() :: #?ADMIN{}. @@ -85,21 +84,6 @@ mnesia(boot) -> add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). --spec add_bootstrap_users() -> ok | {error, _}. -add_bootstrap_users() -> - case emqx:get_config([dashboard, bootstrap_users_file], undefined) of - undefined -> - ok; - File -> - case mnesia:table_info(?ADMIN, size) of - 0 -> - ?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}), - add_bootstrap_users(File); - _ -> - ok - end - end. - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -311,44 +295,3 @@ add_default_user(Username, Password) -> [] -> add_user(Username, Password, <<"administrator">>); _ -> {ok, default_user_exists} end. - -add_bootstrap_users(File) -> - case file:open(File, [read]) of - {ok, Dev} -> - {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), - try - load_bootstrap_user(Dev, MP) - catch - Type:Reason -> - {error, {Type, Reason}} - after - file:close(Dev) - end; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "failed to open the dashboard bootstrap users file", - file => File, - reason => Reason - }), - Error - end. - -load_bootstrap_user(Dev, MP) -> - case file:read_line(Dev) of - {ok, Line} -> - case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of - {match, [[Username, Password]]} -> - case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of - {ok, _} -> - load_bootstrap_user(Dev, MP); - Error -> - Error - end; - _ -> - load_bootstrap_user(Dev, MP) - end; - eof -> - ok; - Error -> - Error - end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index c0ea5c538..9facac59c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index b40f6f1f6..2c3f9b8bc 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,13 +31,8 @@ start(_StartType, _StartArgs) -> case emqx_dashboard:start_listeners() of ok -> emqx_dashboard_cli:load(), - case emqx_dashboard_admin:add_bootstrap_users() of - ok -> - {ok, _} = emqx_dashboard_admin:add_default_user(), - {ok, Sup}; - Error -> - Error - end; + {ok, _} = emqx_dashboard_admin:add_default_user(), + {ok, Sup}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl index 8671eb5b6..0e7489d2d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl index 6f99b761b..52a915ecb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code.erl index e12cc1fc4..f2f653e62 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index 2605ad91e..131b08313 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 3c53c2e3f..112b3ad58 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl index 4188950a6..fe51b7d28 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl index 910e7f1dd..23ac4f35e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index 01f53b2b2..f8b0918be 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_dashboard_monitor_api). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 306720e4d..6742032d5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -56,7 +56,15 @@ fields("dashboard") -> {cors, fun cors/1}, {i18n_lang, fun i18n_lang/1}, {bootstrap_users_file, - ?HOCON(binary(), #{desc => ?DESC(bootstrap_users_file), required => false})} + ?HOCON( + binary(), + #{ + desc => ?DESC(bootstrap_users_file), + required => false, + default => <<>> + %% deprecated => {since, "5.1.0"} + } + )} ]; fields("listeners") -> [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl index 4482d4b9f..896b44859 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 6514ed9ef..102b95f4e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> @@ -675,8 +675,6 @@ typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; -typename_to_spec("host_port()", _Mod) -> - #{type => string, example => <<"example.host.domain:80">>}; typename_to_spec("write_syntax()", _Mod) -> #{ type => string, @@ -707,7 +705,7 @@ typename_to_spec("service_account_json()", _Mod) -> typename_to_spec("#{" ++ _, Mod) -> typename_to_spec("map()", Mod); typename_to_spec("qos()", _Mod) -> - #{type => string, enum => [0, 1, 2]}; + #{type => integer, minimum => 0, maximum => 2, example => 0}; typename_to_spec("{binary(), binary()}", _Mod) -> #{type => object, example => #{}}; typename_to_spec("comma_separated_list()", _Mod) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index a8806bb0f..e8357c458 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/src/proto/emqx_dashboard_proto_v1.erl b/apps/emqx_dashboard/src/proto/emqx_dashboard_proto_v1.erl index 7ec976702..d13dde9d0 100644 --- a/apps/emqx_dashboard/src/proto/emqx_dashboard_proto_v1.erl +++ b/apps/emqx_dashboard/src/proto/emqx_dashboard_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 003fa496e..934d6055d 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl index 8c9d36912..fefc492cc 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index b74a118d2..87a3654ac 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl index 6182327f4..b7fbf889e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index a3ae228f7..5def3c9dd 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 7f1eadcdb..6f4a0e0fd 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 5dd76acf6..5d89fb273 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_swagger_parameter_SUITE). -behaviour(minirest_api). -behaviour(hocon_schema). @@ -96,7 +112,7 @@ t_in_query(_Config) -> description => <<"QOS">>, in => query, name => qos, - schema => #{enum => [0, 1, 2], type => string} + schema => #{minimum => 0, maximum => 2, type => integer, example => 0} } ], validate("/test/in/query", Expect), diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl index 2b08c3a04..a797d3b43 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 5a807eac6..d17725e80 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_swagger_requestBody_SUITE). -behaviour(minirest_api). diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 7c51df0fa..346f4ef71 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_swagger_response_SUITE). -behaviour(minirest_api). diff --git a/apps/emqx_exhook/include/emqx_exhook.hrl b/apps/emqx_exhook/include/emqx_exhook.hrl index 7436b2a1c..591b86bf9 100644 --- a/apps/emqx_exhook/include/emqx_exhook.hrl +++ b/apps/emqx_exhook/include/emqx_exhook.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 9a6923b0c..928e9b20b 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -1,5 +1,5 @@ //------------------------------------------------------------------------------ -// Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +// Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index d08feb5e5..d81819c98 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.erl b/apps/emqx_exhook/src/emqx_exhook.erl index f01ff4e7a..e361f048b 100644 --- a/apps/emqx_exhook/src/emqx_exhook.erl +++ b/apps/emqx_exhook/src/emqx_exhook.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index c342ea66a..4d7de2866 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl index 48bfcd2e4..6e9bc5242 100644 --- a/apps/emqx_exhook/src/emqx_exhook_app.erl +++ b/apps/emqx_exhook/src/emqx_exhook_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index ddf25cfdf..8720f65ae 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_metrics.erl b/apps/emqx_exhook/src/emqx_exhook_metrics.erl index eccb35b84..44321221b 100644 --- a/apps/emqx_exhook/src/emqx_exhook_metrics.erl +++ b/apps/emqx_exhook/src/emqx_exhook_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 57f3413a9..77937a835 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 7b5e9adce..ce79dddac 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 9c89915aa..c2568d82b 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index cd49d89bb..9a2f07baa 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/proto/emqx_exhook_proto_v1.erl b/apps/emqx_exhook/src/proto/emqx_exhook_proto_v1.erl index 06127c620..616ef64fe 100644 --- a/apps/emqx_exhook/src/proto/emqx_exhook_proto_v1.erl +++ b/apps/emqx_exhook/src/proto/emqx_exhook_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index 259e1ced8..aaa8649c0 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 2b826030e..7be940a53 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index b566b7ab2..6a10143e4 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/emqx_exhook_metrics_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_metrics_SUITE.erl index 674814eab..8ccee7f5d 100644 --- a/apps/emqx_exhook/test/emqx_exhook_metrics_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index c42791f7a..075cc736c 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/LICENSE b/apps/emqx_gateway/LICENSE deleted file mode 100644 index 818096217..000000000 --- a/apps/emqx_gateway/LICENSE +++ /dev/null @@ -1,190 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 7acb5d4c2..3466ecd98 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/include/emqx_gateway_http.hrl b/apps/emqx_gateway/include/emqx_gateway_http.hrl index a3c935576..c537f7ca5 100644 --- a/apps/emqx_gateway/include/emqx_gateway_http.hrl +++ b/apps/emqx_gateway/include/emqx_gateway_http.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl index fc78c4dfe..c1a5ea2e8 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 99ac3a38f..7f37061ac 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl index 909d6f7b3..5efd2ee73 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl index df66e2854..d8cb871ef 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index f06071040..0f4c7a053 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index df5432fc3..d6b8594b1 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index 687cb7bc8..4d2479d75 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index 94961fadf..bebcef237 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl index fc0541a33..8f5028f25 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index d7a229532..99c9e0840 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -6,7 +6,7 @@ %% Copyright (c) 2015 Petr Gotthard %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl index c087dce1c..a47f610ea 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index cc451fce4..253f34d4d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index 9e7c400c3..1a0004f8c 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 06c7fc0b1..1e6c5238a 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_coap_transport). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 49573b799..59825a745 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 8587dc1dc..5e14ba9e4 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl index 2f818f653..8e7bd4046 100644 --- a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl +++ b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 6376af6ff..53403a67a 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl index 2269a55c6..0e6871db2 100644 --- a/apps/emqx_gateway/src/emqx_gateway.erl +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index e06748034..1c43340e2 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 71d0e393e..f52b26cd2 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index b26e77e83..705fccf90 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index d219d07cd..ef1c4c386 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 08bf37a47..1b4f2e0ac 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 8d774311f..cb5a16fde 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index 699320e4b..df808f295 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 25665c762..5cba1464a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl index 5c3e8bb45..f5bede084 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 5fe858fa9..07a4c1e2c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 00aa21b39..7ed44bc19 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl index 2d91622af..0f7ff4ffc 100644 --- a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 07b0d0e36..d80e3433f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl index 035c2d10f..b40c2cab8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index f94ae690f..e94510387 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl index b35cda533..50aad9445 100644 --- a/apps/emqx_gateway/src/emqx_gateway_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index c58d2f74c..e89280f14 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_sup.erl b/apps/emqx_gateway/src/emqx_gateway_sup.erl index 5c42a6d9a..4e928bbf9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 68fce7589..cee5baaa8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index be4cddcaa..301154df0 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl index 2d22db727..53f940c27 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% %% Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl index cf8ed76a7..af15ef9d3 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index 5a1932b2b..13bd49e55 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 78e70def0..0c25e5e08 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl index e4c32c7ff..cce3cbc37 100644 --- a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl +++ b/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/exproto/protos/exproto.proto b/apps/emqx_gateway/src/exproto/protos/exproto.proto index d26acf55b..ccdebff43 100644 --- a/apps/emqx_gateway/src/exproto/protos/exproto.proto +++ b/apps/emqx_gateway/src/exproto/protos/exproto.proto @@ -1,5 +1,5 @@ //------------------------------------------------------------------------------ -// Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +// Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/binary_util.erl b/apps/emqx_gateway/src/lwm2m/binary_util.erl index eabdb9c5f..68ac7a0d7 100644 --- a/apps/emqx_gateway/src/lwm2m/binary_util.erl +++ b/apps/emqx_gateway/src/lwm2m/binary_util.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(binary_util). %% copied from https://github.com/arcusfelis/binary2 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index 9bf1d77ea..2cd53d6eb 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index b6f2b870e..16d0f9630 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 3fd78b383..d0b362dda 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2016-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2016-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index ba17f2057..fa4537315 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl index 7e9d418f7..f09a8ea3d 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index 0038a1975..19cd5c25d 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl index 00459a0c1..782bbec5e 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index 53f9f0442..a4dc44f2c 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 3e30cc32a..19335768f 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl index dfb5c5bc2..1f02a1637 100644 --- a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl +++ b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl index ecde268ba..5fc08ad7f 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,7 +18,13 @@ -behaviour(gen_server). +-ifdef(TEST). +%% make rebar3 ct happy when testing with --suite path/to/module_SUITE.erl +-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). +-else. +%% make mix happy -include("src/mqttsn/include/emqx_sn.hrl"). +-endif. -include_lib("emqx/include/logger.hrl"). -export([ diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index df3b4018e..29dce90ee 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl index 4313fb952..39bd9e889 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% %% Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index a51d2f855..db730aee1 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl index 448aa8ad5..689aab8ce 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl index b636cf453..5ab2d4a05 100644 --- a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl +++ b/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/proto/emqx_gateway_api_listeners_proto_v1.erl b/apps/emqx_gateway/src/proto/emqx_gateway_api_listeners_proto_v1.erl index b64db8fe8..e56847fbe 100644 --- a/apps/emqx_gateway/src/proto/emqx_gateway_api_listeners_proto_v1.erl +++ b/apps/emqx_gateway/src/proto/emqx_gateway_api_listeners_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl b/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl index 546a8493c..29b6c7486 100644 --- a/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl +++ b/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/proto/emqx_gateway_http_proto_v1.erl b/apps/emqx_gateway/src/proto/emqx_gateway_http_proto_v1.erl index 7bd827915..caf898254 100644 --- a/apps/emqx_gateway/src/proto/emqx_gateway_http_proto_v1.erl +++ b/apps/emqx_gateway/src/proto/emqx_gateway_http_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index e698ce17f..b95bb827c 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl index b2e4de15d..47e045412 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl index 41021d490..88720c513 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index 04dd81140..c2907c262 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl index 6c81b5909..1083afe81 100644 --- a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl +++ b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_SUITE.erl index f6b32c68c..db99c3df1 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 0f0f3787f..6c1354bc0 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (C) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl index 0e863a14c..b476a40cb 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl index 25f25ba0c..b2e3ad4a7 100644 --- a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index 0bed52a76..f611988a0 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 7d58d14b6..7aac45d61 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index d9d2e0dca..d75bf80eb 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 604321897..208262f22 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index accbcd3ac..2e44415aa 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index c63760224..c66785e00 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl index ba5275e30..c5e8d9a92 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl index b9b7b38f6..77f4058e7 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index dff7b3fd4..6f6c2c45a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl index 094da93d2..0aa3172f1 100644 --- a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl index d677c749d..211315e6c 100644 --- a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index 1208a3758..cc5f7bf37 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 4682aa02e..a6791a36b 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index 1365200d0..f91bbf16e 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (C) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 5b206dffb..c40d1af55 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (C) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl index de6e3eed5..aa3fed707 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 2c8a98c03..adc1e7382 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl index a09705a68..739255e71 100644 --- a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl index ca0dfe3c1..2cf245ce2 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl index 162bc8eee..8ce9ed9fa 100644 --- a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index c77599d41..5dcef7e72 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl index b30ab8a7c..2869a8958 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl index c4d15a465..f2dfbb8e9 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_gateway/test/test_mqtt_broker.erl b/apps/emqx_gateway/test/test_mqtt_broker.erl index de209011e..497f81f76 100644 --- a/apps/emqx_gateway/test/test_mqtt_broker.erl +++ b/apps/emqx_gateway/test/test_mqtt_broker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index a704d603c..f17ab2d16 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 63c6c01ad..fdfd2b28f 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index c2adf7bf5..ec0aff55b 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine_app.erl b/apps/emqx_machine/src/emqx_machine_app.erl index bf3637567..6e7920e88 100644 --- a/apps/emqx_machine/src/emqx_machine_app.erl +++ b/apps/emqx_machine/src/emqx_machine_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index dca3f64d5..4b3e5ea7d 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine_signal_handler.erl b/apps/emqx_machine/src/emqx_machine_signal_handler.erl index cdc21ab90..03d4ca326 100644 --- a/apps/emqx_machine/src/emqx_machine_signal_handler.erl +++ b/apps/emqx_machine/src/emqx_machine_signal_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine_sup.erl b/apps/emqx_machine/src/emqx_machine_sup.erl index 14a7b4387..1045b50bf 100644 --- a/apps/emqx_machine/src/emqx_machine_sup.erl +++ b/apps/emqx_machine/src/emqx_machine_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index 23fcb7df2..314b8c705 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl index f5a52809f..91daaaa75 100644 --- a/apps/emqx_machine/src/emqx_restricted_shell.erl +++ b/apps/emqx_machine/src/emqx_restricted_shell.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,11 +45,7 @@ set_prompt_func() -> prompt_func(PropList) -> Line = proplists:get_value(history, PropList, 1), Version = emqx_release:version(), - Prefix = - case emqx_release:edition() of - ce -> "v"; - ee -> "e" - end, + Prefix = emqx_release:edition_vsn_prefix(), case is_alive() of true -> io_lib:format(<<"~ts~ts(~s)~w> ">>, [Prefix, Version, node(), Line]); false -> io_lib:format(<<"~ts~ts ~w> ">>, [Prefix, Version, Line]) diff --git a/apps/emqx_machine/src/user_default.erl b/apps/emqx_machine/src/user_default.erl index 91899be61..ce5397c26 100644 --- a/apps/emqx_machine/src/user_default.erl +++ b/apps/emqx_machine/src/user_default.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 3e0274337..691cda677 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/test/emqx_machine_tests.erl b/apps/emqx_machine/test/emqx_machine_tests.erl index c8d5fb7e8..4fa58229c 100644 --- a/apps/emqx_machine/test/emqx_machine_tests.erl +++ b/apps/emqx_machine/test/emqx_machine_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_machine/test/emqx_restricted_shell_SUITE.erl b/apps/emqx_machine/test/emqx_restricted_shell_SUITE.erl index 8ab5637ee..9f6bf5480 100644 --- a/apps/emqx_machine/test/emqx_restricted_shell_SUITE.erl +++ b/apps/emqx_machine/test/emqx_restricted_shell_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf new file mode 100644 index 000000000..eae559660 --- /dev/null +++ b/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf @@ -0,0 +1,33 @@ +emqx_mgmt_api_key_schema { + api_key { + desc { + en: """API Key, can be used to request API other than the management API key and the Dashboard user management API""" + zh: """API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" + } + label { + en: "API Key" + zh: "API 密钥" + } + } + bootstrap_file { + desc { + en: """Bootstrap file is used to add an api_key when emqx is launched, + the format is: + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ``` +""" + zh: """用于在启动 emqx 时，添加 API 密钥，其格式为： + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ``` +""" + } + label { + en: "Initialize api_key file." + zh: "API 密钥初始化文件" + } + } +} diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf index d845bff4b..4ba7e8dba 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf @@ -63,12 +63,6 @@ result of each individual message in the batch. zh: "MQTT 消息的 QoS" } } - clientid { - desc { - en: "Each message can be published as if it is done on behalf of an MQTT client whos ID can be specified in this field." - zh: "每个消息都可以带上一个 MQTT 客户端 ID，用于模拟 MQTT 客户端的发布行为。" - } - } payload { desc { en: "The MQTT message payload." diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index 19634a388..b68a9a634 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 5df8fe4df..c0cb05401 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 38b26444c..6b38e8ca0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 9f69ed3f0..893007ebf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index 80f93ffe4..88c2be89d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl similarity index 98% rename from apps/emqx_management/src/emqx_mgmt_api_app.erl rename to apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index 7050ea1af..c39b11273 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_app). +-module(emqx_mgmt_api_api_keys). -behaviour(minirest_api). diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 2eb8908c6..0a5cc3afe 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 588031fc1..7c45206fd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -65,7 +65,6 @@ {<<"ip_address">>, ip}, {<<"conn_state">>, atom}, {<<"clean_start">>, atom}, - {<<"proto_name">>, binary}, {<<"proto_ver">>, integer}, {<<"like_clientid">>, binary}, {<<"like_username">>, binary}, @@ -145,14 +144,6 @@ schema("/clients") -> required => false, description => <<"Whether the client uses a new session">> })}, - {proto_name, - hoconsc:mk(hoconsc:enum(['MQTT', 'CoAP', 'LwM2M', 'MQTT-SN']), #{ - in => query, - required => false, - description => - <<"Client protocol name, ", - "the possible values are MQTT,CoAP,LwM2M,MQTT-SN">> - })}, {proto_ver, hoconsc:mk(binary(), #{ in => query, @@ -558,8 +549,8 @@ fields(keepalive) -> ]; fields(subscribe) -> [ - {topic, hoconsc:mk(binary(), #{desc => <<"Topic">>})}, - {qos, hoconsc:mk(emqx_schema:qos(), #{desc => <<"QoS">>})}, + {topic, hoconsc:mk(binary(), #{required => true, desc => <<"Topic">>})}, + {qos, hoconsc:mk(emqx_schema:qos(), #{default => 0, desc => <<"QoS">>})}, {nl, hoconsc:mk(integer(), #{default => 0, desc => <<"No Local">>})}, {rap, hoconsc:mk(integer(), #{default => 0, desc => <<"Retain as Published">>})}, {rh, hoconsc:mk(integer(), #{default => 0, desc => <<"Retain Handling">>})} @@ -727,15 +718,18 @@ subscribe(#{clientid := ClientID, topic := Topic} = Sub) -> end. subscribe_batch(#{clientid := ClientID, topics := Topics}) -> - case lookup(#{clientid => ClientID}) of - {200, _} -> + %% We use emqx_channel instead of emqx_channel_info (used by the emqx_mgmt:lookup_client/2), + %% as the emqx_channel_info table will only be populated after the hook `client.connected` + %% has returned. So if one want to subscribe topics in this hook, it will fail. + case ets:lookup(emqx_channel, ClientID) of + [] -> + {404, ?CLIENT_ID_NOT_FOUND}; + _ -> ArgList = [ [ClientID, Topic, maps:with([qos, nl, rap, rh], Sub)] || #{topic := Topic} = Sub <- Topics ], - {200, emqx_mgmt_util:batch_operation(?MODULE, do_subscribe, ArgList)}; - {404, ?CLIENT_ID_NOT_FOUND} -> - {404, ?CLIENT_ID_NOT_FOUND} + {200, emqx_mgmt_util:batch_operation(?MODULE, do_subscribe, ArgList)} end. unsubscribe(#{clientid := ClientID, topic := Topic}) -> @@ -830,8 +824,6 @@ ms(ip_address, X) -> #{conninfo => #{peername => {X, '_'}}}; ms(clean_start, X) -> #{conninfo => #{clean_start => X}}; -ms(proto_name, X) -> - #{conninfo => #{proto_name => X}}; ms(proto_ver, X) -> #{conninfo => #{proto_ver => X}}; ms(connected_at, X) -> @@ -879,7 +871,8 @@ format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) -> ClientInfoMap2 = maps:put(node, Node, ClientInfoMap1), ClientInfoMap3 = maps:put(ip_address, IpAddress, ClientInfoMap2), ClientInfoMap4 = maps:put(port, Port, ClientInfoMap3), - ClientInfoMap = maps:put(connected, Connected, ClientInfoMap4), + ClientInfoMap5 = convert_expiry_interval_unit(ClientInfoMap4), + ClientInfoMap = maps:put(connected, Connected, ClientInfoMap5), RemoveList = [ @@ -949,6 +942,9 @@ peername_dispart({Addr, Port}) -> %% PortBinary = integer_to_binary(Port), {AddrBinary, Port}. +convert_expiry_interval_unit(ClientInfoMap = #{expiry_interval := Interval}) -> + ClientInfoMap#{expiry_interval := Interval div 1000}. + format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> #{ access => PubSub, diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index 74d5f3ba7..37e94200e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 3508431b1..d9cdf6477 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -62,7 +62,9 @@ <<"event_message">>, <<"prometheus">>, <<"telemetry">>, - <<"listeners">> + <<"listeners">>, + <<"license">>, + <<"api_key">> ] ++ global_zone_roots() ). diff --git a/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl b/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl new file mode 100644 index 000000000..556e4308f --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_key_schema). + +-include_lib("hocon/include/hoconsc.hrl"). + +-export([ + roots/0, + fields/1, + namespace/0, + desc/1 +]). + +namespace() -> api_key. +roots() -> ["api_key"]. + +fields("api_key") -> + [ + {bootstrap_file, + ?HOCON( + binary(), + #{ + desc => ?DESC(bootstrap_file), + required => false, + default => <<>> + } + )} + ]. + +desc("api_key") -> + ?DESC(api_key). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 925c20ff1..7bf68ee4d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 004913206..72b616fae 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index d0d2e4b8c..64ef3c1ef 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index ac9c8644d..28deac98d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index b1b1f1b5e..245b56c1d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -104,9 +104,7 @@ fields(message) -> })}, {clientid, hoconsc:mk(binary(), #{ - desc => ?DESC(clientid), - required => false, - example => <<"api_example_client">> + deprecated => {since, "v5.0.14"} })}, {payload, hoconsc:mk(binary(), #{ @@ -254,7 +252,6 @@ is_ok_deliver({_NodeOrShare, _MatchedTopic, {error, _}}) -> false. %% %%%%%% Below error codes are not implemented so far %%%% %% %% If HTTP request passes HTTP authentication, it is considered trusted. -%% In the future, we may choose to check ACL for the provided MQTT Client ID %% 135 Not authorized 401 %% %% %%%%%% Below error codes are not applicable %%%%%%% @@ -326,7 +323,6 @@ make_message(Map) -> Encoding = maps:get(<<"payload_encoding">>, Map, plain), case decode_payload(Encoding, maps:get(<<"payload">>, Map)) of {ok, Payload} -> - From = maps:get(<<"clientid">>, Map, http_api), QoS = maps:get(<<"qos">>, Map, 0), Topic = maps:get(<<"topic">>, Map), Retain = maps:get(<<"retain">>, Map, false), @@ -346,7 +342,9 @@ make_message(Map) -> error:_Reason -> throw(invalid_topic_name) end, - Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, Headers), + Message = emqx_message:make( + http_api, QoS, Topic, Payload, #{retain => Retain}, Headers + ), Size = emqx_message:estimate_size(Message), (Size > size_limit()) andalso throw(packet_too_large), {ok, Message}; diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index b6462c713..19bb3e737 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index ea91f1c03..7d5c18e59 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 2cb5a4efe..b3380f4d1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 5e2bd1ea0..a64badd3a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 17895786b..d90aea9ef 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index 9ad1930a0..137f4502c 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,10 +28,15 @@ -include("emqx_mgmt.hrl"). start(_Type, _Args) -> - {ok, Sup} = emqx_mgmt_sup:start_link(), ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity), - emqx_mgmt_cli:load(), - {ok, Sup}. + case emqx_mgmt_auth:init_bootstrap_file() of + ok -> + {ok, Sup} = emqx_mgmt_sup:start_link(), + ok = emqx_mgmt_cli:load(), + {ok, Sup}; + {error, Reason} -> + {error, Reason} + end. stop(_State) -> ok. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6723be9b9..0bf849d3c 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_mgmt_auth). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). %% API -export([mnesia/1]). @@ -25,7 +26,8 @@ read/1, update/4, delete/1, - list/0 + list/0, + init_bootstrap_file/0 ]). -export([authorize/3]). @@ -34,7 +36,8 @@ -export([ do_update/4, do_delete/1, - do_create_app/3 + do_create_app/3, + do_force_create_app/3 ]). -define(APP, emqx_app). @@ -45,7 +48,7 @@ api_secret_hash = <<>> :: binary() | '_', enable = true :: boolean() | '_', desc = <<>> :: binary() | '_', - expired_at = 0 :: integer() | undefined | '_', + expired_at = 0 :: integer() | undefined | infinity | '_', created_at = 0 :: integer() | '_' }). @@ -58,8 +61,14 @@ mnesia(boot) -> {attributes, record_info(fields, ?APP)} ]). +-spec init_bootstrap_file() -> ok | {error, _}. +init_bootstrap_file() -> + File = bootstrap_file(), + ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}), + init_bootstrap_file(File). + create(Name, Enable, ExpiredAt, Desc) -> - case mnesia:table_info(?APP, size) < 30 of + case mnesia:table_info(?APP, size) < 100 of true -> create_app(Name, Enable, ExpiredAt, Desc); false -> {error, "Maximum ApiKey"} end. @@ -169,6 +178,9 @@ create_app(Name, Enable, ExpiredAt, Desc) -> create_app(App = #?APP{api_key = ApiKey, name = Name}) -> trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]). +force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) -> + trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]). + do_create_app(App, ApiKey, Name) -> case mnesia:read(?APP, Name) of [_] -> @@ -183,6 +195,22 @@ do_create_app(App, ApiKey, Name) -> end end. +do_force_create_app(App, ApiKey, NamePrefix) -> + case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of + [] -> + NewName = generate_unique_name(NamePrefix), + ok = mnesia:write(App#?APP{name = NewName}); + [#?APP{name = Name}] -> + ok = mnesia:write(App#?APP{name = Name}) + end. + +generate_unique_name(NamePrefix) -> + New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + case mnesia:read(?APP, New) of + [] -> New; + _ -> generate_unique_name(NamePrefix) + end. + trans(Fun, Args) -> case mria:transaction(?COMMON_SHARD, Fun, Args) of {atomic, Res} -> {ok, Res}; @@ -192,3 +220,84 @@ trans(Fun, Args) -> generate_api_secret() -> Random = crypto:strong_rand_bytes(32), emqx_base62:encode(Random). + +bootstrap_file() -> + case emqx:get_config([api_key, bootstrap_file], <<>>) of + %% For compatible remove until 5.1.0 + <<>> -> + emqx:get_config([dashboard, bootstrap_users_file], <<>>); + File -> + File + end. + +init_bootstrap_file(<<>>) -> + ok; +init_bootstrap_file(File) -> + case file:open(File, [read, binary]) of + {ok, Dev} -> + {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), + init_bootstrap_file(File, Dev, MP); + {error, Reason0} -> + Reason = emqx_misc:explain_posix(Reason0), + ?SLOG( + error, + #{ + msg => "failed_to_open_the_bootstrap_file", + file => File, + reason => Reason + } + ), + {error, Reason} + end. + +init_bootstrap_file(File, Dev, MP) -> + try + add_bootstrap_file(File, Dev, MP, 1) + catch + throw:Error -> {error, Error}; + Type:Reason:Stacktrace -> {error, {Type, Reason, Stacktrace}} + after + file:close(Dev) + end. + +-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>). + +add_bootstrap_file(File, Dev, MP, Line) -> + case file:read_line(Dev) of + {ok, Bin} -> + case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of + {match, [[AppKey, ApiSecret]]} -> + App = + #?APP{ + enable = true, + expired_at = infinity, + desc = ?BOOTSTRAP_TAG, + created_at = erlang:system_time(second), + api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), + api_key = AppKey + }, + case force_create_app("from_bootstrap_file_", App) of + {ok, ok} -> + add_bootstrap_file(File, Dev, MP, Line + 1); + {error, Reason} -> + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + _ -> + Reason = "invalid_format", + ?SLOG( + error, + #{ + msg => "failed_to_load_bootstrap_file", + file => File, + line => Line, + content => Bin, + reason => Reason + } + ), + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + eof -> + ok; + {error, Reason} -> + throw(#{file => File, line => Line, reason => Reason}) + end. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 6a0f8e8be..8d19716ce 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_sup.erl b/apps/emqx_management/src/emqx_mgmt_sup.erl index 74f77dbd0..329532fa1 100644 --- a/apps/emqx_management/src/emqx_mgmt_sup.erl +++ b/apps/emqx_management/src/emqx_mgmt_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 58fd6e952..c0d9e6036 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v1.erl b/apps/emqx_management/src/proto/emqx_management_proto_v1.erl index c9456058e..93545b36c 100644 --- a/apps/emqx_management/src/proto/emqx_management_proto_v1.erl +++ b/apps/emqx_management/src/proto/emqx_management_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v2.erl b/apps/emqx_management/src/proto/emqx_management_proto_v2.erl index 34da390b5..2b550fc02 100644 --- a/apps/emqx_management/src/proto/emqx_management_proto_v2.erl +++ b/apps/emqx_management/src/proto/emqx_management_proto_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v3.erl b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl index 937a948e5..7ab06b99b 100644 --- a/apps/emqx_management/src/proto/emqx_management_proto_v3.erl +++ b/apps/emqx_management/src/proto/emqx_management_proto_v3.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v1.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v1.erl index 4ac594d2c..d1fbf6706 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v1.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v1.erl b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v1.erl index aeda928fa..59a6500b9 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v1.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v1.erl b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v1.erl index 4f847219e..d6b548884 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v1.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl index 8502b81d9..2f593d465 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a065b9c83..a14305d8b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index a499e7cd2..adff41214 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_app_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl similarity index 75% rename from apps/emqx_management/test/emqx_mgmt_api_app_SUITE.erl rename to apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 44fc09902..079351538 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_app_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_app_SUITE). +-module(emqx_mgmt_api_api_keys_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -25,15 +25,62 @@ suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]}, - {sequence, [], [t_create_failed]} + {sequence, [], [t_bootstrap_file, t_create_failed]} ]. init_per_suite(Config) -> - emqx_mgmt_api_test_util:init_suite(), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. end_per_suite(_) -> - emqx_mgmt_api_test_util:end_suite(). + emqx_mgmt_api_test_util:end_suite([emqx_conf]). + +t_bootstrap_file(_) -> + TestPath = <<"/api/v5/status">>, + Bin = <<"test-1:secret-1\ntest-2:secret-2">>, + File = "./bootstrap_api_keys.txt", + ok = file:write_file(File, Bin), + emqx:update_config([api_key, bootstrap_file], File), + ok = emqx_mgmt_auth:init_bootstrap_file(), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)), + ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-1">>)), + + %% relaunch to check if the table is changed. + Bin1 = <<"test-1:new-secret-1\ntest-2:new-secret-2">>, + ok = file:write_file(File, Bin1), + ok = emqx_mgmt_auth:init_bootstrap_file(), + ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)), + ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), + + %% Compatibility + Bin2 = <<"test-3:new-secret-3\ntest-4:new-secret-4">>, + ok = file:write_file(File, Bin2), + emqx:update_config([api_key, bootstrap_file], <<>>), + emqx:update_config([dashboard, bootstrap_users_file], File), + ok = emqx_mgmt_auth:init_bootstrap_file(), + ?assertMatch(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), + ?assertMatch(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-3">>, <<"new-secret-3">>)), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-4">>, <<"new-secret-4">>)), + + %% not found + NotFoundFile = "./bootstrap_apps_not_exist.txt", + emqx:update_config([api_key, bootstrap_file], NotFoundFile), + ?assertMatch({error, "No such file or directory"}, emqx_mgmt_auth:init_bootstrap_file()), + + %% bad format + BadBin = <<"test-1:secret-11\ntest-2 secret-12">>, + ok = file:write_file(File, BadBin), + emqx:update_config([api_key, bootstrap_file], File), + ?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()), + ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-11">>)), + ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-12">>)), + emqx:update_config([api_key, bootstrap_file], <<>>), + emqx:update_config([dashboard, bootstrap_users_file], <<>>), + ok. t_create(_Config) -> Name = <<"EMQX-API-KEY-1">>, @@ -69,7 +116,7 @@ t_create_failed(_Config) -> ?assertEqual(BadRequest, create_app(LongName)), {ok, List} = list_app(), - CreateNum = 30 - erlang:length(List), + CreateNum = 100 - erlang:length(List), Names = lists:map( fun(Seq) -> <<"EMQX-API-FAILED-KEY-", (integer_to_binary(Seq))/binary>> diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index 5c985db6c..c765f00bc 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 24c857288..16ba99ad6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,7 +44,10 @@ t_clients(_) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), {ok, C1} = emqtt:start_link(#{ - username => Username1, clientid => ClientId1, proto_ver => v5 + username => Username1, + clientid => ClientId1, + proto_ver => v5, + properties => #{'Session-Expiry-Interval' => 120} }), {ok, _} = emqtt:connect(C1), {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), @@ -70,6 +73,7 @@ t_clients(_) -> Client1Response = emqx_json:decode(Client1, [return_maps]), ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), + ?assertEqual(120, maps:get(<<"expiry_interval">>, Client1Response)), %% delete /clients/:clientid kickout Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 067fe312f..d26f4480b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index a623b6fbf..e5c47ac4d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl index 1166903d7..93cb69f0a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 73b796afe..2bbdf938d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index ddf4de1a4..fd8d8b02e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 783a90185..44b8069f5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 2b260aa9e..7236ac9e4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index a768d2bfe..f0200c410 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 9a6de9938..965ed0997 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index ec40e3cc2..5bb0ba818 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 419c17adf..dcea88d59 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index d0f22f134..8e8c5b06f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl index 43a1b55e1..e6543d6a1 100644 --- a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/include/emqx_modules.hrl b/apps/emqx_modules/include/emqx_modules.hrl index 0e27bd56e..01b3b38f5 100644 --- a/apps/emqx_modules/include/emqx_modules.hrl +++ b/apps/emqx_modules/include/emqx_modules.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 241ac5a8a..91cdbedd3 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index c6f6d5062..d4e7e5b90 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index fa68e17cb..20f8a76fc 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {applications, [kernel, stdlib, emqx]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index 5b5188f2b..f1c95222d 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index 448dee817..2162afc70 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index d2d6258d1..ddb8f37a3 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -88,8 +88,13 @@ regular_expression(_) -> undefined. is_re(Bin) -> case re:compile(Bin) of - {ok, _} -> ok; - {error, Reason} -> {error, {Bin, Reason}} + {ok, _} -> + ok; + {error, Reason} -> + {error, #{ + regexp => Bin, + compile_error => Reason + }} end. array(Name, Meta) -> {Name, ?HOCON(?ARRAY(?R_REF(Name)), Meta)}. diff --git a/apps/emqx_modules/src/emqx_modules_sup.erl b/apps/emqx_modules/src/emqx_modules_sup.erl index 42b000eaa..463be28ea 100644 --- a/apps/emqx_modules/src/emqx_modules_sup.erl +++ b/apps/emqx_modules/src/emqx_modules_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_observer_cli.erl b/apps/emqx_modules/src/emqx_observer_cli.erl index d8fa1822d..abed31edc 100644 --- a/apps/emqx_modules/src/emqx_observer_cli.erl +++ b/apps/emqx_modules/src/emqx_observer_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl index 79465045d..6bf8abb89 100644 --- a/apps/emqx_modules/src/emqx_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -67,8 +67,12 @@ list() -> emqx_conf:get_raw([<<"rewrite">>], []). update(Rules0) -> - {ok, _} = emqx_conf:update([rewrite], Rules0, #{override_to => cluster}), - ok. + case emqx_conf:update([rewrite], Rules0, #{override_to => cluster}) of + {ok, _} -> + ok; + {error, Reason} -> + throw(Reason) + end. post_config_update(_KeyPath, _Config, Rules, _OldConf, _AppEnvs) -> register_hook(Rules). diff --git a/apps/emqx_modules/src/emqx_rewrite_api.erl b/apps/emqx_modules/src/emqx_rewrite_api.erl index a62b2a9b2..cf22daec6 100644 --- a/apps/emqx_modules/src/emqx_rewrite_api.erl +++ b/apps/emqx_modules/src/emqx_rewrite_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index f55457e9d..16fef8d34 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 4883de2fc..798c3ad17 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index 5a0925771..dfc6b07ab 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index 1fb9c3a7b..ef3c2be69 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/proto/emqx_delayed_proto_v1.erl b/apps/emqx_modules/src/proto/emqx_delayed_proto_v1.erl index 5825b8a29..29a509cfe 100644 --- a/apps/emqx_modules/src/proto/emqx_delayed_proto_v1.erl +++ b/apps/emqx_modules/src/proto/emqx_delayed_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%%Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl b/apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl index ebd23e534..342145d3c 100644 --- a/apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl +++ b/apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%%Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/src/proto/emqx_topic_metrics_proto_v1.erl b/apps/emqx_modules/src/proto/emqx_topic_metrics_proto_v1.erl index 55cd33032..5f2233435 100644 --- a/apps/emqx_modules/src/proto/emqx_topic_metrics_proto_v1.erl +++ b/apps/emqx_modules/src/proto/emqx_topic_metrics_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index 3d1576b0b..ab35e37dc 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index b29cd54f4..96cdf7840 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl index 10847e8ec..666af9ef0 100644 --- a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl +++ b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_observer_cli_tests.erl b/apps/emqx_modules/test/emqx_observer_cli_tests.erl index af1ea3f15..a0a3898dd 100644 --- a/apps/emqx_modules/test/emqx_observer_cli_tests.erl +++ b/apps/emqx_modules/test/emqx_observer_cli_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 85688d3db..1847f876e 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -215,16 +215,16 @@ t_update_re_failed(_Config) -> <<"action">> => <<"publish">> } ], - ?assertError( - {badmatch, - {error, - {_, [ - #{ - kind := validation_error, - reason := {Re, {"nothing to repeat", 0}}, - value := Re - } - ]}}}, + ?assertThrow( + #{ + kind := validation_error, + path := "rewrite.1.re", + reason := #{ + regexp := <<"*^test/*">>, + compile_error := {"nothing to repeat", 0} + }, + value := <<"*^test/*">> + }, emqx_rewrite:update(Rules) ), ok. diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index 19bcc007a..90e90d788 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index cd25a3f51..37c1115aa 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index d2c8079b9..288a155d9 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl index b2fa561b4..10ce5d0df 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index 09964b328..8c23d042c 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index dd99724e2..3b15a389d 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl index 4edc24beb..e237ffe82 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index f0b1f0bb5..289d39032 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index e94d62b53..57bdd16e5 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl b/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl index b8936a94f..8cd9952e0 100644 --- a/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl +++ b/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl index bb83ce40c..6baaaefc6 100644 --- a/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl index 9f7117324..bf509f362 100644 --- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/include/emqx_plugins.hrl b/apps/emqx_plugins/include/emqx_plugins.hrl index 4595e28db..59a4825f2 100644 --- a/apps/emqx_plugins/include/emqx_plugins.hrl +++ b/apps/emqx_plugins/include/emqx_plugins.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index c7c67ed23..613148a24 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/src/emqx_plugins_app.erl b/apps/emqx_plugins/src/emqx_plugins_app.erl index 5d3828fb5..c42936d56 100644 --- a/apps/emqx_plugins/src/emqx_plugins_app.erl +++ b/apps/emqx_plugins/src/emqx_plugins_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/src/emqx_plugins_cli.erl b/apps/emqx_plugins/src/emqx_plugins_cli.erl index 2ea965ce7..2cc5f023c 100644 --- a/apps/emqx_plugins/src/emqx_plugins_cli.erl +++ b/apps/emqx_plugins/src/emqx_plugins_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 7ca2e1d00..8b3cca8fd 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/src/emqx_plugins_sup.erl b/apps/emqx_plugins/src/emqx_plugins_sup.erl index 687e5d39e..31427aaf6 100644 --- a/apps/emqx_plugins/src/emqx_plugins_sup.erl +++ b/apps/emqx_plugins/src/emqx_plugins_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index e933a36d4..3823d940d 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index c4982a8ff..c24a3345a 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/include/emqx_prometheus.hrl b/apps/emqx_prometheus/include/emqx_prometheus.hrl index 36066a55d..8d552f025 100644 --- a/apps/emqx_prometheus/include/emqx_prometheus.hrl +++ b/apps/emqx_prometheus/include/emqx_prometheus.hrl @@ -1,2 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -define(APP, emqx_prometheus). -define(PROMETHEUS, [prometheus]). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index de9349b97..5424c4e24 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 125eed560..7466a1fd1 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_app.erl b/apps/emqx_prometheus/src/emqx_prometheus_app.erl index bdee12d0e..99ed06872 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_app.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index b4914f216..39d2d4f6b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_mria.erl b/apps/emqx_prometheus/src/emqx_prometheus_mria.erl index c81e0885e..91a923a2f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_mria.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_mria.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 09908167c..688c9be58 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index a70fda322..c284328ff 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl index e11f8e3ad..b3ba1b6ce 100644 --- a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl +++ b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index e26bcfeb4..b9df1103b 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index 59b3b9a17..0968100b8 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (C) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl index 3a9406fdb..65bdeab48 100644 --- a/apps/emqx_psk/src/emqx_psk.erl +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk/src/emqx_psk_app.erl b/apps/emqx_psk/src/emqx_psk_app.erl index 3e5936783..b8572e6a0 100644 --- a/apps/emqx_psk/src/emqx_psk_app.erl +++ b/apps/emqx_psk/src/emqx_psk_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 769078b5b..45a1a077e 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk/src/emqx_psk_sup.erl b/apps/emqx_psk/src/emqx_psk_sup.erl index 17e9f171d..2bc8430eb 100644 --- a/apps/emqx_psk/src/emqx_psk_sup.erl +++ b/apps/emqx_psk/src/emqx_psk_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index af19cae38..00702efa0 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 332dfdd8c..52756f70d 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -126,8 +126,8 @@ When disabled the messages are buffered in RAM only.""" batch_size { desc { - en: """Maximum batch count.""" - zh: """批量请求大小。""" + en: """Maximum batch count. If equal to 1, there's effectively no batching.""" + zh: """批量请求大小。如果设为1，则无批处理。""" } label { en: """Batch size""" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 051b57e31..d7b080ae8 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -64,10 +64,8 @@ %% If the resource disconnected, we can set to retry starting the resource %% periodically. auto_restart_interval => pos_integer(), - enable_batch => boolean(), batch_size => pos_integer(), batch_time => pos_integer(), - enable_queue => boolean(), max_queue_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), @@ -90,7 +88,7 @@ -define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). %% count --define(DEFAULT_BATCH_SIZE, 100). +-define(DEFAULT_BATCH_SIZE, 1). %% milliseconds -define(DEFAULT_BATCH_TIME, 20). diff --git a/apps/emqx_resource/include/emqx_resource_errors.hrl b/apps/emqx_resource/include/emqx_resource_errors.hrl index 6d1b3e92f..a3bd8723a 100644 --- a/apps/emqx_resource/include/emqx_resource_errors.hrl +++ b/apps/emqx_resource/include/emqx_resource_errors.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl index 3df64b1e5..cf0804251 100644 --- a/apps/emqx_resource/include/emqx_resource_utils.hrl +++ b/apps/emqx_resource/include/emqx_resource_utils.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 78f5d8342..00389261b 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 8086dfa25..aff66c287 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -111,7 +111,7 @@ list_group_instances/1 ]). --export([inc_received/1, apply_reply_fun/2]). +-export([apply_reply_fun/2]). -optional_callbacks([ on_query/3, @@ -467,8 +467,5 @@ apply_reply_fun(From, Result) -> %% ================================================================================= -inc_received(ResId) -> - emqx_metrics_worker:inc(?RES_METRICS, ResId, 'received'). - filter_instances(Filter) -> [Id || #{id := Id, mod := Mod} <- list_instances_verbose(), Filter(Id, Mod)]. diff --git a/apps/emqx_resource/src/emqx_resource_app.erl b/apps/emqx_resource/src/emqx_resource_app.erl index 51e7b2556..b11644c3a 100644 --- a/apps/emqx_resource/src/emqx_resource_app.erl +++ b/apps/emqx_resource/src/emqx_resource_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 10c501865..8531f1641 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -112,7 +112,7 @@ recreate(ResId, ResourceType, NewConfig, Opts) -> end. create_and_return_data(MgrId, ResId, Group, ResourceType, Config, Opts) -> - create(MgrId, ResId, Group, ResourceType, Config, Opts), + _ = create(MgrId, ResId, Group, ResourceType, Config, Opts), {ok, _Group, Data} = lookup(ResId), {ok, Data}. @@ -136,14 +136,10 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> 'success', 'failed', 'dropped', - 'dropped.queue_not_enabled', 'dropped.queue_full', 'dropped.resource_not_found', 'dropped.resource_stopped', 'dropped.other', - 'queuing', - 'batching', - 'inflight', 'received' ], [matched] @@ -308,7 +304,7 @@ init({Data, Opts}) -> process_flag(trap_exit, true), %% init the cache so that lookup/1 will always return something insert_cache(Data#data.id, Data#data.group, Data), - case maps:get(start_after_created, Opts, true) of + case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of true -> {ok, connecting, Data, {next_event, internal, start_resource}}; false -> {ok, stopped, Data} end. diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index b29c28f70..5b731d6cf 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index e6637b68f..455be9c22 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,11 +24,9 @@ ]). -export([ - batching_change/2, - batching_get/1, - inflight_change/2, + inflight_set/3, inflight_get/1, - queuing_change/2, + queuing_set/3, queuing_get/1, dropped_inc/1, dropped_inc/2, @@ -39,9 +37,6 @@ dropped_queue_full_inc/1, dropped_queue_full_inc/2, dropped_queue_full_get/1, - dropped_queue_not_enabled_inc/1, - dropped_queue_not_enabled_inc/2, - dropped_queue_not_enabled_get/1, dropped_resource_not_found_inc/1, dropped_resource_not_found_inc/2, dropped_resource_not_found_get/1, @@ -54,6 +49,9 @@ matched_inc/1, matched_inc/2, matched_get/1, + received_inc/1, + received_inc/2, + received_get/1, retried_inc/1, retried_inc/2, retried_get/1, @@ -76,16 +74,15 @@ events() -> [ [?TELEMETRY_PREFIX, Event] || Event <- [ - batching, dropped_other, dropped_queue_full, - dropped_queue_not_enabled, dropped_resource_not_found, dropped_resource_stopped, failed, inflight, matched, queuing, + received, retried_failed, retried_success, success @@ -114,17 +111,12 @@ handle_telemetry_event( _HandlerConfig ) -> case Event of - batching -> - emqx_metrics_worker:inc(?RES_METRICS, ID, 'batching', Val); dropped_other -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val), emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other', Val); dropped_queue_full -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val), emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full', Val); - dropped_queue_not_enabled -> - emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val), - emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled', Val); dropped_resource_not_found -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val), emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found', Val); @@ -133,12 +125,10 @@ handle_telemetry_event( emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped', Val); failed -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val); - inflight -> - emqx_metrics_worker:inc(?RES_METRICS, ID, 'inflight', Val); matched -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched', Val); - queuing -> - emqx_metrics_worker:inc(?RES_METRICS, ID, 'queuing', Val); + received -> + emqx_metrics_worker:inc(?RES_METRICS, ID, 'received', Val); retried_failed -> emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried', Val), emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val), @@ -152,34 +142,49 @@ handle_telemetry_event( _ -> ok end; +handle_telemetry_event( + [?TELEMETRY_PREFIX, Event], + _Measurements = #{gauge_set := Val}, + _Metadata = #{resource_id := ID, worker_id := WorkerID}, + _HandlerConfig +) -> + case Event of + inflight -> + emqx_metrics_worker:set_gauge(?RES_METRICS, ID, WorkerID, 'inflight', Val); + queuing -> + emqx_metrics_worker:set_gauge(?RES_METRICS, ID, WorkerID, 'queuing', Val); + _ -> + ok + end; handle_telemetry_event(_EventName, _Measurements, _Metadata, _HandlerConfig) -> ok. %% Gauges (value can go both up and down): %% -------------------------------------- -%% @doc Count of messages that are currently accumulated in memory waiting for -%% being sent in one batch -batching_change(ID, Val) -> - telemetry:execute([?TELEMETRY_PREFIX, batching], #{counter_inc => Val}, #{resource_id => ID}). - -batching_get(ID) -> - emqx_metrics_worker:get(?RES_METRICS, ID, 'batching'). - -%% @doc Count of messages that are currently queuing. [Gauge] -queuing_change(ID, Val) -> - telemetry:execute([?TELEMETRY_PREFIX, queuing], #{counter_inc => Val}, #{resource_id => ID}). +%% @doc Count of batches of messages that are currently +%% queuing. [Gauge] +queuing_set(ID, WorkerID, Val) -> + telemetry:execute( + [?TELEMETRY_PREFIX, queuing], + #{gauge_set => Val}, + #{resource_id => ID, worker_id => WorkerID} + ). queuing_get(ID) -> - emqx_metrics_worker:get(?RES_METRICS, ID, 'queuing'). + emqx_metrics_worker:get_gauge(?RES_METRICS, ID, 'queuing'). -%% @doc Count of messages that were sent asynchronously but ACKs are not -%% received. [Gauge] -inflight_change(ID, Val) -> - telemetry:execute([?TELEMETRY_PREFIX, inflight], #{counter_inc => Val}, #{resource_id => ID}). +%% @doc Count of batches of messages that were sent asynchronously but +%% ACKs are not yet received. [Gauge] +inflight_set(ID, WorkerID, Val) -> + telemetry:execute( + [?TELEMETRY_PREFIX, inflight], + #{gauge_set => Val}, + #{resource_id => ID, worker_id => WorkerID} + ). inflight_get(ID) -> - emqx_metrics_worker:get(?RES_METRICS, ID, 'inflight'). + emqx_metrics_worker:get_gauge(?RES_METRICS, ID, 'inflight'). %% Counters (value can only got up): %% -------------------------------------- @@ -218,18 +223,6 @@ dropped_queue_full_inc(ID, Val) -> dropped_queue_full_get(ID) -> emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_full'). -%% @doc Count of messages dropped because the queue was not enabled -dropped_queue_not_enabled_inc(ID) -> - dropped_queue_not_enabled_inc(ID, 1). - -dropped_queue_not_enabled_inc(ID, Val) -> - telemetry:execute([?TELEMETRY_PREFIX, dropped_queue_not_enabled], #{counter_inc => Val}, #{ - resource_id => ID - }). - -dropped_queue_not_enabled_get(ID) -> - emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_not_enabled'). - %% @doc Count of messages dropped because the resource was not found dropped_resource_not_found_inc(ID) -> dropped_resource_not_found_inc(ID, 1). @@ -264,6 +257,16 @@ matched_inc(ID, Val) -> matched_get(ID) -> emqx_metrics_worker:get(?RES_METRICS, ID, 'matched'). +%% @doc The number of messages that have been received from a bridge +received_inc(ID) -> + received_inc(ID, 1). + +received_inc(ID, Val) -> + telemetry:execute([?TELEMETRY_PREFIX, received], #{counter_inc => Val}, #{resource_id => ID}). + +received_get(ID) -> + emqx_metrics_worker:get(?RES_METRICS, ID, 'received'). + %% @doc The number of times message sends have been retried retried_inc(ID) -> retried_inc(ID, 1). diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 920743101..ea31b8b6b 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource_utils.erl b/apps/emqx_resource/src/emqx_resource_utils.erl index 715691d2a..55d8474f4 100644 --- a/apps/emqx_resource/src/emqx_resource_utils.erl +++ b/apps/emqx_resource/src/emqx_resource_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index a5404a65e..bc733ef80 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 482c82f6a..93bb22551 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,6 @@ sync_query/3, async_query/3, block/1, - block/2, resume/1 ]). @@ -52,10 +51,14 @@ -export([queue_item_marshaller/1, estimate_size/1]). --export([reply_after_query/6, batch_reply_after_query/6]). +-export([reply_after_query/7, batch_reply_after_query/7]). + +-elvis([{elvis_style, dont_repeat_yourself, disable}]). -define(Q_ITEM(REQUEST), {q_item, REQUEST}). +-define(COLLECT_REQ_LIMIT, 1000). +-define(SEND_REQ(FROM, REQUEST), {'$send_req', FROM, REQUEST}). -define(QUERY(FROM, REQUEST, SENT), {query, FROM, REQUEST, SENT}). -define(REPLY(FROM, REQUEST, SENT, RESULT), {reply, FROM, REQUEST, SENT, RESULT}). -define(EXPAND(RESULT, BATCH), [ @@ -64,12 +67,22 @@ ]). -type id() :: binary(). --type query() :: {query, from(), request()}. +-type index() :: pos_integer(). +-type queue_query() :: ?QUERY(from(), request(), HasBeenSent :: boolean()). -type request() :: term(). --type from() :: pid() | reply_fun(). - --callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> - {{from(), result()}, NewCbState :: term()}. +-type from() :: pid() | reply_fun() | request_from(). +-type request_from() :: undefined | gen_statem:from(). +-type state() :: blocked | running. +-type data() :: #{ + id => id(), + index => index(), + name => atom(), + batch_size => pos_integer(), + batch_time => timer:time(), + queue => replayq:q(), + resume_interval => timer:time(), + tref => undefined | timer:tref() +}. callback_mode() -> [state_functions, state_enter]. @@ -80,23 +93,43 @@ start_link(Id, Index, Opts) -> sync_query(Id, Request, Opts) -> PickKey = maps:get(pick_key, Opts, self()), Timeout = maps:get(timeout, Opts, infinity), + emqx_resource_metrics:matched_inc(Id), pick_call(Id, PickKey, {query, Request, Opts}, Timeout). -spec async_query(id(), request(), query_opts()) -> Result :: term(). async_query(Id, Request, Opts) -> PickKey = maps:get(pick_key, Opts, self()), + emqx_resource_metrics:matched_inc(Id), pick_cast(Id, PickKey, {query, Request, Opts}). %% simple query the resource without batching and queuing messages. -spec simple_sync_query(id(), request()) -> Result :: term(). simple_sync_query(Id, Request) -> - Result = call_query(sync, Id, ?QUERY(self(), Request, false), #{}), + %% Note: since calling this function implies in bypassing the + %% buffer workers, and each buffer worker index is used when + %% collecting gauge metrics, we use this dummy index. If this + %% call ends up calling buffering functions, that's a bug and + %% would mess up the metrics anyway. `undefined' is ignored by + %% `emqx_resource_metrics:*_shift/3'. + Index = undefined, + QueryOpts = #{}, + emqx_resource_metrics:matched_inc(Id), + Result = call_query(sync, Id, Index, ?QUERY(self(), Request, false), QueryOpts), _ = handle_query_result(Id, Result, false, false), Result. -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term(). simple_async_query(Id, Request, ReplyFun) -> - Result = call_query(async, Id, ?QUERY(ReplyFun, Request, false), #{}), + %% Note: since calling this function implies in bypassing the + %% buffer workers, and each buffer worker index is used when + %% collecting gauge metrics, we use this dummy index. If this + %% call ends up calling buffering functions, that's a bug and + %% would mess up the metrics anyway. `undefined' is ignored by + %% `emqx_resource_metrics:*_shift/3'. + Index = undefined, + QueryOpts = #{}, + emqx_resource_metrics:matched_inc(Id), + Result = call_query(async, Id, Index, ?QUERY(ReplyFun, Request, false), QueryOpts), _ = handle_query_result(Id, Result, false, false), Result. @@ -104,14 +137,11 @@ simple_async_query(Id, Request, ReplyFun) -> block(ServerRef) -> gen_statem:cast(ServerRef, block). --spec block(pid() | atom(), [query()]) -> ok. -block(ServerRef, Query) -> - gen_statem:cast(ServerRef, {block, Query}). - -spec resume(pid() | atom()) -> ok. resume(ServerRef) -> gen_statem:cast(ServerRef, resume). +-spec init({id(), pos_integer(), map()}) -> gen_statem:init_result(state(), data()). init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), @@ -120,88 +150,89 @@ init({Id, Index, Opts}) -> SegBytes0 = maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE), TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), SegBytes = min(SegBytes0, TotalBytes), - Queue = - case maps:get(enable_queue, Opts, false) of - true -> - replayq:open(#{ - dir => disk_queue_dir(Id, Index), - seg_bytes => SegBytes, - max_total_bytes => TotalBytes, - sizer => fun ?MODULE:estimate_size/1, - marshaller => fun ?MODULE:queue_item_marshaller/1 - }); - false -> - undefined - end, - emqx_resource_metrics:queuing_change(Id, queue_count(Queue)), + QueueOpts = + #{ + dir => disk_queue_dir(Id, Index), + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + %% we don't want to retain the queue after + %% resource restarts. + offload => true, + seg_bytes => SegBytes, + sizer => fun ?MODULE:estimate_size/1 + }, + Queue = replayq:open(QueueOpts), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), + emqx_resource_metrics:inflight_set(Id, Index, 0), InfltWinSZ = maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT), - ok = inflight_new(Name, InfltWinSZ), + ok = inflight_new(Name, InfltWinSZ, Id, Index), HCItvl = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), St = #{ id => Id, index => Index, name => Name, - enable_batch => maps:get(enable_batch, Opts, false), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), queue => Queue, resume_interval => maps:get(resume_interval, Opts, HCItvl), - acc => [], - acc_left => BatchSize, tref => undefined }, {ok, blocked, St, {next_event, cast, resume}}. -running(enter, _, _St) -> - keep_state_and_data; +running(enter, _, St) -> + ?tp(resource_worker_enter_running, #{}), + maybe_flush(St); running(cast, resume, _St) -> keep_state_and_data; running(cast, block, St) -> {next_state, blocked, St}; -running(cast, {block, [?QUERY(_, _, _) | _] = Batch}, #{id := Id, queue := Q} = St) when - is_list(Batch) --> - Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]), - {next_state, blocked, St#{queue := Q1}}; -running({call, From}, {query, Request, _Opts}, St) -> - query_or_acc(From, Request, St); -running(cast, {query, Request, Opts}, St) -> - ReplyFun = maps:get(async_reply_fun, Opts, undefined), - query_or_acc(ReplyFun, Request, St); +running(info, ?SEND_REQ(_From, _Req) = Request0, Data) -> + handle_query_requests(Request0, Data); running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> flush(St#{tref := undefined}); +running(internal, flush, St) -> + flush(St); running(info, {flush, _Ref}, _St) -> keep_state_and_data; running(info, Info, _St) -> - ?SLOG(error, #{msg => unexpected_msg, info => Info}), + ?SLOG(error, #{msg => unexpected_msg, state => running, info => Info}), keep_state_and_data. blocked(enter, _, #{resume_interval := ResumeT} = _St) -> + ?tp(resource_worker_enter_blocked, #{}), {keep_state_and_data, {state_timeout, ResumeT, resume}}; blocked(cast, block, _St) -> keep_state_and_data; -blocked(cast, {block, [?QUERY(_, _, _) | _] = Batch}, #{id := Id, queue := Q} = St) when - is_list(Batch) --> - Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]), - {keep_state, St#{queue := Q1}}; blocked(cast, resume, St) -> do_resume(St); blocked(state_timeout, resume, St) -> do_resume(St); -blocked({call, From}, {query, Request, _Opts}, #{id := Id, queue := Q} = St) -> +blocked(info, ?SEND_REQ(ReqFrom, {query, Request, Opts}), Data0) -> + #{ + id := Id, + index := Index, + queue := Q + } = Data0, + From = + case ReqFrom of + undefined -> maps:get(async_reply_fun, Opts, undefined); + From1 -> From1 + end, Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), - _ = reply_caller(Id, ?REPLY(From, Request, false, Error)), - {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(From, Request, false))])}}; -blocked(cast, {query, Request, Opts}, #{id := Id, queue := Q} = St) -> - ReplyFun = maps:get(async_reply_fun, Opts, undefined), - Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), - _ = reply_caller(Id, ?REPLY(ReplyFun, Request, false, Error)), - {keep_state, St#{ - queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(ReplyFun, Request, false))]) - }}. + HasBeenSent = false, + _ = reply_caller(Id, ?REPLY(From, Request, HasBeenSent, Error)), + NewQ = append_queue(Id, Index, Q, [?QUERY(From, Request, HasBeenSent)]), + Data = Data0#{queue := NewQ}, + {keep_state, Data}; +blocked(info, {flush, _Ref}, _Data) -> + keep_state_and_data; +blocked(info, Info, _Data) -> + ?SLOG(error, #{msg => unexpected_msg, state => blocked, info => Info}), + keep_state_and_data. -terminate(_Reason, #{id := Id, index := Index}) -> +terminate(_Reason, #{id := Id, index := Index, queue := Q}) -> + emqx_resource_metrics:inflight_set(Id, Index, 0), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q)), gproc_pool:disconnect_worker(Id, {Id, Index}). code_change(_OldVsn, State, _Extra) -> @@ -223,121 +254,274 @@ code_change(_OldVsn, State, _Extra) -> ). pick_call(Id, Key, Query, Timeout) -> - ?PICK(Id, Key, gen_statem:call(Pid, Query, {clean_timeout, Timeout})). + ?PICK(Id, Key, begin + Caller = self(), + MRef = erlang:monitor(process, Pid, [{alias, reply_demonitor}]), + From = {Caller, MRef}, + erlang:send(Pid, ?SEND_REQ(From, Query)), + receive + {MRef, Response} -> + erlang:demonitor(MRef, [flush]), + Response; + {'DOWN', MRef, process, Pid, Reason} -> + error({worker_down, Reason}) + after Timeout -> + erlang:demonitor(MRef, [flush]), + receive + {MRef, Response} -> + Response + after 0 -> + error(timeout) + end + end + end). pick_cast(Id, Key, Query) -> - ?PICK(Id, Key, gen_statem:cast(Pid, Query)). + ?PICK(Id, Key, begin + From = undefined, + erlang:send(Pid, ?SEND_REQ(From, Query)), + ok + end). -do_resume(#{id := Id, name := Name} = St) -> +do_resume(#{id := Id, name := Name} = Data) -> case inflight_get_first(Name) of empty -> - retry_queue(St); + retry_queue(Data); {Ref, FirstQuery} -> %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. - retry_inflight_sync(Id, Ref, FirstQuery, Name, St) + retry_inflight_sync(Id, Ref, FirstQuery, Name, Data) end. -retry_queue(#{queue := undefined} = St) -> - {next_state, running, St}; -retry_queue(#{queue := Q, id := Id, enable_batch := false, resume_interval := ResumeT} = St) -> - case get_first_n_from_queue(Q, 1) of - [] -> - {next_state, running, St}; - [?QUERY(_, Request, HasSent) = Query] -> - QueryOpts = #{inflight_name => maps:get(name, St)}, - Result = call_query(configured, Id, Query, QueryOpts), - case reply_caller(Id, ?REPLY(undefined, Request, HasSent, Result)) of +retry_queue( + #{ + queue := Q0, + id := Id, + index := Index, + batch_size := 1, + name := Name, + resume_interval := ResumeT + } = Data0 +) -> + %% no batching + case get_first_n_from_queue(Q0, 1) of + empty -> + {next_state, running, Data0}; + {Q1, QAckRef, [?QUERY(_, Request, HasBeenSent) = Query]} -> + QueryOpts = #{inflight_name => Name}, + Result = call_query(configured, Id, Index, Query, QueryOpts), + Reply = ?REPLY(undefined, Request, HasBeenSent, Result), + case reply_caller(Id, Reply) of true -> - {keep_state, St, {state_timeout, ResumeT, resume}}; + {keep_state, Data0, {state_timeout, ResumeT, resume}}; false -> - retry_queue(St#{queue := drop_head(Q, Id)}) + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + Data = Data0#{queue := Q1}, + retry_queue(Data) end end; retry_queue( #{ queue := Q, id := Id, - enable_batch := true, + index := Index, batch_size := BatchSize, + name := Name, resume_interval := ResumeT - } = St + } = Data0 ) -> + %% batching case get_first_n_from_queue(Q, BatchSize) of - [] -> - {next_state, running, St}; - Batch0 -> - QueryOpts = #{inflight_name => maps:get(name, St)}, - Result = call_query(configured, Id, Batch0, QueryOpts), + empty -> + {next_state, running, Data0}; + {Q1, QAckRef, Batch0} -> + QueryOpts = #{inflight_name => Name}, + Result = call_query(configured, Id, Index, Batch0, QueryOpts), %% The caller has been replied with ?RESOURCE_ERROR(blocked, _) before saving into the queue, %% we now change the 'from' field to 'undefined' so it will not reply the caller again. - Batch = [?QUERY(undefined, Request, HasSent) || ?QUERY(_, Request, HasSent) <- Batch0], + Batch = [ + ?QUERY(undefined, Request, HasBeenSent0) + || ?QUERY(_, Request, HasBeenSent0) <- Batch0 + ], case batch_reply_caller(Id, Result, Batch) of true -> - {keep_state, St, {state_timeout, ResumeT, resume}}; + ?tp(resource_worker_retry_queue_batch_failed, #{batch => Batch}), + {keep_state, Data0, {state_timeout, ResumeT, resume}}; false -> - retry_queue(St#{queue := drop_first_n_from_queue(Q, length(Batch), Id)}) + ?tp(resource_worker_retry_queue_batch_succeeded, #{batch => Batch}), + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + Data = Data0#{queue := Q1}, + retry_queue(Data) end end. retry_inflight_sync( - Id, Ref, ?QUERY(_, _, HasSent) = Query, Name, #{resume_interval := ResumeT} = St0 + Id, + Ref, + QueryOrBatch, + Name, + #{index := Index, resume_interval := ResumeT} = Data0 ) -> - Result = call_query(sync, Id, Query, #{}), - case handle_query_result(Id, Result, HasSent, false) of - %% Send failed because resource down + QueryOpts = #{}, + %% if we are retrying an inflight query, it has been sent + HasBeenSent = true, + Result = call_query(sync, Id, Index, QueryOrBatch, QueryOpts), + BlockWorker = false, + case handle_query_result(Id, Result, HasBeenSent, BlockWorker) of + %% Send failed because resource is down true -> - {keep_state, St0, {state_timeout, ResumeT, resume}}; + {keep_state, Data0, {state_timeout, ResumeT, resume}}; %% Send ok or failed but the resource is working false -> - inflight_drop(Name, Ref), - do_resume(St0) + inflight_drop(Name, Ref, Id, Index), + do_resume(Data0) end. -query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) -> - Acc1 = [?QUERY(From, Request, false) | Acc], - emqx_resource_metrics:batching_change(Id, 1), - St = St0#{acc := Acc1, acc_left := Left - 1}, - case Left =< 1 of - true -> flush(St); - false -> {keep_state, ensure_flush_timer(St)} - end; -query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St) -> - QueryOpts = #{ - inflight_name => maps:get(name, St) - }, - Result = call_query(configured, Id, ?QUERY(From, Request, false), QueryOpts), - case reply_caller(Id, ?REPLY(From, Request, false, Result)) of - true -> - Query = ?QUERY(From, Request, false), - {next_state, blocked, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(Query)])}}; - false -> - {keep_state, St} - end. - -flush(#{acc := []} = St) -> - {keep_state, St}; -flush( +%% Called during the `running' state only. +-spec handle_query_requests(?SEND_REQ(request_from(), request()), data()) -> data(). +handle_query_requests(Request0, Data0) -> #{ id := Id, - acc := Batch0, - batch_size := Size, - queue := Q0 - } = St -) -> - Batch = lists:reverse(Batch0), - QueryOpts = #{ - inflight_name => maps:get(name, St) - }, - emqx_resource_metrics:batching_change(Id, -length(Batch)), - Result = call_query(configured, Id, Batch, QueryOpts), - St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}), - case batch_reply_caller(Id, Result, Batch) of + index := Index, + queue := Q + } = Data0, + Requests = collect_requests([Request0], ?COLLECT_REQ_LIMIT), + QueueItems = + lists:map( + fun + (?SEND_REQ(undefined = _From, {query, Req, Opts})) -> + ReplyFun = maps:get(async_reply_fun, Opts, undefined), + HasBeenSent = false, + ?QUERY(ReplyFun, Req, HasBeenSent); + (?SEND_REQ(From, {query, Req, _Opts})) -> + HasBeenSent = false, + ?QUERY(From, Req, HasBeenSent) + end, + Requests + ), + NewQ = append_queue(Id, Index, Q, QueueItems), + Data = Data0#{queue := NewQ}, + maybe_flush(Data). + +maybe_flush(Data) -> + #{ + batch_size := BatchSize, + queue := Q + } = Data, + QueueCount = queue_count(Q), + case QueueCount >= BatchSize of true -> - Q1 = maybe_append_queue(Id, Q0, [?Q_ITEM(Query) || Query <- Batch]), - {next_state, blocked, St1#{queue := Q1}}; + flush(Data); false -> - {keep_state, St1} + {keep_state, ensure_flush_timer(Data)} + end. + +%% Called during the `running' state only. +-spec flush(data()) -> gen_statem:event_handler_result(state(), data()). +flush(Data0) -> + #{ + batch_size := BatchSize, + queue := Q0 + } = Data0, + case replayq:count(Q0) of + 0 -> + Data = cancel_flush_timer(Data0), + {keep_state, Data}; + _ -> + {Q1, QAckRef, Batch0} = replayq:pop(Q0, #{count_limit => BatchSize}), + Batch = [Item || ?Q_ITEM(Item) <- Batch0], + IsBatch = BatchSize =/= 1, + do_flush(Data0, #{ + new_queue => Q1, + is_batch => IsBatch, + batch => Batch, + ack_ref => QAckRef + }) + end. + +-spec do_flush(data(), #{ + is_batch := boolean(), + batch := [?QUERY(from(), request(), boolean())], + ack_ref := replayq:ack_ref() +}) -> + gen_statem:event_handler_result(state(), data()). +do_flush(Data0, #{is_batch := false, batch := Batch, ack_ref := QAckRef, new_queue := Q1}) -> + #{ + id := Id, + index := Index, + name := Name + } = Data0, + %% unwrap when not batching (i.e., batch size == 1) + [?QUERY(From, CoreReq, HasBeenSent) = Request] = Batch, + QueryOpts = #{inflight_name => Name}, + Result = call_query(configured, Id, Index, Request, QueryOpts), + IsAsync = is_async(Id), + Data1 = cancel_flush_timer(Data0), + Reply = ?REPLY(From, CoreReq, HasBeenSent, Result), + case {reply_caller(Id, Reply), IsAsync} of + %% failed and is not async; keep the request in the queue to + %% be retried + {true, false} -> + {next_state, blocked, Data1}; + %% failed and is async; remove the request from the queue, as + %% it is already in inflight table + {true, true} -> + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + Data = Data1#{queue := Q1}, + {next_state, blocked, Data}; + %% success; just ack + {false, _} -> + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + Data2 = Data1#{queue := Q1}, + case replayq:count(Q1) > 0 of + true -> + {keep_state, Data2, [{next_event, internal, flush}]}; + false -> + {keep_state, Data2} + end + end; +do_flush(Data0, #{is_batch := true, batch := Batch, ack_ref := QAckRef, new_queue := Q1}) -> + #{ + id := Id, + index := Index, + batch_size := BatchSize, + name := Name + } = Data0, + QueryOpts = #{inflight_name => Name}, + Result = call_query(configured, Id, Index, Batch, QueryOpts), + IsAsync = is_async(Id), + Data1 = cancel_flush_timer(Data0), + case {batch_reply_caller(Id, Result, Batch), IsAsync} of + %% failed and is not async; keep the request in the queue to + %% be retried + {true, false} -> + {next_state, blocked, Data1}; + %% failed and is async; remove the request from the queue, as + %% it is already in inflight table + {true, true} -> + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + Data = Data1#{queue := Q1}, + {next_state, blocked, Data}; + %% success; just ack + {false, _} -> + ok = replayq:ack(Q1, QAckRef), + emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + CurrentCount = replayq:count(Q1), + Data2 = Data1#{queue := Q1}, + case {CurrentCount > 0, CurrentCount >= BatchSize} of + {false, _} -> + {keep_state, Data2}; + {true, true} -> + {keep_state, Data2, [{next_event, internal, flush}]}; + {true, false} -> + Data3 = ensure_flush_timer(Data2), + {keep_state, Data3} + end end. batch_reply_caller(Id, BatchResult, Batch) -> @@ -352,11 +536,12 @@ batch_reply_caller(Id, BatchResult, Batch) -> ). reply_caller(Id, Reply) -> - reply_caller(Id, Reply, false). + BlockWorker = false, + reply_caller(Id, Reply, BlockWorker). -reply_caller(Id, ?REPLY(undefined, _, HasSent, Result), BlockWorker) -> - handle_query_result(Id, Result, HasSent, BlockWorker); -reply_caller(Id, ?REPLY({ReplyFun, Args}, _, HasSent, Result), BlockWorker) when +reply_caller(Id, ?REPLY(undefined, _, HasBeenSent, Result), BlockWorker) -> + handle_query_result(Id, Result, HasBeenSent, BlockWorker); +reply_caller(Id, ?REPLY({ReplyFun, Args}, _, HasBeenSent, Result), BlockWorker) when is_function(ReplyFun) -> _ = @@ -364,55 +549,55 @@ reply_caller(Id, ?REPLY({ReplyFun, Args}, _, HasSent, Result), BlockWorker) when {async_return, _} -> no_reply_for_now; _ -> apply(ReplyFun, Args ++ [Result]) end, - handle_query_result(Id, Result, HasSent, BlockWorker); -reply_caller(Id, ?REPLY(From, _, HasSent, Result), BlockWorker) -> + handle_query_result(Id, Result, HasBeenSent, BlockWorker); +reply_caller(Id, ?REPLY(From, _, HasBeenSent, Result), BlockWorker) -> gen_statem:reply(From, Result), - handle_query_result(Id, Result, HasSent, BlockWorker). + handle_query_result(Id, Result, HasBeenSent, BlockWorker). -handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), HasSent, BlockWorker) -> +handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), HasBeenSent, BlockWorker) -> ?SLOG(error, #{msg => resource_exception, info => Msg}), - inc_sent_failed(Id, HasSent), + inc_sent_failed(Id, HasBeenSent), BlockWorker; -handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasSent, _) when +handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasBeenSent, _) when NotWorking == not_connected; NotWorking == blocked -> true; -handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasSent, BlockWorker) -> +handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasBeenSent, BlockWorker) -> ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}), emqx_resource_metrics:dropped_resource_not_found_inc(Id), BlockWorker; -handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasSent, BlockWorker) -> +handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasBeenSent, BlockWorker) -> ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}), emqx_resource_metrics:dropped_resource_stopped_inc(Id), BlockWorker; -handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), _HasSent, BlockWorker) -> +handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), _HasBeenSent, BlockWorker) -> ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}), emqx_resource_metrics:dropped_other_inc(Id), BlockWorker; -handle_query_result(Id, {error, {recoverable_error, Reason}}, _HasSent, _BlockWorker) -> +handle_query_result(Id, {error, {recoverable_error, Reason}}, _HasBeenSent, _BlockWorker) -> %% the message will be queued in replayq or inflight window, %% i.e. the counter 'queuing' or 'dropped' will increase, so we pretend that we have not %% sent this message. ?SLOG(warning, #{id => Id, msg => recoverable_error, reason => Reason}), true; -handle_query_result(Id, {error, Reason}, HasSent, BlockWorker) -> +handle_query_result(Id, {error, Reason}, HasBeenSent, BlockWorker) -> ?SLOG(error, #{id => Id, msg => send_error, reason => Reason}), - inc_sent_failed(Id, HasSent), + inc_sent_failed(Id, HasBeenSent), BlockWorker; -handle_query_result(_Id, {async_return, inflight_full}, _HasSent, _BlockWorker) -> +handle_query_result(_Id, {async_return, inflight_full}, _HasBeenSent, _BlockWorker) -> true; -handle_query_result(Id, {async_return, {error, Msg}}, HasSent, BlockWorker) -> +handle_query_result(Id, {async_return, {error, Msg}}, HasBeenSent, BlockWorker) -> ?SLOG(error, #{id => Id, msg => async_send_error, info => Msg}), - inc_sent_failed(Id, HasSent), + inc_sent_failed(Id, HasBeenSent), BlockWorker; -handle_query_result(_Id, {async_return, ok}, _HasSent, BlockWorker) -> +handle_query_result(_Id, {async_return, ok}, _HasBeenSent, BlockWorker) -> BlockWorker; -handle_query_result(Id, Result, HasSent, BlockWorker) -> +handle_query_result(Id, Result, HasBeenSent, BlockWorker) -> assert_ok_result(Result), - inc_sent_success(Id, HasSent), + inc_sent_success(Id, HasBeenSent), BlockWorker. -call_query(QM0, Id, Query, QueryOpts) -> +call_query(QM0, Id, Index, Query, QueryOpts) -> ?tp(call_query_enter, #{id => Id, query => Query}), case emqx_resource_manager:ets_lookup(Id) of {ok, _Group, #{mod := Mod, state := ResSt, status := connected} = Data} -> @@ -422,13 +607,10 @@ call_query(QM0, Id, Query, QueryOpts) -> _ -> QM0 end, CM = maps:get(callback_mode, Data), - emqx_resource_metrics:matched_inc(Id), - apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts); + apply_query_fun(call_mode(QM, CM), Mod, Id, Index, Query, ResSt, QueryOpts); {ok, _Group, #{status := stopped}} -> - emqx_resource_metrics:matched_inc(Id), ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); {ok, _Group, #{status := S}} when S == connecting; S == disconnected -> - emqx_resource_metrics:matched_inc(Id), ?RESOURCE_ERROR(not_connected, "resource not connected"); {error, not_found} -> ?RESOURCE_ERROR(not_found, "resource not found") @@ -452,90 +634,80 @@ call_query(QM0, Id, Query, QueryOpts) -> end ). -apply_query_fun(sync, Mod, Id, ?QUERY(_, Request, _) = _Query, ResSt, _QueryOpts) -> +apply_query_fun(sync, Mod, Id, _Index, ?QUERY(_, Request, _) = _Query, ResSt, _QueryOpts) -> ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}), ?APPLY_RESOURCE(call_query, Mod:on_query(Id, Request, ResSt), Request); -apply_query_fun(async, Mod, Id, ?QUERY(_, Request, _) = Query, ResSt, QueryOpts) -> +apply_query_fun(async, Mod, Id, Index, ?QUERY(_, Request, _) = Query, ResSt, QueryOpts) -> ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}), Name = maps:get(inflight_name, QueryOpts, undefined), ?APPLY_RESOURCE( call_query_async, - case inflight_is_full(Name) of + case is_inflight_full(Name) of true -> {async_return, inflight_full}; false -> - ok = emqx_resource_metrics:inflight_change(Id, 1), - ReplyFun = fun ?MODULE:reply_after_query/6, + ReplyFun = fun ?MODULE:reply_after_query/7, Ref = make_message_ref(), - Args = [self(), Id, Name, Ref, Query], - ok = inflight_append(Name, Ref, Query), + Args = [self(), Id, Index, Name, Ref, Query], + ok = inflight_append(Name, Ref, Query, Id, Index), Result = Mod:on_query_async(Id, Request, {ReplyFun, Args}, ResSt), {async_return, Result} end, Request ); -apply_query_fun(sync, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, _QueryOpts) -> +apply_query_fun(sync, Mod, Id, _Index, [?QUERY(_, _, _) | _] = Batch, ResSt, _QueryOpts) -> ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), Requests = [Request || ?QUERY(_From, Request, _) <- Batch], ?APPLY_RESOURCE(call_batch_query, Mod:on_batch_query(Id, Requests, ResSt), Batch); -apply_query_fun(async, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, QueryOpts) -> +apply_query_fun(async, Mod, Id, Index, [?QUERY(_, _, _) | _] = Batch, ResSt, QueryOpts) -> ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), Name = maps:get(inflight_name, QueryOpts, undefined), ?APPLY_RESOURCE( call_batch_query_async, - case inflight_is_full(Name) of + case is_inflight_full(Name) of true -> {async_return, inflight_full}; false -> - BatchLen = length(Batch), - ok = emqx_resource_metrics:inflight_change(Id, BatchLen), - ReplyFun = fun ?MODULE:batch_reply_after_query/6, + ReplyFun = fun ?MODULE:batch_reply_after_query/7, Ref = make_message_ref(), - Args = {ReplyFun, [self(), Id, Name, Ref, Batch]}, + ReplyFunAndArgs = {ReplyFun, [self(), Id, Index, Name, Ref, Batch]}, Requests = [Request || ?QUERY(_From, Request, _) <- Batch], - ok = inflight_append(Name, Ref, Batch), - Result = Mod:on_batch_query_async(Id, Requests, Args, ResSt), + ok = inflight_append(Name, Ref, Batch, Id, Index), + Result = Mod:on_batch_query_async(Id, Requests, ReplyFunAndArgs, ResSt), {async_return, Result} end, Batch ). -reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request, HasSent), Result) -> - %% NOTE: 'inflight' is message count that sent async but no ACK received, - %% NOT the message number ququed in the inflight window. - emqx_resource_metrics:inflight_change(Id, -1), - case reply_caller(Id, ?REPLY(From, Request, HasSent, Result)) of +reply_after_query(Pid, Id, Index, Name, Ref, ?QUERY(From, Request, HasBeenSent), Result) -> + %% NOTE: 'inflight' is the count of messages that were sent async + %% but received no ACK, NOT the number of messages queued in the + %% inflight window. + case reply_caller(Id, ?REPLY(From, Request, HasBeenSent, Result)) of true -> - %% we marked these messages are 'queuing' although they are actually - %% keeped in inflight window, not replayq - emqx_resource_metrics:queuing_change(Id, 1), ?MODULE:block(Pid); false -> - drop_inflight_and_resume(Pid, Name, Ref) + drop_inflight_and_resume(Pid, Name, Ref, Id, Index) end. -batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) -> - %% NOTE: 'inflight' is message count that sent async but no ACK received, - %% NOT the message number ququed in the inflight window. - BatchLen = length(Batch), - emqx_resource_metrics:inflight_change(Id, -BatchLen), +batch_reply_after_query(Pid, Id, Index, Name, Ref, Batch, Result) -> + %% NOTE: 'inflight' is the count of messages that were sent async + %% but received no ACK, NOT the number of messages queued in the + %% inflight window. case batch_reply_caller(Id, Result, Batch) of true -> - %% we marked these messages are 'queuing' although they are actually - %% kept in inflight window, not replayq - emqx_resource_metrics:queuing_change(Id, BatchLen), ?MODULE:block(Pid); false -> - drop_inflight_and_resume(Pid, Name, Ref) + drop_inflight_and_resume(Pid, Name, Ref, Id, Index) end. -drop_inflight_and_resume(Pid, Name, Ref) -> - case inflight_is_full(Name) of +drop_inflight_and_resume(Pid, Name, Ref, Id, Index) -> + case is_inflight_full(Name) of true -> - inflight_drop(Name, Ref), + inflight_drop(Name, Ref, Id, Index), ?MODULE:resume(Pid); false -> - inflight_drop(Name, Ref) + inflight_drop(Name, Ref, Id, Index) end. %%============================================================================== @@ -548,10 +720,8 @@ queue_item_marshaller(Bin) when is_binary(Bin) -> estimate_size(QItem) -> size(queue_item_marshaller(QItem)). -maybe_append_queue(Id, undefined, _Items) -> - emqx_resource_metrics:dropped_queue_not_enabled_inc(Id), - undefined; -maybe_append_queue(Id, Q, Items) -> +-spec append_queue(id(), index(), replayq:q(), [queue_query()]) -> replayq:q(). +append_queue(Id, Index, Q, Queries) -> Q2 = case replayq:overflow(Q) of Overflow when Overflow =< 0 -> @@ -561,46 +731,42 @@ maybe_append_queue(Id, Q, Items) -> {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts), ok = replayq:ack(Q1, QAckRef), Dropped = length(Items2), - emqx_resource_metrics:queuing_change(Id, -Dropped), emqx_resource_metrics:dropped_queue_full_inc(Id), ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}), Q1 end, - emqx_resource_metrics:queuing_change(Id, 1), - replayq:append(Q2, Items). + Items = [?Q_ITEM(X) || X <- Queries], + Q3 = replayq:append(Q2, Items), + emqx_resource_metrics:queuing_set(Id, Index, replayq:count(Q3)), + ?tp(resource_worker_appended_to_queue, #{id => Id, items => Queries}), + Q3. +-spec get_first_n_from_queue(replayq:q(), pos_integer()) -> + empty | {replayq:q(), replayq:ack_ref(), [?Q_ITEM(?QUERY(_From, _Request, _HasBeenSent))]}. get_first_n_from_queue(Q, N) -> - get_first_n_from_queue(Q, N, []). - -get_first_n_from_queue(_Q, 0, Acc) -> - lists:reverse(Acc); -get_first_n_from_queue(Q, N, Acc) when N > 0 -> - case replayq:peek(Q) of - empty -> Acc; - ?Q_ITEM(Query) -> get_first_n_from_queue(Q, N - 1, [Query | Acc]) + case replayq:count(Q) of + 0 -> + empty; + _ -> + {NewQ, QAckRef, Items} = replayq:pop(Q, #{count_limit => N}), + Queries = [X || ?Q_ITEM(X) <- Items], + {NewQ, QAckRef, Queries} end. -drop_first_n_from_queue(Q, 0, _Id) -> - Q; -drop_first_n_from_queue(Q, N, Id) when N > 0 -> - drop_first_n_from_queue(drop_head(Q, Id), N - 1, Id). - -drop_head(Q, Id) -> - {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}), - ok = replayq:ack(Q1, AckRef), - emqx_resource_metrics:queuing_change(Id, -1), - Q1. - %%============================================================================== %% the inflight queue for async query --define(SIZE_REF, -1). -inflight_new(Name, InfltWinSZ) -> +-define(MAX_SIZE_REF, -1). +-define(SIZE_REF, -2). +inflight_new(Name, InfltWinSZ, Id, Index) -> _ = ets:new(Name, [named_table, ordered_set, public, {write_concurrency, true}]), - inflight_append(Name, ?SIZE_REF, {max_size, InfltWinSZ}), + inflight_append(Name, ?MAX_SIZE_REF, {max_size, InfltWinSZ}, Id, Index), + %% we use this counter because we might deal with batches as + %% elements. + inflight_append(Name, ?SIZE_REF, 0, Id, Index), ok. inflight_get_first(Name) -> - case ets:next(Name, ?SIZE_REF) of + case ets:next(Name, ?MAX_SIZE_REF) of '$end_of_table' -> empty; Ref -> @@ -613,43 +779,72 @@ inflight_get_first(Name) -> end end. -inflight_is_full(undefined) -> +is_inflight_full(undefined) -> false; -inflight_is_full(Name) -> - [{_, {max_size, MaxSize}}] = ets:lookup(Name, ?SIZE_REF), +is_inflight_full(Name) -> + [{_, {max_size, MaxSize}}] = ets:lookup(Name, ?MAX_SIZE_REF), + %% we consider number of batches rather than number of messages + %% because one batch request may hold several messages. + Size = inflight_num_batches(Name), + Size >= MaxSize. + +inflight_num_batches(Name) -> + %% Note: we subtract 2 because there're 2 metadata rows that hold + %% the maximum size value and the number of messages. + MetadataRowCount = 2, case ets:info(Name, size) of - Size when Size > MaxSize -> true; - _ -> false + undefined -> 0; + Size -> max(0, Size - MetadataRowCount) end. -inflight_append(undefined, _Ref, _Query) -> +inflight_num_msgs(Name) -> + [{_, Size}] = ets:lookup(Name, ?SIZE_REF), + Size. + +inflight_append(undefined, _Ref, _Query, _Id, _Index) -> ok; -inflight_append(Name, Ref, [?QUERY(_, _, _) | _] = Batch) -> - ets:insert(Name, {Ref, [?QUERY(From, Req, true) || ?QUERY(From, Req, _) <- Batch]}), +inflight_append(Name, Ref, [?QUERY(_, _, _) | _] = Batch0, Id, Index) -> + Batch = mark_as_sent(Batch0), + ets:insert(Name, {Ref, Batch}), + BatchSize = length(Batch), + ets:update_counter(Name, ?SIZE_REF, {2, BatchSize}), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(Name)), ok; -inflight_append(Name, Ref, ?QUERY(From, Req, _)) -> - ets:insert(Name, {Ref, ?QUERY(From, Req, true)}), +inflight_append(Name, Ref, ?QUERY(_From, _Req, _HasBeenSent) = Query0, Id, Index) -> + Query = mark_as_sent(Query0), + ets:insert(Name, {Ref, Query}), + ets:update_counter(Name, ?SIZE_REF, {2, 1}), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(Name)), ok; -inflight_append(Name, Ref, Data) -> +inflight_append(Name, Ref, Data, _Id, _Index) -> ets:insert(Name, {Ref, Data}), + %% this is a metadata row being inserted; therefore, we don't bump + %% the inflight metric. ok. -inflight_drop(undefined, _) -> +inflight_drop(undefined, _, _Id, _Index) -> ok; -inflight_drop(Name, Ref) -> - ets:delete(Name, Ref), +inflight_drop(Name, Ref, Id, Index) -> + Count = + case ets:take(Name, Ref) of + [{Ref, ?QUERY(_, _, _)}] -> 1; + [{Ref, [?QUERY(_, _, _) | _] = Batch}] -> length(Batch); + _ -> 0 + end, + Count > 0 andalso ets:update_counter(Name, ?SIZE_REF, {2, -Count, 0, 0}), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(Name)), ok. %%============================================================================== -inc_sent_failed(Id, _HasSent = true) -> +inc_sent_failed(Id, _HasBeenSent = true) -> emqx_resource_metrics:retried_failed_inc(Id); -inc_sent_failed(Id, _HasSent) -> +inc_sent_failed(Id, _HasBeenSent) -> emqx_resource_metrics:failed_inc(Id). -inc_sent_success(Id, _HasSent = true) -> +inc_sent_success(Id, _HasBeenSent = true) -> emqx_resource_metrics:retried_success_inc(Id); -inc_sent_success(Id, _HasSent) -> +inc_sent_success(Id, _HasBeenSent) -> emqx_resource_metrics:success_inc(Id). call_mode(sync, _) -> sync; @@ -670,8 +865,6 @@ assert_ok_result(R) when is_tuple(R) -> assert_ok_result(R) -> error({not_ok_result, R}). -queue_count(undefined) -> - 0; queue_count(Q) -> replayq:count(Q). @@ -686,12 +879,12 @@ disk_queue_dir(Id, Index) -> QDir = binary_to_list(Id) ++ ":" ++ integer_to_list(Index), filename:join([emqx:data_dir(), "resource_worker", node(), QDir]). -ensure_flush_timer(St = #{tref := undefined, batch_time := T}) -> +ensure_flush_timer(Data = #{tref := undefined, batch_time := T}) -> Ref = make_ref(), TRef = erlang:send_after(T, self(), {flush, Ref}), - St#{tref => {TRef, Ref}}; -ensure_flush_timer(St) -> - St. + Data#{tref => {TRef, Ref}}; +ensure_flush_timer(Data) -> + Data. cancel_flush_timer(St = #{tref := undefined}) -> St; @@ -701,3 +894,31 @@ cancel_flush_timer(St = #{tref := {TRef, _Ref}}) -> make_message_ref() -> erlang:unique_integer([monotonic, positive]). + +collect_requests(Acc, Limit) -> + Count = length(Acc), + do_collect_requests(Acc, Count, Limit). + +do_collect_requests(Acc, Count, Limit) when Count >= Limit -> + lists:reverse(Acc); +do_collect_requests(Acc, Count, Limit) -> + receive + ?SEND_REQ(_From, _Req) = Request -> + do_collect_requests([Request | Acc], Count + 1, Limit) + after 0 -> + lists:reverse(Acc) + end. + +mark_as_sent(Batch) when is_list(Batch) -> + lists:map(fun mark_as_sent/1, Batch); +mark_as_sent(?QUERY(From, Req, _)) -> + HasBeenSent = true, + ?QUERY(From, Req, HasBeenSent). + +is_async(ResourceId) -> + case emqx_resource_manager:ets_lookup(ResourceId) of + {ok, _Group, #{query_mode := QM, callback_mode := CM}} -> + call_mode(QM, CM) =:= async; + _ -> + false + end. diff --git a/apps/emqx_resource/src/emqx_resource_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_worker_sup.erl index 2db7b5c4c..b6557620c 100644 --- a/apps/emqx_resource/src/emqx_resource_worker_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_worker_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl index 11af1a62c..72175865a 100644 --- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl +++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 465a935c3..d105b21ef 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -83,12 +83,14 @@ query_mode(_) -> undefined. enable_batch(type) -> boolean(); enable_batch(required) -> false; enable_batch(default) -> true; +enable_batch(deprecated) -> {since, "v5.0.14"}; enable_batch(desc) -> ?DESC("enable_batch"); enable_batch(_) -> undefined. enable_queue(type) -> boolean(); enable_queue(required) -> false; enable_queue(default) -> false; +enable_queue(deprecated) -> {since, "v5.0.14"}; enable_queue(desc) -> ?DESC("enable_queue"); enable_queue(_) -> undefined. diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 105bcad77..692895548 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ -module(emqx_connector_demo). -include_lib("typerefl/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(emqx_resource). @@ -28,6 +29,7 @@ on_query/3, on_query_async/4, on_batch_query/3, + on_batch_query_async/4, on_get_status/2 ]). @@ -36,6 +38,8 @@ %% callbacks for emqx_resource config schema -export([roots/0]). +-define(CM_KEY, {?MODULE, callback_mode}). + roots() -> [ {name, fun name/1}, @@ -51,7 +55,6 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. --define(CM_KEY, {?MODULE, callback_mode}). callback_mode() -> persistent_term:get(?CM_KEY). @@ -60,17 +63,12 @@ set_callback_mode(Mode) -> on_start(_InstId, #{create_error := true}) -> error("some error"); -on_start(InstId, #{name := Name, stop_error := true} = Opts) -> - Register = maps:get(register, Opts, false), - {ok, Opts#{ - id => InstId, - stop_error => true, - pid => spawn_counter_process(Name, Register) - }}; on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), + StopError = maps:get(stop_error, Opts, false), {ok, Opts#{ id => InstId, + stop_error => StopError, pid => spawn_counter_process(Name, Register) }}. @@ -95,8 +93,11 @@ on_query(_InstId, {inc_counter, N}, #{pid := Pid}) -> From = {self(), ReqRef}, Pid ! {From, {inc, N}}, receive - {ReqRef, ok} -> ok; - {ReqRef, incorrect_status} -> {error, {recoverable_error, incorrect_status}} + {ReqRef, ok} -> + ?tp(connector_demo_inc_counter, #{n => N}), + ok; + {ReqRef, incorrect_status} -> + {error, {recoverable_error, incorrect_status}} after 1000 -> {error, timeout} end; @@ -127,18 +128,30 @@ on_query_async(_InstId, get_counter, ReplyFun, #{pid := Pid}) -> ok. on_batch_query(InstId, BatchReq, State) -> - %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed. + %% Requests can be either 'get_counter' or 'inc_counter', but + %% cannot be mixed. case hd(BatchReq) of {inc_counter, _} -> - batch_inc_counter(InstId, BatchReq, State); + batch_inc_counter(sync, InstId, BatchReq, State); get_counter -> - batch_get_counter(InstId, State) + batch_get_counter(sync, InstId, State) end. -batch_inc_counter(InstId, BatchReq, State) -> +on_batch_query_async(InstId, BatchReq, ReplyFunAndArgs, State) -> + %% Requests can be either 'get_counter' or 'inc_counter', but + %% cannot be mixed. + case hd(BatchReq) of + {inc_counter, _} -> + batch_inc_counter({async, ReplyFunAndArgs}, InstId, BatchReq, State); + get_counter -> + batch_get_counter({async, ReplyFunAndArgs}, InstId, State) + end. + +batch_inc_counter(CallMode, InstId, BatchReq, State) -> TotalN = lists:foldl( fun ({inc_counter, N}, Total) -> + ?tp(connector_demo_batch_inc_individual, #{n => N}), Total + N; (Req, _Total) -> error({mixed_requests_not_allowed, {inc_counter, Req}}) @@ -146,10 +159,17 @@ batch_inc_counter(InstId, BatchReq, State) -> 0, BatchReq ), - on_query(InstId, {inc_counter, TotalN}, State). + case CallMode of + sync -> + on_query(InstId, {inc_counter, TotalN}, State); + {async, ReplyFunAndArgs} -> + on_query_async(InstId, {inc_counter, TotalN}, ReplyFunAndArgs, State) + end. -batch_get_counter(InstId, State) -> - on_query(InstId, get_counter, State). +batch_get_counter(sync, InstId, State) -> + on_query(InstId, get_counter, State); +batch_get_counter({async, ReplyFunAndArgs}, InstId, State) -> + on_query_async(InstId, get_counter, ReplyFunAndArgs, State). on_get_status(_InstId, #{health_check_error := true}) -> disconnected; @@ -187,6 +207,7 @@ counter_loop( {inc, N, ReplyFun} when Status == running -> %ct:pal("async counter recv: ~p", [{inc, N}]), apply_reply(ReplyFun, ok), + ?tp(connector_demo_inc_counter_async, #{n => N}), State#{counter => Num + N}; {{FromPid, ReqRef}, {inc, N}} when Status == running -> %ct:pal("sync counter recv: ~p", [{inc, N}]), diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 107ca2a93..4bea0a1ee 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,6 +30,8 @@ -define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}). -define(TRACE_OPTS, #{timetrap => 10000, timeout => 1000}). +-import(emqx_common_test_helpers, [on_exit/1]). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,10 +39,15 @@ groups() -> []. init_per_testcase(_, Config) -> + ct:timetrap({seconds, 30}), emqx_connector_demo:set_callback_mode(always_sync), Config. + end_per_testcase(_, _Config) -> - _ = emqx_resource:remove(?ID). + snabbkaffe:stop(), + _ = emqx_resource:remove(?ID), + emqx_common_test_helpers:call_janitor(), + ok. init_per_suite(Config) -> code:ensure_loaded(?TEST_RESOURCE), @@ -139,6 +146,7 @@ t_create_remove_local(_) -> ?assertNot(is_process_alive(Pid)). t_do_not_start_after_created(_) -> + ct:pal("creating resource"), {ok, _} = emqx_resource:create_local( ?ID, ?DEFAULT_RESOURCE_GROUP, @@ -158,16 +166,19 @@ t_do_not_start_after_created(_) -> ), %% start the resource manually.. + ct:pal("starting resource manually"), ok = emqx_resource:start(?ID), {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid)), %% restart the resource + ct:pal("restarting resource"), ok = emqx_resource:restart(?ID), ?assertNot(is_process_alive(Pid)), {ok, #{pid := Pid2}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid2)), + ct:pal("removing resource"), ok = emqx_resource:remove_local(?ID), ?assertNot(is_process_alive(Pid2)). @@ -206,12 +217,13 @@ t_query_counter(_) -> ok = emqx_resource:remove_local(?ID). t_batch_query_counter(_) -> + BatchSize = 100, {ok, _} = emqx_resource:create_local( ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{enable_batch => true, query_mode => sync} + #{batch_size => BatchSize, query_mode => sync} ), ?check_trace( @@ -224,15 +236,26 @@ t_batch_query_counter(_) -> end ), + NMsgs = 1_000, ?check_trace( ?TRACE_OPTS, - inc_counter_in_parallel(1000), + begin + NEvents = round(math:ceil(NMsgs / BatchSize)), + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter}), + NEvents, + _Timeout = 10_000 + ), + inc_counter_in_parallel(NMsgs), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, fun(Trace) -> QueryTrace = ?of_kind(call_batch_query, Trace), ?assertMatch([#{batch := BatchReq} | _] when length(BatchReq) > 1, QueryTrace) end ), - {ok, 1000} = emqx_resource:query(?ID, get_counter), + {ok, NMsgs} = emqx_resource:query(?ID, get_counter), ok = emqx_resource:remove_local(?ID). @@ -242,20 +265,28 @@ t_query_counter_async_query(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{query_mode => async, enable_batch => false} + #{query_mode => async, batch_size => 1} ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + NMsgs = 1_000, ?check_trace( ?TRACE_OPTS, - inc_counter_in_parallel(1000), + begin + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter}), + NMsgs, + _Timeout = 60_000 + ), + inc_counter_in_parallel(NMsgs), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, fun(Trace) -> - %% the callback_mode if 'emqx_connector_demo' is 'always_sync'. + %% the callback_mode of 'emqx_connector_demo' is 'always_sync'. QueryTrace = ?of_kind(call_query, Trace), ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace) end ), - %% wait for 1s to make sure all the aysnc query is sent to the resource. - timer:sleep(1000), %% simple query ignores the query_mode and batching settings in the resource_worker ?check_trace( ?TRACE_OPTS, @@ -284,20 +315,32 @@ t_query_counter_async_callback(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{query_mode => async, enable_batch => false, async_inflight_window => 1000000} + #{ + query_mode => async, + batch_size => 1, + async_inflight_window => 1000000 + } ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + NMsgs = 1_000, ?check_trace( ?TRACE_OPTS, - inc_counter_in_parallel(1000, ReqOpts), + begin + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + NMsgs, + _Timeout = 60_000 + ), + inc_counter_in_parallel(NMsgs, ReqOpts), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, fun(Trace) -> QueryTrace = ?of_kind(call_query_async, Trace), ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace) end ), - %% wait for 1s to make sure all the aysnc query is sent to the resource. - timer:sleep(1000), %% simple query ignores the query_mode and batching settings in the resource_worker ?check_trace( ?TRACE_OPTS, @@ -324,12 +367,29 @@ t_query_counter_async_callback(_) -> t_query_counter_async_inflight(_) -> emqx_connector_demo:set_callback_mode(async_if_possible), + MetricsTab = ets:new(metrics_tab, [ordered_set, public]), + ok = telemetry:attach_many( + ?FUNCTION_NAME, + emqx_resource_metrics:events(), + fun(Event, Measurements, Meta, _Config) -> + ets:insert( + MetricsTab, + {erlang:monotonic_time(), #{ + event => Event, measurements => Measurements, metadata => Meta + }} + ), + ok + end, + unused_config + ), + on_exit(fun() -> telemetry:detach(?FUNCTION_NAME) end), Tab0 = ets:new(?FUNCTION_NAME, [bag, public]), - Insert0 = fun(Tab, Result) -> - ets:insert(Tab, {make_ref(), Result}) + Insert0 = fun(Tab, Ref, Result) -> + ct:pal("inserting ~p", [{Ref, Result}]), + ets:insert(Tab, {Ref, Result}) end, - ReqOpts = #{async_reply_fun => {Insert0, [Tab0]}}, + ReqOpts = fun() -> #{async_reply_fun => {Insert0, [Tab0, make_ref()]}} end, WindowSize = 15, {ok, _} = emqx_resource:create_local( ?ID, @@ -338,11 +398,10 @@ t_query_counter_async_inflight(_) -> #{name => test_resource, register => true}, #{ query_mode => async, - enable_batch => false, + batch_size => 1, async_inflight_window => WindowSize, worker_pool_size => 1, - resume_interval => 300, - enable_queue => false + resume_interval => 300 } ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), @@ -359,40 +418,76 @@ t_query_counter_async_inflight(_) -> ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace) end ), + tap_metrics(?LINE), %% this will block the resource_worker as the inflight window is full now - ok = emqx_resource:query(?ID, {inc_counter, 1}), + {ok, {ok, _}} = + ?wait_async_action( + emqx_resource:query(?ID, {inc_counter, 2}), + #{?snk_kind := resource_worker_enter_blocked}, + 1_000 + ), ?assertMatch(0, ets:info(Tab0, size)), - %% sleep to make the resource_worker resume some times - timer:sleep(2000), + tap_metrics(?LINE), %% send query now will fail because the resource is blocked. Insert = fun(Tab, Ref, Result) -> - ets:insert(Tab, {Ref, Result}) + ct:pal("inserting ~p", [{Ref, Result}]), + ets:insert(Tab, {Ref, Result}), + ?tp(tmp_query_inserted, #{}) end, - ok = emqx_resource:query(?ID, {inc_counter, 1}, #{ - async_reply_fun => {Insert, [Tab0, tmp_query]} - }), - timer:sleep(100), + {ok, {ok, _}} = + ?wait_async_action( + emqx_resource:query(?ID, {inc_counter, 3}, #{ + async_reply_fun => {Insert, [Tab0, tmp_query]} + }), + #{?snk_kind := tmp_query_inserted}, + 1_000 + ), + %% since this counts as a failure, it'll be enqueued and retried + %% later, when the resource is unblocked. ?assertMatch([{_, {error, {resource_error, #{reason := blocked}}}}], ets:take(Tab0, tmp_query)), + tap_metrics(?LINE), - %% all response should be received after the resource is resumed. + %% all responses should be received after the resource is resumed. + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + %% +1 because the tmp_query above will be retried and succeed + %% this time. + WindowSize + 1, + _Timeout = 60_000 + ), ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), - timer:sleep(1000), + tap_metrics(?LINE), + {ok, _} = snabbkaffe:receive_events(SRef0), + %% since the previous tmp_query was enqueued to be retried, we + %% take it again from the table; this time, it should have + %% succeeded. + ?assertMatch([{tmp_query, ok}], ets:take(Tab0, tmp_query)), ?assertEqual(WindowSize, ets:info(Tab0, size)), + tap_metrics(?LINE), %% send async query, this time everything should be ok. Num = 10, ?check_trace( ?TRACE_OPTS, - inc_counter_in_parallel(Num, ReqOpts), + begin + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + Num, + _Timeout = 60_000 + ), + inc_counter_in_parallel(Num, ReqOpts), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, fun(Trace) -> QueryTrace = ?of_kind(call_query_async, Trace), - ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace) + ?assertMatch([#{query := {query, _, {inc_counter, _}, _}} | _], QueryTrace) end ), - timer:sleep(1000), - ?assertEqual(WindowSize + Num, ets:info(Tab0, size)), + ?assertEqual(WindowSize + Num, ets:info(Tab0, size), #{tab => ets:tab2list(Tab0)}), + tap_metrics(?LINE), %% block the resource ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), @@ -410,27 +505,253 @@ t_query_counter_async_inflight(_) -> ok = emqx_resource:query(?ID, {inc_counter, 1}), Sent = WindowSize + Num + WindowSize, + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + WindowSize, + _Timeout = 60_000 + ), ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), - timer:sleep(1000), + {ok, _} = snabbkaffe:receive_events(SRef1), ?assertEqual(Sent, ets:info(Tab0, size)), {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter), ct:pal("get_counter: ~p, sent: ~p", [Counter, Sent]), ?assert(Sent =< Counter), - {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), - ct:pal("metrics: ~p", [C]), - {ok, IncorrectStatusCount} = emqx_resource:simple_sync_query(?ID, get_incorrect_status_count), - %% The `simple_sync_query' we just did also increases the matched - %% count, hence the + 1. - ExtraSimpleCallCount = IncorrectStatusCount + 1, + %% give the metrics some time to stabilize. + ct:sleep(1000), + #{counters := C, gauges := G} = tap_metrics(?LINE), ?assertMatch( - #{matched := M, success := Ss, dropped := Dp, 'retried.success' := Rs} when - M == Ss + Dp - Rs + ExtraSimpleCallCount, - C, #{ - metrics => C, - extra_simple_call_count => ExtraSimpleCallCount + counters := + #{matched := M, success := Ss, dropped := Dp}, + gauges := #{queuing := Qing, inflight := Infl} + } when + M == Ss + Dp + Qing + Infl, + #{counters => C, gauges => G}, + #{ + metrics => #{counters => C, gauges => G}, + results => ets:tab2list(Tab0), + metrics_trace => ets:tab2list(MetricsTab) + } + ), + ?assert( + lists:all( + fun + ({_, ok}) -> true; + (_) -> false + end, + ets:tab2list(Tab0) + ) + ), + ok = emqx_resource:remove_local(?ID). + +t_query_counter_async_inflight_batch(_) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + MetricsTab = ets:new(metrics_tab, [ordered_set, public]), + ok = telemetry:attach_many( + ?FUNCTION_NAME, + emqx_resource_metrics:events(), + fun(Event, Measurements, Meta, _Config) -> + ets:insert( + MetricsTab, + {erlang:monotonic_time(), #{ + event => Event, measurements => Measurements, metadata => Meta + }} + ), + ok + end, + unused_config + ), + on_exit(fun() -> telemetry:detach(?FUNCTION_NAME) end), + + Tab0 = ets:new(?FUNCTION_NAME, [bag, public]), + Insert0 = fun(Tab, Ref, Result) -> + ct:pal("inserting ~p", [{Ref, Result}]), + ets:insert(Tab, {Ref, Result}) + end, + ReqOpts = fun() -> #{async_reply_fun => {Insert0, [Tab0, make_ref()]}} end, + BatchSize = 2, + WindowSize = 3, + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true}, + #{ + query_mode => async, + batch_size => BatchSize, + async_inflight_window => WindowSize, + worker_pool_size => 1, + resume_interval => 300 + } + ), + ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + + %% block the resource + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), + + %% send async query to make the inflight window full + NumMsgs = BatchSize * WindowSize, + ?check_trace( + begin + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := call_batch_query_async}), + WindowSize, + _Timeout = 60_000 + ), + inc_counter_in_parallel(NumMsgs, ReqOpts), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, + fun(Trace) -> + QueryTrace = ?of_kind(call_batch_query_async, Trace), + ?assertMatch( + [ + #{ + batch := [ + {query, _, {inc_counter, 1}, _}, + {query, _, {inc_counter, 1}, _} + ] + } + | _ + ], + QueryTrace + ) + end + ), + tap_metrics(?LINE), + + ?check_trace( + begin + %% this will block the resource_worker as the inflight window is full now + {ok, {ok, _}} = + ?wait_async_action( + emqx_resource:query(?ID, {inc_counter, 2}), + #{?snk_kind := resource_worker_enter_blocked}, + 5_000 + ), + ?assertMatch(0, ets:info(Tab0, size)), + ok + end, + [] + ), + + tap_metrics(?LINE), + %% send query now will fail because the resource is blocked. + Insert = fun(Tab, Ref, Result) -> + ct:pal("inserting ~p", [{Ref, Result}]), + ets:insert(Tab, {Ref, Result}), + ?tp(tmp_query_inserted, #{}) + end, + {ok, {ok, _}} = + ?wait_async_action( + emqx_resource:query(?ID, {inc_counter, 3}, #{ + async_reply_fun => {Insert, [Tab0, tmp_query]} + }), + #{?snk_kind := tmp_query_inserted}, + 1_000 + ), + %% since this counts as a failure, it'll be enqueued and retried + %% later, when the resource is unblocked. + ?assertMatch([{_, {error, {resource_error, #{reason := blocked}}}}], ets:take(Tab0, tmp_query)), + tap_metrics(?LINE), + + %% all responses should be received after the resource is resumed. + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + %% +1 because the tmp_query above will be retried and succeed + %% this time. + WindowSize + 1, + _Timeout = 60_000 + ), + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), + tap_metrics(?LINE), + {ok, _} = snabbkaffe:receive_events(SRef0), + %% since the previous tmp_query was enqueued to be retried, we + %% take it again from the table; this time, it should have + %% succeeded. + ?assertMatch([{tmp_query, ok}], ets:take(Tab0, tmp_query)), + ?assertEqual(NumMsgs, ets:info(Tab0, size), #{tab => ets:tab2list(Tab0)}), + tap_metrics(?LINE), + + %% send async query, this time everything should be ok. + NumBatches1 = 3, + NumMsgs1 = BatchSize * NumBatches1, + ?check_trace( + ?TRACE_OPTS, + begin + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + NumBatches1, + _Timeout = 60_000 + ), + inc_counter_in_parallel(NumMsgs1, ReqOpts), + {ok, _} = snabbkaffe:receive_events(SRef), + ok + end, + fun(Trace) -> + QueryTrace = ?of_kind(call_batch_query_async, Trace), + ?assertMatch( + [#{batch := [{query, _, {inc_counter, _}, _} | _]} | _], + QueryTrace + ) + end + ), + ?assertEqual( + NumMsgs + NumMsgs1, + ets:info(Tab0, size), + #{tab => ets:tab2list(Tab0)} + ), + tap_metrics(?LINE), + + %% block the resource + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), + %% again, send async query to make the inflight window full + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(WindowSize, ReqOpts), + fun(Trace) -> + QueryTrace = ?of_kind(call_batch_query_async, Trace), + ?assertMatch( + [#{batch := [{query, _, {inc_counter, _}, _} | _]} | _], + QueryTrace + ) + end + ), + + %% this will block the resource_worker + ok = emqx_resource:query(?ID, {inc_counter, 1}), + + Sent = NumMsgs + NumMsgs1 + WindowSize, + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := connector_demo_inc_counter_async}), + WindowSize, + _Timeout = 60_000 + ), + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), + {ok, _} = snabbkaffe:receive_events(SRef1), + ?assertEqual(Sent, ets:info(Tab0, size)), + + {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter), + ct:pal("get_counter: ~p, sent: ~p", [Counter, Sent]), + ?assert(Sent =< Counter), + + %% give the metrics some time to stabilize. + ct:sleep(1000), + #{counters := C, gauges := G} = tap_metrics(?LINE), + ?assertMatch( + #{ + counters := + #{matched := M, success := Ss, dropped := Dp}, + gauges := #{queuing := Qing, inflight := Infl} + } when + M == Ss + Dp + Qing + Infl, + #{counters => C, gauges => G}, + #{ + metrics => #{counters => C, gauges => G}, + results => ets:tab2list(Tab0), + metrics_trace => ets:tab2list(MetricsTab) } ), ?assert( @@ -503,8 +824,11 @@ t_stop_start(_) -> ), %% add some metrics to test their persistence - emqx_resource_metrics:batching_change(?ID, 5), - ?assertEqual(5, emqx_resource_metrics:batching_get(?ID)), + WorkerID0 = <<"worker:0">>, + WorkerID1 = <<"worker:1">>, + emqx_resource_metrics:inflight_set(?ID, WorkerID0, 2), + emqx_resource_metrics:inflight_set(?ID, WorkerID1, 3), + ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), {ok, _} = emqx_resource:check_and_recreate( ?ID, @@ -518,7 +842,7 @@ t_stop_start(_) -> ?assert(is_process_alive(Pid0)), %% metrics are reset when recreating - ?assertEqual(0, emqx_resource_metrics:batching_get(?ID)), + ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), ok = emqx_resource:stop(?ID), @@ -537,10 +861,11 @@ t_stop_start(_) -> ?assert(is_process_alive(Pid1)), %% now stop while resetting the metrics - emqx_resource_metrics:batching_change(?ID, 5), - ?assertEqual(5, emqx_resource_metrics:batching_get(?ID)), + emqx_resource_metrics:inflight_set(?ID, WorkerID0, 1), + emqx_resource_metrics:inflight_set(?ID, WorkerID1, 4), + ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), ok = emqx_resource:stop(?ID), - ?assertEqual(0, emqx_resource_metrics:batching_get(?ID)), + ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), ok. @@ -636,18 +961,22 @@ create_dry_run_local_succ() -> ?assertEqual(undefined, whereis(test_resource)). t_create_dry_run_local_failed(_) -> + ct:timetrap({seconds, 120}), + ct:pal("creating with creation error"), Res1 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, #{create_error => true} ), ?assertMatch({error, _}, Res1), + ct:pal("creating with health check error"), Res2 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), ?assertMatch({error, _}, Res2), + ct:pal("creating with stop error"), Res3 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, #{name => test_resource, stop_error => true} @@ -684,16 +1013,116 @@ t_auto_retry(_) -> ), ?assertEqual(ok, Res). +t_retry_batch(_Config) -> + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + batch_size => 5, + worker_pool_size => 1, + resume_interval => 1_000 + } + ), + + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), + Matched0 = emqx_resource_metrics:matched_get(?ID), + ?assertEqual(1, Matched0), + + %% these requests will batch together and fail; the buffer worker + %% will enter the `blocked' state and they'll be retried later, + %% after it unblocks. + Payloads = lists:seq(1, 5), + NumPayloads = length(Payloads), + ExpectedCount = 15, + + ?check_trace( + begin + {ok, {ok, _}} = + ?wait_async_action( + lists:foreach( + fun(N) -> + ok = emqx_resource:query(?ID, {inc_counter, N}) + end, + Payloads + ), + #{?snk_kind := resource_worker_enter_blocked}, + 5_000 + ), + %% now the individual messages should have been counted + Matched1 = emqx_resource_metrics:matched_get(?ID), + ?assertEqual(Matched0 + NumPayloads, Matched1), + + %% wait for two more retries while the failure is enabled; the + %% batch shall remain enqueued. + {ok, _} = + snabbkaffe:block_until( + ?match_n_events(2, #{?snk_kind := resource_worker_retry_queue_batch_failed}), + 5_000 + ), + %% should not have increased the matched count with the retries + Matched2 = emqx_resource_metrics:matched_get(?ID), + ?assertEqual(Matched1, Matched2), + + %% now unblock the buffer worker so it may retry the batch, + %% but it'll still fail + {ok, {ok, _}} = + ?wait_async_action( + ok = emqx_resource:simple_sync_query(?ID, resume), + #{?snk_kind := resource_worker_retry_queue_batch_succeeded}, + 5_000 + ), + %% 1 more because of the `resume' call + Matched3 = emqx_resource_metrics:matched_get(?ID), + ?assertEqual(Matched2 + 1, Matched3), + + {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter), + {Counter, Matched3} + end, + fun({Counter, Matched3}, Trace) -> + %% 1 original attempt + 2 failed retries + final + %% successful attempt. + %% each time should be the original batch (no duplicate + %% elements or reordering). + ExpectedSeenPayloads = lists:flatten(lists:duplicate(4, Payloads)), + ?assertEqual( + ExpectedSeenPayloads, + ?projection(n, ?of_kind(connector_demo_batch_inc_individual, Trace)) + ), + ?assertMatch( + [#{n := ExpectedCount}], + ?of_kind(connector_demo_inc_counter, Trace) + ), + ?assertEqual(ExpectedCount, Counter), + %% matched should count only the original requests, and not retries + %% + 1 for `resume' call + %% + 1 for `block' call + %% + 1 for `get_counter' call + %% and the message count (1 time) + Matched4 = emqx_resource_metrics:matched_get(?ID), + ?assertEqual(Matched3 + 1, Matched4), + ok + end + ), + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ inc_counter_in_parallel(N) -> inc_counter_in_parallel(N, #{}). -inc_counter_in_parallel(N, Opts) -> +inc_counter_in_parallel(N, Opts0) -> Parent = self(), Pids = [ erlang:spawn(fun() -> + Opts = + case is_function(Opts0) of + true -> Opts0(); + false -> Opts0 + end, emqx_resource:query(?ID, {inc_counter, 1}, Opts), Parent ! {complete, self()} end) @@ -714,3 +1143,8 @@ bin_config() -> config() -> {ok, Config} = hocon:binary(bin_config()), Config. + +tap_metrics(Line) -> + {ok, _, #{metrics := #{counters := C, gauges := G}}} = emqx_resource:get_instance(?ID), + ct:pal("metrics (l. ~b): ~p", [Line, #{counters => C, gauges => G}]), + #{counters => C, gauges => G}. diff --git a/apps/emqx_retainer/include/emqx_retainer.hrl b/apps/emqx_retainer/include/emqx_retainer.hrl index e5d8a3112..e4a24f9e2 100644 --- a/apps/emqx_retainer/include/emqx_retainer.hrl +++ b/apps/emqx_retainer/include/emqx_retainer.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index f61468d9b..d151ad4e7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d911b5f4..b81ea2446 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -321,16 +321,23 @@ update_config( OldConf ) -> #{ - backend := BackendCfg, + backend := #{ + type := BackendType, + storage_type := StorageType + }, msg_clear_interval := ClearInterval } = NewConf, - #{backend := OldBackendCfg} = OldConf, - - StorageType = maps:get(type, BackendCfg), - OldStrorageType = maps:get(type, OldBackendCfg), - case OldStrorageType of - StorageType -> + #{ + backend := #{ + type := OldBackendType, + storage_type := OldStorageType + } + } = OldConf, + SameBackendType = BackendType =:= OldBackendType, + SameStorageType = StorageType =:= OldStorageType, + case SameBackendType andalso SameStorageType of + true -> State#{ clear_timer := check_timer( ClearTimer, @@ -338,7 +345,7 @@ update_config( clear_expired ) }; - _ -> + false -> State2 = disable_retainer(State), enable_retainer(State2, NewConf) end. diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 2c0bd725c..fa11b00f4 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_app.erl b/apps/emqx_retainer/src/emqx_retainer_app.erl index 061679cf7..77429e7a6 100644 --- a/apps/emqx_retainer/src/emqx_retainer_app.erl +++ b/apps/emqx_retainer/src/emqx_retainer_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index abecbbeb1..454a65eb3 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_index.erl b/apps/emqx_retainer/src/emqx_retainer_index.erl index 3f5b87b90..55fe7ce3f 100644 --- a/apps/emqx_retainer/src/emqx_retainer_index.erl +++ b/apps/emqx_retainer/src/emqx_retainer_index.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index d147877e8..cadb9110f 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -146,7 +146,9 @@ store_retained(_, Msg = #message{topic = Topic}) -> reason => table_is_full }); false -> - do_store_retained(Msg, Tokens, ExpiryTime) + do_store_retained(Msg, Tokens, ExpiryTime), + ?tp(message_retained, #{topic => Topic}), + ok end. clear_expired(_) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl index 402c8003f..5710e4df3 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 25fd27538..472ecc284 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_retainer_schema). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 9938f9881..307357c0a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/proto/emqx_retainer_proto_v1.erl b/apps/emqx_retainer/src/proto/emqx_retainer_proto_v1.erl index dfe71b196..025c6b928 100644 --- a/apps/emqx_retainer/src/proto/emqx_retainer_proto_v1.erl +++ b/apps/emqx_retainer/src/proto/emqx_retainer_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/proto/emqx_retainer_proto_v2.erl b/apps/emqx_retainer/src/proto/emqx_retainer_proto_v2.erl index 4b98f945f..a0d9288ff 100644 --- a/apps/emqx_retainer/src/proto/emqx_retainer_proto_v2.erl +++ b/apps/emqx_retainer/src/proto/emqx_retainer_proto_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index e6f4a404e..845f07802 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index f97284a84..ba96887a2 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -103,11 +103,12 @@ t_messages(_) -> end, ?check_trace( - ?wait_async_action( - lists:foreach(Each, lists:seq(1, 5)), - #{?snk_kind := message_retained, topic := <<"retained/A">>}, - 500 - ), + {ok, {ok, _}} = + ?wait_async_action( + lists:foreach(Each, lists:seq(1, 5)), + #{?snk_kind := message_retained, topic := <<"retained/A">>}, + 500 + ), [] ), @@ -149,11 +150,12 @@ t_messages_page(_) -> end, ?check_trace( - ?wait_async_action( - lists:foreach(Each, lists:seq(1, 5)), - #{?snk_kind := message_retained, topic := <<"retained/A">>}, - 500 - ), + {ok, {ok, _}} = + ?wait_async_action( + lists:foreach(Each, lists:seq(1, 5)), + #{?snk_kind := message_retained, topic := <<"retained/A">>}, + 500 + ), [] ), Page = 4, @@ -219,6 +221,92 @@ t_lookup_and_delete(_) -> ok = emqtt:disconnect(C1). +t_change_storage_type(_Config) -> + Path = api_path(["mqtt", "retainer"]), + {ok, ConfJson} = request_api(get, Path), + RawConf = emqx_json:decode(ConfJson, [return_maps]), + %% pre-conditions + ?assertMatch( + #{ + <<"backend">> := #{ + <<"type">> := <<"built_in_database">>, + <<"storage_type">> := <<"ram">> + }, + <<"enable">> := true + }, + RawConf + ), + ?assertEqual(ram_copies, mnesia:table_info(?TAB_INDEX_META, storage_type)), + ?assertEqual(ram_copies, mnesia:table_info(?TAB_MESSAGE, storage_type)), + ?assertEqual(ram_copies, mnesia:table_info(?TAB_INDEX, storage_type)), + %% insert some retained messages + {ok, C0} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C0), + ok = snabbkaffe:start_trace(), + Topic = <<"retained">>, + Payload = <<"retained">>, + {ok, {ok, _}} = + ?wait_async_action( + emqtt:publish(C0, Topic, Payload, [{qos, 0}, {retain, true}]), + #{?snk_kind := message_retained, topic := Topic}, + 500 + ), + emqtt:stop(C0), + ok = snabbkaffe:stop(), + {ok, MsgsJson0} = request_api(get, api_path(["mqtt", "retainer", "messages"])), + #{data := Msgs0, meta := _} = decode_json(MsgsJson0), + ?assertEqual(1, length(Msgs0)), + + ChangedConf = emqx_map_lib:deep_merge( + RawConf, + #{ + <<"backend">> => + #{<<"storage_type">> => <<"disc">>} + } + ), + {ok, UpdateResJson} = request_api( + put, + Path, + [], + auth_header_(), + ChangedConf + ), + UpdatedRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + ?assertMatch( + #{ + <<"backend">> := #{ + <<"type">> := <<"built_in_database">>, + <<"storage_type">> := <<"disc">> + }, + <<"enable">> := true + }, + UpdatedRawConf + ), + ?assertEqual(disc_copies, mnesia:table_info(?TAB_INDEX_META, storage_type)), + ?assertEqual(disc_copies, mnesia:table_info(?TAB_MESSAGE, storage_type)), + ?assertEqual(disc_copies, mnesia:table_info(?TAB_INDEX, storage_type)), + %% keep retained messages + {ok, MsgsJson1} = request_api(get, api_path(["mqtt", "retainer", "messages"])), + #{data := Msgs1, meta := _} = decode_json(MsgsJson1), + ?assertEqual(1, length(Msgs1)), + {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + {ok, _, _} = emqtt:subscribe(C1, Topic), + + receive + {publish, #{topic := T, payload := P, retain := R}} -> + ?assertEqual(Payload, P), + ?assertEqual(Topic, T), + ?assert(R), + ok + after 500 -> + emqtt:stop(C1), + ct:fail("should have preserved retained messages") + end, + emqtt:stop(C1), + + ok. + %%-------------------------------------------------------------------- %% HTTP Request %%-------------------------------------------------------------------- diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl index 8f1a75e30..bddad5fb3 100644 --- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_index_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_index_SUITE.erl index 793df0410..e044f2f5f 100644 --- a/apps/emqx_retainer/test/emqx_retainer_index_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_index_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl index b34650dc4..bfe5b6a00 100644 --- a/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index d15db24be..c69a24244 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 8971159e7..c4a6e2e73 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 959c8876d..23c2aab50 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_rule_api_schema). -behaviour(hocon_schema). diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl index 83056754b..a41beb20d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_date.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_date.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index edca35839..ea0bf6f9e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 85fcc63a6..f15290547 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -268,7 +268,7 @@ load_hooks_for_rule(#{from := Topics}) -> maybe_add_metrics_for_rule(Id) -> case emqx_metrics_worker:has_metrics(rule_metrics, Id) of true -> - ok; + ok = reset_metrics_for_rule(Id); false -> ok = emqx_metrics_worker:create_metrics(rule_metrics, Id, ?METRICS, ?RATE_METRICS) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b91fcff58..95c028a1e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl index 802154339..d7ea6c6d1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index 753b5a956..7f4e06252 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index d299a6bb4..cbe7dae82 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl index 4818727b4..8b4b715e9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index b8e227a90..0c962f1fa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index d41e9b56e..b8bfeb84c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 5d887b68b..3e0ebc72d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 29f2c6bf6..ed6cd22de 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl index 1435d2e60..9b6ed7eae 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 4de63e94f..ff966b912 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/proto/emqx_rule_engine_proto_v1.erl b/apps/emqx_rule_engine/src/proto/emqx_rule_engine_proto_v1.erl index ea7e4a53b..dddda6dc3 100644 --- a/apps/emqx_rule_engine/src/proto/emqx_rule_engine_proto_v1.erl +++ b/apps/emqx_rule_engine/src/proto/emqx_rule_engine_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 8f68ae576..c986cd365 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index b15a42bc4..82a305009 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_rule_engine_api_SUITE). -compile(nowarn_export_all). @@ -173,6 +189,38 @@ t_list_rule_api(_Config) -> ?assertEqual(maps:get(data, Result1), maps:get(data, Result6)), ok. +t_reset_metrics_on_disable(_Config) -> + Params = #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => atom_to_binary(?FUNCTION_NAME) + }, + {201, #{id := RuleId}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + + %% generate some fake metrics + emqx_metrics_worker:inc(rule_metrics, RuleId, 'matched', 10), + emqx_metrics_worker:inc(rule_metrics, RuleId, 'passed', 10), + {200, #{metrics := Metrics0}} = emqx_rule_engine_api:'/rules/:id/metrics'( + get, + #{bindings => #{id => RuleId}} + ), + ?assertMatch(#{passed := 10, matched := 10}, Metrics0), + + %% disable the rule; metrics should be reset + {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ + bindings => #{id => RuleId}, + body => Params#{<<"enable">> := false} + }), + + {200, #{metrics := Metrics1}} = emqx_rule_engine_api:'/rules/:id/metrics'( + get, + #{bindings => #{id => RuleId}} + ), + ?assertMatch(#{passed := 0, matched := 0}, Metrics1), + ok. + test_rule_params() -> #{ body => #{ diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl index 4a1c9d6f5..f8fe49ca8 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_rule_events_SUITE). -compile(export_all). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 504211440..5d78f5e4a 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl index 965173b08..9fdd60c56 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/prop_rule_maps.erl b/apps/emqx_rule_engine/test/prop_rule_maps.erl index 15eb4ab6d..718a26cd2 100644 --- a/apps/emqx_rule_engine/test/prop_rule_maps.erl +++ b/apps/emqx_rule_engine/test/prop_rule_maps.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(prop_rule_maps). -include_lib("proper/include/proper.hrl"). diff --git a/apps/emqx_slow_subs/include/emqx_slow_subs.hrl b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl index 4c6b1cc8c..b965c7d7f 100644 --- a/apps/emqx_slow_subs/include/emqx_slow_subs.hrl +++ b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl index 9a82abeb9..efaf46f5b 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl index c24d043e6..8ebdd50c3 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl index f1de12088..3d91b0fb2 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 51c86d6a0..8ae015ae4 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_slow_subs_schema). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl index 5c94f594b..cd47b4388 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/src/proto/emqx_slow_subs_proto_v1.erl b/apps/emqx_slow_subs/src/proto/emqx_slow_subs_proto_v1.erl index 2e6fc7044..ea3089f52 100644 --- a/apps/emqx_slow_subs/src/proto/emqx_slow_subs_proto_v1.erl +++ b/apps/emqx_slow_subs/src/proto/emqx_slow_subs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl index f7ef78ed8..d93a0b4a2 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 3f5f63f29..5b5ed063f 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_statsd/LICENSE b/apps/emqx_statsd/LICENSE deleted file mode 100644 index 818096217..000000000 --- a/apps/emqx_statsd/LICENSE +++ /dev/null @@ -1,190 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/apps/emqx_statsd/include/emqx_statsd.hrl b/apps/emqx_statsd/include/emqx_statsd.hrl index 92d856670..557ebc684 100644 --- a/apps/emqx_statsd/include/emqx_statsd.hrl +++ b/apps/emqx_statsd/include/emqx_statsd.hrl @@ -1,2 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -define(APP, emqx_statsd). -define(STATSD, [statsd]). +-define(SERVER_PARSE_OPTS, #{default_port => 8125}). diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 5f32567d6..638c5a33b 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index 4b0a98cd3..defaf78e0 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -75,10 +75,11 @@ init([]) -> process_flag(trap_exit, true), #{ tags := TagsRaw, - server := {Host, Port}, + server := Server, sample_time_interval := SampleTimeInterval, flush_time_interval := FlushTimeInterval } = emqx_conf:get([statsd]), + {Host, Port} = emqx_schema:parse_server(Server, ?SERVER_PARSE_OPTS), Tags = maps:fold(fun(K, V, Acc) -> [{to_bin(K), to_bin(V)} | Acc] end, [], TagsRaw), Opts = [{tags, Tags}, {host, Host}, {port, Port}, {prefix, <<"emqx">>}], {ok, Pid} = estatsd:start_link(Opts), diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 6007a3327..b1b3601aa 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_statsd/src/emqx_statsd_app.erl b/apps/emqx_statsd/src/emqx_statsd_app.erl index b885772e0..0090c8465 100644 --- a/apps/emqx_statsd/src/emqx_statsd_app.erl +++ b/apps/emqx_statsd/src/emqx_statsd_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_statsd/src/emqx_statsd_config.erl b/apps/emqx_statsd/src/emqx_statsd_config.erl index 4ec71ed32..b818d2691 100644 --- a/apps/emqx_statsd/src/emqx_statsd_config.erl +++ b/apps/emqx_statsd/src/emqx_statsd_config.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index 3fb51f3bd..1e5aa6e5f 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). +-include("emqx_statsd.hrl"). -behaviour(hocon_schema). @@ -40,11 +41,10 @@ fields("statsd") -> boolean(), #{ default => false, - required => true, desc => ?DESC(enable) } )}, - {server, fun server/1}, + {server, server()}, {sample_time_interval, fun sample_interval/1}, {flush_time_interval, fun flush_interval/1}, {tags, fun tags/1} @@ -53,26 +53,24 @@ fields("statsd") -> desc("statsd") -> ?DESC(statsd); desc(_) -> undefined. -server(type) -> emqx_schema:host_port(); -server(required) -> true; -server(default) -> "127.0.0.1:8125"; -server(desc) -> ?DESC(?FUNCTION_NAME); -server(_) -> undefined. +server() -> + Meta = #{ + default => <<"127.0.0.1:8125">>, + desc => ?DESC(?FUNCTION_NAME) + }, + emqx_schema:servers_sc(Meta, ?SERVER_PARSE_OPTS). sample_interval(type) -> emqx_schema:duration_ms(); -sample_interval(required) -> true; sample_interval(default) -> "30s"; sample_interval(desc) -> ?DESC(?FUNCTION_NAME); sample_interval(_) -> undefined. flush_interval(type) -> emqx_schema:duration_ms(); -flush_interval(required) -> true; flush_interval(default) -> "30s"; flush_interval(desc) -> ?DESC(?FUNCTION_NAME); flush_interval(_) -> undefined. tags(type) -> map(); -tags(required) -> false; tags(default) -> #{}; tags(desc) -> ?DESC(?FUNCTION_NAME); tags(_) -> undefined. diff --git a/apps/emqx_statsd/src/emqx_statsd_sup.erl b/apps/emqx_statsd/src/emqx_statsd_sup.erl index f14242113..2845fb505 100644 --- a/apps/emqx_statsd/src/emqx_statsd_sup.erl +++ b/apps/emqx_statsd/src/emqx_statsd_sup.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- %%%------------------------------------------------------------------- %% @doc emqx_statsd top level supervisor. %% @end diff --git a/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl b/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl index 2e01f7d5e..7b696038b 100644 --- a/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl +++ b/apps/emqx_statsd/src/proto/emqx_statsd_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index 2b5074f48..2f8fa5a69 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_statsd_SUITE). -compile(export_all). diff --git a/bin/emqx b/bin/emqx index e2e49a62e..c5bf88149 100755 --- a/bin/emqx +++ b/bin/emqx @@ -5,7 +5,13 @@ set -euo pipefail DEBUG="${DEBUG:-0}" -[ "$DEBUG" -eq 1 ] && set -x +if [ "$DEBUG" -eq 1 ]; then + set -x +fi +if [ "$DEBUG" -eq 2 ]; then + set -x + export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' +fi # We need to find real directory with emqx files on all platforms # even when bin/emqx is symlinked on several levels @@ -36,6 +42,7 @@ export RUNNER_ROOT_DIR export EMQX_ETC_DIR export REL_VSN export SCHEMA_MOD +export IS_ENTERPRISE RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" @@ -540,6 +547,121 @@ check_license() { fi } +# When deciding which install upgrade script to run, we have to check +# our own version so we may avoid infinite loops and call the correct +# version. +current_script_version() { + curr_script=$(basename "${BASH_SOURCE[0]}") + suffix=${curr_script#*-} + if [[ "${suffix}" == "${curr_script}" ]]; then + # there's no suffix, so we're running the default `emqx` script; + # we'll have to trust the REL_VSN variable + echo "$REL_VSN" + else + echo "${suffix}" + fi +} + +parse_semver() { + echo "$1" | tr '.|-' ' ' +} + +max_version_of() { + local vsn1="$1" + local vsn2="$2" + + echo "${vsn1}" "${vsn2}" | tr " " "\n" | sort -rV | head -n1 +} + +versioned_script_path() { + local script_name="$1" + local vsn="$2" + + echo "$RUNNER_ROOT_DIR/bin/$script_name-$vsn" +} + +does_script_version_exist() { + local script_name="$1" + local vsn="$2" + + if [[ -f "$(versioned_script_path "$script_name" "$vsn")" ]]; then + return 0 + else + return 1 + fi +} + +# extract_from_package packege_path destination file1 file2 +extract_from_package() { + local package="$1" + local dest_dir="$2" + shift 2 + + tar -C "$dest_dir" -xf "$package" "$@" +} + +am_i_the_newest_script() { + local curr_vsn other_vsn + curr_vsn="$(current_script_version)" + other_vsn="$1" + max_vsn="$(max_version_of "$other_vsn" "$curr_vsn")" + + if [[ "$max_vsn" == "$curr_vsn" ]]; then + return 0 + else + return 1 + fi +} + +locate_package() { + local package_path candidates vsn + vsn="$1" + + if [[ "${IS_ENTERPRISE}" == "yes" ]]; then + package_pattern="$RUNNER_ROOT_DIR/releases/emqx-enterprise-$vsn-*.tar.gz" + else + package_pattern="$RUNNER_ROOT_DIR/releases/emqx-$vsn-*.tar.gz" + fi + + # shellcheck disable=SC2207,SC2086 + candidates=($(ls $package_pattern)) + + if [[ "${#candidates[@]}" == 0 ]]; then + logerr "No package matching $package_pattern found." + exit 1 + elif [[ "${#candidates[@]}" -gt 1 ]]; then + logerr "Multiple packages matching $package_pattern found. Ensure only one exists." + exit 1 + else + echo "${candidates[0]}" + fi +} + +ensure_newest_script_is_extracted() { + local newest_vsn="$1" + local package_path tmpdir + + if does_script_version_exist "emqx" "$newest_vsn" \ + && does_script_version_exist "install_upgrade.escript" "$newest_vsn"; then + return + else + package_path="$(locate_package "$newest_vsn")" + tmpdir="$(mktemp -dp /tmp emqx.XXXXXXXXXXX)" + + extract_from_package \ + "$package_path" \ + "$tmpdir" \ + "bin/emqx-$newest_vsn" \ + "bin/install_upgrade.escript-$newest_vsn" + + cp "$tmpdir/bin/emqx-$newest_vsn" \ + "$tmpdir/bin/install_upgrade.escript-$newest_vsn" \ + "$RUNNER_ROOT_DIR/bin/" + + rm -rf "$tmpdir" + fi +} + # Run an escript in the node's environment relx_escript() { shift; scriptpath="$1"; shift @@ -922,8 +1044,21 @@ case "${COMMAND}" in assert_node_alive + curr_vsn="$(current_script_version)" + target_vsn="$1" + newest_vsn="$(max_version_of "$target_vsn" "$curr_vsn")" + ensure_newest_script_is_extracted "$newest_vsn" + # if we are not the newest script, run the same command from it + if ! am_i_the_newest_script "$newest_vsn"; then + script_path="$(versioned_script_path emqx "$newest_vsn")" + exec "$script_path" "$COMMAND" "$@" + fi + + upgrade_script_path="$(versioned_script_path install_upgrade.escript "$newest_vsn")" + echo "using ${upgrade_script_path} to run ${COMMAND} $*" + ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARGS" \ - exec "$BINDIR/escript" "$RUNNER_ROOT_DIR/bin/install_upgrade.escript" \ + exec "$BINDIR/escript" "$upgrade_script_path" \ "$COMMAND" "{'$REL_NAME', \"$NAME_TYPE\", '$NAME', '$COOKIE'}" "$@" ;; diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 4ab4947c0..f7f340f31 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -18,22 +18,31 @@ main([Command0, DistInfoStr | CommandArgs]) -> Opts = parse_arguments(CommandArgs), %% invoke the command passed as argument F = case Command0 of - "install" -> fun(A, B) -> install(A, B) end; - "unpack" -> fun(A, B) -> unpack(A, B) end; - "upgrade" -> fun(A, B) -> upgrade(A, B) end; - "downgrade" -> fun(A, B) -> downgrade(A, B) end; - "uninstall" -> fun(A, B) -> uninstall(A, B) end; - "versions" -> fun(A, B) -> versions(A, B) end + %% "install" -> fun(A, B) -> install(A, B) end; + %% "unpack" -> fun(A, B) -> unpack(A, B) end; + %% "upgrade" -> fun(A, B) -> upgrade(A, B) end; + %% "downgrade" -> fun(A, B) -> downgrade(A, B) end; + %% "uninstall" -> fun(A, B) -> uninstall(A, B) end; + "versions" -> fun(A, B) -> versions(A, B) end; + _ -> fun fail_upgrade/2 end, F(DistInfo, Opts); main(Args) -> ?INFO("unknown args: ~p", [Args]), erlang:halt(1). +%% temporary block for hot-upgrades; next release will just remove +%% this and the new script version shall be used instead of this +%% current version. +%% TODO: always deny relup for macos (unsupported) +fail_upgrade(_DistInfo, _Opts) -> + ?ERROR("Unsupported upgrade path", []), + erlang:halt(1). + unpack({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), Version = proplists:get_value(version, Opts), - case unpack_release(RelName, TargetNode, Version) of + case unpack_release(RelName, TargetNode, Version, Opts) of {ok, Vsn} -> ?INFO("Unpacked successfully: ~p", [Vsn]); old -> @@ -57,7 +66,7 @@ install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), Version = proplists:get_value(version, Opts), validate_target_version(Version, TargetNode), - case unpack_release(RelName, TargetNode, Version) of + case unpack_release(RelName, TargetNode, Version, Opts) of {ok, Vsn} -> ?INFO("Unpacked successfully: ~p.", [Vsn]), check_and_install(TargetNode, Vsn), @@ -132,12 +141,13 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) -> uninstall(_, Args) -> ?INFO("uninstall: unknown args ~p", [Args]). -versions({_RelName, NameTypeArg, NodeName, Cookie}, []) -> +versions({_RelName, NameTypeArg, NodeName, Cookie}, _Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), print_existing_versions(TargetNode). parse_arguments(Args) -> - parse_arguments(Args, []). + IsEnterprise = os:getenv("IS_ENTERPRISE") == "yes", + parse_arguments(Args, [{is_enterprise, IsEnterprise}]). parse_arguments([], Acc) -> Acc; parse_arguments(["--no-permanent"|Rest], Acc) -> @@ -146,9 +156,10 @@ parse_arguments([VersionStr|Rest], Acc) -> Version = parse_version(VersionStr), parse_arguments(Rest, [{version, Version}] ++ Acc). -unpack_release(RelName, TargetNode, Version) -> - StartScriptExists = filelib:is_dir(filename:join(["releases", Version, "start.boot"])), +unpack_release(RelName, TargetNode, Version, Opts) -> + StartScriptExists = filelib:is_regular(filename:join(["releases", Version, "start.boot"])), WhichReleases = which_releases(TargetNode), + IsEnterprise = proplists:get_value(is_enterprise, Opts), case proplists:get_value(Version, WhichReleases) of Res when Res =:= undefined; (Res =:= unpacked andalso not StartScriptExists) -> %% not installed, so unpack tarball: @@ -156,7 +167,7 @@ unpack_release(RelName, TargetNode, Version) -> %% releases/-.tar.gz %% releases//-.tar.gz %% releases//.tar.gz - case find_and_link_release_package(Version, RelName) of + case find_and_link_release_package(Version, RelName, IsEnterprise) of {_, undefined} -> {error, release_package_not_found}; {ReleasePackage, ReleasePackageLink} -> @@ -206,7 +217,7 @@ extract_tar(Cwd, Tar) -> %% to the release package tarball found in 1. %% 3. return a tuple with the paths to the release package and %% to the symlink that is to be provided to release handler -find_and_link_release_package(Version, RelName) -> +find_and_link_release_package(Version, RelName, IsEnterprise) -> RelNameStr = atom_to_list(RelName), %% regardless of the location of the release package, we'll %% always give release handler the same path which is the symlink @@ -217,7 +228,13 @@ find_and_link_release_package(Version, RelName) -> %% we've found where the actual release package is located ReleaseLink = filename:join(["releases", Version, RelNameStr ++ ".tar.gz"]), - TarBalls = filename:join(["releases", RelNameStr ++ "-*" ++ Version ++ "*.tar.gz"]), + ReleaseNamePattern = + case IsEnterprise of + false -> RelNameStr; + true -> RelNameStr ++ "-enterprise" + end, + FilePattern = lists:flatten([ReleaseNamePattern, "-", Version, "*.tar.gz"]), + TarBalls = filename:join(["releases", FilePattern]), case filelib:wildcard(TarBalls) of [] -> {undefined, undefined}; diff --git a/build b/build index 05a3a33d3..87c719a53 100755 --- a/build +++ b/build @@ -135,6 +135,9 @@ assert_no_compile_time_only_deps() { make_rel() { ./scripts/pre-compile.sh "$PROFILE" + # make_elixir_rel always create rebar.lock + # delete it to make git clone + checkout work because we use shallow close for rebar deps + rm -f rebar.lock # compile all beams ./rebar3 as "$PROFILE" compile # generate docs (require beam compiled), generated to etc and priv dirs diff --git a/changes/refactor-9653.en.md b/changes/refactor-9653.en.md new file mode 100644 index 000000000..2807f81d5 --- /dev/null +++ b/changes/refactor-9653.en.md @@ -0,0 +1 @@ +Make authorization config validation error message more readable. diff --git a/changes/refactor-9653.zh.md b/changes/refactor-9653.zh.md new file mode 100644 index 000000000..755fd1683 --- /dev/null +++ b/changes/refactor-9653.zh.md @@ -0,0 +1 @@ +改进授权配置检查错误日志的可读性。 diff --git a/changes/v5.0.14/feat-8329.en.md b/changes/v5.0.14/feat-8329.en.md new file mode 100644 index 000000000..2876e1754 --- /dev/null +++ b/changes/v5.0.14/feat-8329.en.md @@ -0,0 +1 @@ +The MongoDB library has been upgraded to support MongoDB 5.1+ diff --git a/changes/v5.0.14/feat-8329.zh.md b/changes/v5.0.14/feat-8329.zh.md new file mode 100644 index 000000000..31e73086d --- /dev/null +++ b/changes/v5.0.14/feat-8329.zh.md @@ -0,0 +1 @@ +MongoDB 的驱动现在已经升级到 MongoDB 5.1+ 了。 diff --git a/changes/v5.0.14/feat-9593.en.md b/changes/v5.0.14/feat-9593.en.md new file mode 100644 index 000000000..d6db43a79 --- /dev/null +++ b/changes/v5.0.14/feat-9593.en.md @@ -0,0 +1 @@ +Obfuscated sensitive data in the response when querying `bridges` information by API. diff --git a/changes/v5.0.14/feat-9593.zh.md b/changes/v5.0.14/feat-9593.zh.md new file mode 100644 index 000000000..fd3133050 --- /dev/null +++ b/changes/v5.0.14/feat-9593.zh.md @@ -0,0 +1 @@ +通过 API 查询 `bridges` 信息时将混淆响应中的敏感数据。 diff --git a/changes/v5.0.14/feat-9614.en.md b/changes/v5.0.14/feat-9614.en.md new file mode 100644 index 000000000..2ccf62c76 --- /dev/null +++ b/changes/v5.0.14/feat-9614.en.md @@ -0,0 +1,4 @@ +Make possible to configure `host:port` from environment variables without quotes. +Prior to this change, when overriding a `host:port` config value from environment variable, one has to quote it as: +`env EMQX_BRIDGES__MQTT__XYZ__SERVER='"localhost:1883"'`. +Now it's possible to set it without quote as `env EMQX_BRIDGES__MQTT__XYZ__SERVER='localhost:1883'`. diff --git a/changes/v5.0.14/feat-9614.zh.md b/changes/v5.0.14/feat-9614.zh.md new file mode 100644 index 000000000..7f67d92af --- /dev/null +++ b/changes/v5.0.14/feat-9614.zh.md @@ -0,0 +1,4 @@ +允许环境变量重载 `host:port` 值时不使用引号。 +在此修复前，环境变量中使用 `host:port` 这种配置时，用户必须使用引号，例如： +`env EMQX_BRIDGES__MQTT__XYZ__SERVER='"localhost:1883"'`。 +此修复后，可以不使用引号，例如 `env EMQX_BRIDGES__MQTT__XYZ__SERVER='localhost:1883'`。 diff --git a/changes/v5.0.14/feat-9642.en.md b/changes/v5.0.14/feat-9642.en.md new file mode 100644 index 000000000..19de4d946 --- /dev/null +++ b/changes/v5.0.14/feat-9642.en.md @@ -0,0 +1 @@ +Deprecates `enable_batch` and `enable_queue` options for bridges/resources. After this change, queuing is always enabled for bridges, and batching is controlled by the `batch_size` option: `batch_size > 1` means batching will be enabled. diff --git a/changes/v5.0.14/feat-9642.zh.md b/changes/v5.0.14/feat-9642.zh.md new file mode 100644 index 000000000..e394abdbb --- /dev/null +++ b/changes/v5.0.14/feat-9642.zh.md @@ -0,0 +1 @@ +废弃了桥接的 `enable_batch` 和 `enable_queue` 配置项 。在这一改变之后，桥接的工作进程总是启用缓存队列，而批处理由 `batch_size` 选项控制：`batch_size > 1` 则意味着启用批处理。 diff --git a/changes/v5.0.14/feat-9671.en.md b/changes/v5.0.14/feat-9671.en.md new file mode 100644 index 000000000..dd5ed5e4d --- /dev/null +++ b/changes/v5.0.14/feat-9671.en.md @@ -0,0 +1 @@ +Implement sliding window average metrics. diff --git a/changes/v5.0.14/feat-9671.zh.md b/changes/v5.0.14/feat-9671.zh.md new file mode 100644 index 000000000..cbcae01fe --- /dev/null +++ b/changes/v5.0.14/feat-9671.zh.md @@ -0,0 +1 @@ +实施滑动窗口平均度量。 diff --git a/changes/v5.0.14/feat-9674.en.md b/changes/v5.0.14/feat-9674.en.md new file mode 100644 index 000000000..a87725b43 --- /dev/null +++ b/changes/v5.0.14/feat-9674.en.md @@ -0,0 +1 @@ +Made rule engine behavior more consistent with bridge behavior regarding metrics: if a rule engine is disabled, its metrics are now reset diff --git a/changes/v5.0.14/feat-9674.zh.md b/changes/v5.0.14/feat-9674.zh.md new file mode 100644 index 000000000..c9d776525 --- /dev/null +++ b/changes/v5.0.14/feat-9674.zh.md @@ -0,0 +1 @@ +使得规则引擎的行为与桥梁的指标行为更加一致：如果一个规则引擎被禁用，其指标现在会被重置。 diff --git a/changes/v5.0.14/feat-9675.en.md b/changes/v5.0.14/feat-9675.en.md new file mode 100644 index 000000000..5249e9826 --- /dev/null +++ b/changes/v5.0.14/feat-9675.en.md @@ -0,0 +1,2 @@ +HTTP client library `ehttpc` upgraded from `0.4.2` to `0.4.3`. +Library `eredis_cluster` which manages clients to redis clusters upgraded from `0.7.1` to `0.7.5`. diff --git a/changes/v5.0.14/feat-9675.zh.md b/changes/v5.0.14/feat-9675.zh.md new file mode 100644 index 000000000..d14f260ae --- /dev/null +++ b/changes/v5.0.14/feat-9675.zh.md @@ -0,0 +1,2 @@ +HTTP 客户端库 `ehttpc` 从 `0.4.2` 升级到 `0.4.3` +Redis cluster 客户端库 `eredis_cluster` 从 `0.7.1` 升级到 `0.7.5`. diff --git a/changes/v5.0.14/feat-9713.en.md b/changes/v5.0.14/feat-9713.en.md new file mode 100644 index 000000000..e8dbe4c6c --- /dev/null +++ b/changes/v5.0.14/feat-9713.en.md @@ -0,0 +1,3 @@ +Introduce `api_key.bootstrap_file` to initialize the api key at boot time. +Deprecate `dashboard.bootstrap_users_file`. +Limit the maximum number of api keys to 100 instead of 30. diff --git a/changes/v5.0.14/feat-9713.zh.md b/changes/v5.0.14/feat-9713.zh.md new file mode 100644 index 000000000..7535b8bd5 --- /dev/null +++ b/changes/v5.0.14/feat-9713.zh.md @@ -0,0 +1,3 @@ +引入 `api_key.bootstrap_file`，用于启动时初始化api密钥。 +废弃 `dashboard.boostrap_users_file`。 +将 API 密钥的最大数量限制提升为 100（原来为30）。 diff --git a/changes/v5.0.14/fix-8648.en.md b/changes/v5.0.14/fix-8648.en.md new file mode 100644 index 000000000..ac608a2e1 --- /dev/null +++ b/changes/v5.0.14/fix-8648.en.md @@ -0,0 +1 @@ +When deleting a non-existing bridge the server gave a success response. This has been fixed so that the server instead gives an error response when the user attempts to delete a non-existing bridge. diff --git a/changes/v5.0.14/fix-8648.zh.md b/changes/v5.0.14/fix-8648.zh.md new file mode 100644 index 000000000..6512c5aa5 --- /dev/null +++ b/changes/v5.0.14/fix-8648.zh.md @@ -0,0 +1 @@ +修复了当通过 API 删除一个不存在的桥接时，服务器会返回操作成功的问题，现在将会返回操作失败的信息。 diff --git a/changes/v5.0.14/fix-9637.en.md b/changes/v5.0.14/fix-9637.en.md new file mode 100644 index 000000000..d93ed493c --- /dev/null +++ b/changes/v5.0.14/fix-9637.en.md @@ -0,0 +1 @@ +Fix the expiry_interval fields of the clients HTTP API to measure in seconds. diff --git a/changes/v5.0.14/fix-9637.zh.md b/changes/v5.0.14/fix-9637.zh.md new file mode 100644 index 000000000..8164a0bbf --- /dev/null +++ b/changes/v5.0.14/fix-9637.zh.md @@ -0,0 +1 @@ +修复 clients HTTP API 下的 expiry_interval 字段的时间单位为秒。 diff --git a/changes/v5.0.14/fix-9638.en.md b/changes/v5.0.14/fix-9638.en.md new file mode 100644 index 000000000..7a9265fb4 --- /dev/null +++ b/changes/v5.0.14/fix-9638.en.md @@ -0,0 +1 @@ +Fix the problem of data loss and bad match when the MySQL driver is disconnected. diff --git a/changes/v5.0.14/fix-9638.zh.md b/changes/v5.0.14/fix-9638.zh.md new file mode 100644 index 000000000..6ae2a3c1e --- /dev/null +++ b/changes/v5.0.14/fix-9638.zh.md @@ -0,0 +1 @@ +修复 MySQL 驱动断开连接时出现的数据丢失和匹配错误的问题。 diff --git a/changes/v5.0.14/fix-9641.en.md b/changes/v5.0.14/fix-9641.en.md new file mode 100644 index 000000000..29924de62 --- /dev/null +++ b/changes/v5.0.14/fix-9641.en.md @@ -0,0 +1 @@ +Fix an issue where testing the GCP PubSub could leak memory, and an issue where its JWT token would fail to refresh a second time. diff --git a/changes/v5.0.14/fix-9641.zh.md b/changes/v5.0.14/fix-9641.zh.md new file mode 100644 index 000000000..7b9e6d55c --- /dev/null +++ b/changes/v5.0.14/fix-9641.zh.md @@ -0,0 +1 @@ +修复了测试GCP PubSub可能泄露内存的问题，以及其JWT令牌第二次刷新失败的问题。 diff --git a/changes/v5.0.14/fix-9642.en.md b/changes/v5.0.14/fix-9642.en.md new file mode 100644 index 000000000..12906299e --- /dev/null +++ b/changes/v5.0.14/fix-9642.en.md @@ -0,0 +1,3 @@ +Fix some issues that could lead to wrong bridge metrics. +Fix and issue that could lead to message loss and wrong metrics with Kafka Producer bridge when Kafka or the connection to it is down. +Fix some issues that could lead to the same message being delivered more than once when using batching for bridges and when the batch was retried. diff --git a/changes/v5.0.14/fix-9642.zh.md b/changes/v5.0.14/fix-9642.zh.md new file mode 100644 index 000000000..2565c422c --- /dev/null +++ b/changes/v5.0.14/fix-9642.zh.md @@ -0,0 +1,3 @@ +修复一些可能导致错误桥接指标的问题。 +修复当Kafka或其连接中断时，可能导致Kafka Producer桥的消息丢失和错误指标的问题。 +修复一些问题，这些问题可能导致在为桥接使用批处理时，同一消息被多次传递，以及批处理被重试时。 diff --git a/changes/v5.0.14/fix-9667.en.md b/changes/v5.0.14/fix-9667.en.md new file mode 100644 index 000000000..4b0fe7aef --- /dev/null +++ b/changes/v5.0.14/fix-9667.en.md @@ -0,0 +1 @@ +Remove possibility to set `clientid` for `/publish` and `/publish/bulk` HTTP APIs. This is to reduce the risk for security confusion. diff --git a/changes/v5.0.14/fix-9667.zh.md b/changes/v5.0.14/fix-9667.zh.md new file mode 100644 index 000000000..f3952ca14 --- /dev/null +++ b/changes/v5.0.14/fix-9667.zh.md @@ -0,0 +1 @@ +从 HTTP API /publish 和 /publish/bulk 中移除 clientid, 降低安全风险 diff --git a/changes/v5.0.14/fix-9687.en.md b/changes/v5.0.14/fix-9687.en.md new file mode 100644 index 000000000..6c2f53bb3 --- /dev/null +++ b/changes/v5.0.14/fix-9687.en.md @@ -0,0 +1,2 @@ +Fix the problem that sending messages to data-bridges failed because of incorrect handling of some data-bridges without `local_topic` field configured. +Before this change, if some bridges have configured the `local_topic` field but others have not, a `function_clause` error will occur when forwarding messages to the data-bridges. diff --git a/changes/v5.0.14/fix-9687.zh.md b/changes/v5.0.14/fix-9687.zh.md new file mode 100644 index 000000000..089db5986 --- /dev/null +++ b/changes/v5.0.14/fix-9687.zh.md @@ -0,0 +1,2 @@ +修复由于某些数据桥接未配置 `local_topic` 字段，导致的所有数据桥接无法发送消息。 +在此改动之前，如果有些桥接设置了 `local_topic` 字段而有些没有设置，数据桥接转发消息时会出现 `function_clause` 的错误。 diff --git a/changes/v5.0.14/fix-9689.en.md b/changes/v5.0.14/fix-9689.en.md new file mode 100644 index 000000000..7582c8bc5 --- /dev/null +++ b/changes/v5.0.14/fix-9689.en.md @@ -0,0 +1 @@ +Fix handling of HTTP authorization result when a request failure (e.g.: HTTP resource is down) would cause a `function_clause` error. diff --git a/changes/v5.0.14/fix-9689.zh.md b/changes/v5.0.14/fix-9689.zh.md new file mode 100644 index 000000000..62f4a90fb --- /dev/null +++ b/changes/v5.0.14/fix-9689.zh.md @@ -0,0 +1 @@ +修正当请求失败（如：HTTP资源关闭）会导致`function_clause`错误时对HTTP授权结果的处理。 diff --git a/changes/v5.0.14/fix-9703.en.md b/changes/v5.0.14/fix-9703.en.md new file mode 100644 index 000000000..4eb91c7d0 --- /dev/null +++ b/changes/v5.0.14/fix-9703.en.md @@ -0,0 +1,3 @@ +Set the default value of the `qos` field of the HTTP API `/clients/:clientid/subscribe` to 0. +Before this fix, the `qos` field have no default value, which leads to a `function_clause` error +when querying this API. diff --git a/changes/v5.0.14/fix-9703.zh.md b/changes/v5.0.14/fix-9703.zh.md new file mode 100644 index 000000000..863304a66 --- /dev/null +++ b/changes/v5.0.14/fix-9703.zh.md @@ -0,0 +1,2 @@ +将 HTTP 接口 `/clients/:clientid/subscribe` 的 `qos` 字段的默认值设置为 0。 +在此修复之前，`qos` 字段没有默认值，调用订阅接口的时候将导致 `function_clause` 错误。 diff --git a/changes/v5.0.14/fix-9705.en.md b/changes/v5.0.14/fix-9705.en.md new file mode 100644 index 000000000..479d3d4ea --- /dev/null +++ b/changes/v5.0.14/fix-9705.en.md @@ -0,0 +1,8 @@ +Remove the default value of Webhook. +Before this repair, the default value of the `body` field of Webhook is `${payload}`, +but there is no `payload` field in the available fields of other events except message +publishing in the rule, so in this case, the webhook will send a string with the +message body as "undefined" to the HTTP service. +This fix removes the default value of the `body` field. When the `body` field is +not configured, Webhook will send all available fields of the current event in +the format of JSON object. diff --git a/changes/v5.0.14/fix-9705.zh.md b/changes/v5.0.14/fix-9705.zh.md new file mode 100644 index 000000000..6a57eba05 --- /dev/null +++ b/changes/v5.0.14/fix-9705.zh.md @@ -0,0 +1,5 @@ +删除 Webhook 的默认值。 +在此修复之前，Webhook 的 `body` 字段的默认值为 `${payload}`，但规则中除了消息发布之外的其他事件的可用字段中 +都没有 `payload` 字段，所以这种情况下 Webhook 将发送消息正文为 "undefined" 的字符串到 HTTP 服务。 +此修复移除了 `body` 字段的默认值，当未配置 `body` 字段的时候，Webhook 将以 JSON object 的格式发送 +当前事件的全部可用字段。 diff --git a/changes/v5.0.14/fix-9712.en.md b/changes/v5.0.14/fix-9712.en.md new file mode 100644 index 000000000..e110b03e2 --- /dev/null +++ b/changes/v5.0.14/fix-9712.en.md @@ -0,0 +1,2 @@ +Fixed the problem of '404 Not Found' when calling the HTTP API '/clients/:clientid/subscribe/bulk' +from the plug-ins and data-bridges on handling the 'client.connected' event. diff --git a/changes/v5.0.14/fix-9712.zh.md b/changes/v5.0.14/fix-9712.zh.md new file mode 100644 index 000000000..053f6b08c --- /dev/null +++ b/changes/v5.0.14/fix-9712.zh.md @@ -0,0 +1,2 @@ +修复了监听 `client.connected` 事件的插件和数据桥接在调用 `/clients/:clientid/subscribe/bulk` +HTTP 接口时报 `404 Not Found` 的问题。 diff --git a/changes/v5.0.14/fix-9714.en.md b/changes/v5.0.14/fix-9714.en.md new file mode 100644 index 000000000..e1a606744 --- /dev/null +++ b/changes/v5.0.14/fix-9714.en.md @@ -0,0 +1 @@ +Fix `/mqtt/auto_subscribe` API's bad swagger schema, and make sure swagger always checks if the schema is correct. diff --git a/changes/v5.0.14/fix-9714.zh.md b/changes/v5.0.14/fix-9714.zh.md new file mode 100644 index 000000000..cbf38f041 --- /dev/null +++ b/changes/v5.0.14/fix-9714.zh.md @@ -0,0 +1 @@ +修复 `/mqtt/auto_subscribe` API 错误的 swagger 格式，并且保证 swagger 总是检查格式是否正确。 diff --git a/changes/v5.0.14/fix-9716.en.md b/changes/v5.0.14/fix-9716.en.md new file mode 100644 index 000000000..93d4f1823 --- /dev/null +++ b/changes/v5.0.14/fix-9716.en.md @@ -0,0 +1 @@ +MQTT bridge config compatibility fix. The config created from before v5.0.12 may encounter a compatibility issue after upgraded to v5.0.13. diff --git a/changes/v5.0.14/fix-9716.zh.md b/changes/v5.0.14/fix-9716.zh.md new file mode 100644 index 000000000..f368fe325 --- /dev/null +++ b/changes/v5.0.14/fix-9716.zh.md @@ -0,0 +1 @@ +修复 v5.0.12 之前的 MQTT 桥接配置在 升级到 v5.0.13 后 HTTP API 查询 桥接配置时的一个兼容性问题。 diff --git a/changes/v5.0.14/fix-9717.en.md b/changes/v5.0.14/fix-9717.en.md new file mode 100644 index 000000000..9a3b29157 --- /dev/null +++ b/changes/v5.0.14/fix-9717.en.md @@ -0,0 +1 @@ +Prior to this fix, if it always times out when trying to connect a bridge server, it's not possible to change other configs even when the bridge is disabled. diff --git a/changes/v5.0.14/fix-9717.zh.md b/changes/v5.0.14/fix-9717.zh.md new file mode 100644 index 000000000..859d7806f --- /dev/null +++ b/changes/v5.0.14/fix-9717.zh.md @@ -0,0 +1 @@ +修复已禁用的桥接资源服务器连接超时的情况下不能修改其他配置参数的问题。 diff --git a/lib-ee/BSL.txt b/lib-ee/BSL.txt index b361a8bd7..0acc0e696 100644 --- a/lib-ee/BSL.txt +++ b/lib-ee/BSL.txt @@ -2,12 +2,12 @@ Business Source License 1.1 Licensor: Hangzhou EMQ Technologies Co., Ltd. Licensed Work: EMQX Enterprise Edition - The Licensed Work is (c) 2021 + The Licensed Work is (c) 2023 Hangzhou EMQ Technologies Co., Ltd. Additional Use Grant: Students and educators are granted right to copy, modify, and create derivative work for research or education. -Change Date: 2026-03-01 +Change Date: 2027-02-01 Change License: Apache License, Version 2.0 For information about alternative licensing arrangements for the Software, diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index fba33559e..bf990bd7c 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -6,3 +6,4 @@ mongo_rs_sharded mysql redis redis_cluster +pgsql diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 1fdbfedc4..7ccf20d0b 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -81,8 +81,8 @@ emqx_ee_bridge_kafka { } bootstrap_hosts { desc { - en: "A comma separated list of Kafka host:port endpoints to bootstrap the client." - zh: "用逗号分隔的 host:port 主机列表。" + en: "A comma separated list of Kafka host[:port] endpoints to bootstrap the client. Default port number is 9092." + zh: "用逗号分隔的 host[:port] 主机列表。默认端口号为 9092。" } label { en: "Bootstrap Hosts" @@ -438,10 +438,12 @@ emqx_ee_bridge_kafka { desc { en: "Applicable when buffer mode is set to memory or hybrid.\n" "EMQX will drop old cached messages under high memory pressure. " - "The high memory threshold is defined in config sysmon.os.sysmem_high_watermark." + "The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. " + "NOTE: This config only works on Linux." zh: "缓存模式是 memory 或 hybrid 时适用。" "当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。" "内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。" + "注意，该配置仅在 Linux 系统中有效。" } label { en: "Memory Overload Protection" diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf index fef3663ef..f8009f0a4 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf @@ -64,4 +64,26 @@ emqx_ee_bridge_mongodb { zh: "MongoDB（独立）配置" } } + + desc_type { + desc { + en: """The Bridge Type.""" + zh: """桥接类型。""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名称。""" + } + label { + en: "Bridge Name" + zh: "桥接名称" + } + } } diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf new file mode 100644 index 000000000..0f80e1a1b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf @@ -0,0 +1,74 @@ +emqx_ee_bridge_pgsql { + + local_topic { + desc { + en: """The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded. +""" + zh: """发送到 'local_topic' 的消息都会转发到 PostgreSQL。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 +""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + + sql_template { + desc { + en: """SQL Template""" + zh: """SQL 模板""" + } + label { + en: "SQL Template" + zh: "SQL 模板" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + + desc_config { + desc { + en: """Configuration for an PostgreSQL bridge.""" + zh: """PostgreSQL 桥接配置""" + } + label: { + en: "PostgreSQL Bridge Configuration" + zh: "PostgreSQL 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名字""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index d551bb1b5..6ca554c72 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,9 +1,8 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} - , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}} - , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}} +{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}} + , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} - , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.4"}}} + , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 343325c5c..1563cb8ef 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index cf20d7110..a709601bb 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge). @@ -17,6 +17,7 @@ api_schemas(Method) -> ref(emqx_ee_bridge_gcp_pubsub, Method), ref(emqx_ee_bridge_kafka, Method), ref(emqx_ee_bridge_mysql, Method), + ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), ref(emqx_ee_bridge_mongodb, Method ++ "_single"), @@ -25,7 +26,9 @@ api_schemas(Method) -> ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2"), ref(emqx_ee_bridge_redis, Method ++ "_single"), ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), - ref(emqx_ee_bridge_redis, Method ++ "_cluster") + ref(emqx_ee_bridge_redis, Method ++ "_cluster"), + ref(emqx_ee_bridge_timescale, Method), + ref(emqx_ee_bridge_matrix, Method) ]. schema_modules() -> @@ -36,7 +39,10 @@ schema_modules() -> emqx_ee_bridge_influxdb, emqx_ee_bridge_mongodb, emqx_ee_bridge_mysql, - emqx_ee_bridge_redis + emqx_ee_bridge_redis, + emqx_ee_bridge_pgsql, + emqx_ee_bridge_timescale, + emqx_ee_bridge_matrix ]. examples(Method) -> @@ -63,7 +69,10 @@ resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb; resource_type(redis_single) -> emqx_ee_connector_redis; resource_type(redis_sentinel) -> emqx_ee_connector_redis; -resource_type(redis_cluster) -> emqx_ee_connector_redis. +resource_type(redis_cluster) -> emqx_ee_connector_redis; +resource_type(pgsql) -> emqx_connector_pgsql; +resource_type(timescale) -> emqx_connector_pgsql; +resource_type(matrix) -> emqx_connector_pgsql. fields(bridges) -> [ @@ -99,7 +108,7 @@ fields(bridges) -> required => false } )} - ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs(). + ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs(). mongodb_structs() -> [ @@ -146,3 +155,20 @@ redis_structs() -> redis_cluster ] ]. + +pgsql_structs() -> + [ + {Type, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_pgsql, "config")), + #{ + desc => <>, + required => false + } + )} + || {Type, Name} <- [ + {pgsql, <<"PostgreSQL">>}, + {timescale, <<"Timescale">>}, + {matrix, <<"Matrix">>} + ] + ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl index 63566bed9..760aba9e1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_gcp_pubsub). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl index dfae764c8..135087929 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_hstreamdb). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index a2f125722..62c8b6ab7 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_influxdb). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 2540b987c..3f2f6a85f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_kafka). @@ -26,7 +26,8 @@ namespace/0, roots/0, fields/1, - desc/1 + desc/1, + host_opts/0 ]). %% ------------------------------------------------------------------------------------------------- @@ -54,6 +55,9 @@ values(put) -> %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions +host_opts() -> + #{default_port => 9092}. + namespace() -> "bridge_kafka". roots() -> ["config"]. @@ -67,7 +71,17 @@ fields("get") -> fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {bootstrap_hosts, mk(binary(), #{required => true, desc => ?DESC(bootstrap_hosts)})}, + {bootstrap_hosts, + mk( + binary(), + #{ + required => true, + desc => ?DESC(bootstrap_hosts), + validator => emqx_schema:servers_validator( + host_opts(), _Required = true + ) + } + )}, {connect_timeout, mk(emqx_schema:duration_ms(), #{ default => "5s", diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl new file mode 100644 index 000000000..106fac48a --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_matrix.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_matrix). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"matrix">> => #{ + summary => <<"Matrix Bridge">>, + value => emqx_ee_bridge_pgsql:values(Method, matrix) + } + } + ]. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_matrix". + +roots() -> []. + +fields("post") -> + emqx_ee_bridge_pgsql:fields("post", matrix); +fields(Method) -> + emqx_ee_bridge_pgsql:fields(Method). + +desc(_) -> + undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index 9d9a5e4d0..516c75f65 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_mongodb). @@ -46,11 +46,11 @@ fields(mongodb_sharded) -> fields(mongodb_single) -> emqx_connector_mongo:fields(single) ++ fields("config"); fields("post_rs") -> - fields(mongodb_rs); + fields(mongodb_rs) ++ type_and_name_fields(mongodb_rs); fields("post_sharded") -> - fields(mongodb_sharded); + fields(mongodb_sharded) ++ type_and_name_fields(mongodb_sharded); fields("post_single") -> - fields(mongodb_single); + fields(mongodb_single) ++ type_and_name_fields(mongodb_single); fields("put_rs") -> fields(mongodb_rs); fields("put_sharded") -> @@ -58,11 +58,17 @@ fields("put_sharded") -> fields("put_single") -> fields(mongodb_single); fields("get_rs") -> - emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_rs); + emqx_bridge_schema:metrics_status_fields() ++ + fields(mongodb_rs) ++ + type_and_name_fields(mongodb_rs); fields("get_sharded") -> - emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_sharded); + emqx_bridge_schema:metrics_status_fields() ++ + fields(mongodb_sharded) ++ + type_and_name_fields(mongodb_sharded); fields("get_single") -> - emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_single). + emqx_bridge_schema:metrics_status_fields() ++ + fields(mongodb_single) ++ + type_and_name_fields(mongodb_single). conn_bridge_examples(Method) -> [ @@ -103,6 +109,12 @@ desc(_) -> %% Internal fns %%================================================================================================= +type_and_name_fields(MongoType) -> + [ + {type, mk(MongoType, #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. + values(mongodb_rs = MongoType, Method) -> TypeOpts = #{ servers => <<"localhost:27017, localhost:27018">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index bdbf96424..bf5d2e140 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_mysql). @@ -58,11 +58,9 @@ values(post) -> worker_pool_size => 1, health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, - enable_batch => true, batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - enable_queue => false, max_queue_bytes => ?DEFAULT_QUEUE_SIZE } }; diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl new file mode 100644 index 000000000..dc8697e37 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_pgsql). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1, + values/2, + fields/2 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into t_mqtt_msg(msgid, topic, qos, payload, arrived) " + "values (${id}, ${topic}, ${qos}, ${payload}, TO_TIMESTAMP((${timestamp} :: bigint)/1000))" +>>). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"pgsql">> => #{ + summary => <<"PostgreSQL Bridge">>, + value => values(Method, pgsql) + } + } + ]. + +values(get, Type) -> + maps:merge(values(post, Type), ?METRICS_EXAMPLE); +values(post, Type) -> + #{ + enable => true, + type => Type, + name => <<"foo">>, + server => <<"127.0.0.1:5432">>, + database => <<"mqtt">>, + pool_size => 8, + username => <<"root">>, + password => <<"public">>, + auto_reconnect => true, + sql => ?DEFAULT_SQL, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_queue_bytes => ?DEFAULT_QUEUE_SIZE + } + }; +values(put, Type) -> + values(post, Type). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_pgsql". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + emqx_connector_pgsql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields(); +fields("creation_opts") -> + Opts = emqx_resource_schema:fields("creation_opts"), + [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)]; +fields("post") -> + fields("post", pgsql); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:metrics_status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for PostgreSQL using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal +is_hidden_opts(Field) -> + lists:member(Field, [ + async_inflight_window + ]). + +type_field(Type) -> + {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index 5360efa7f..a0c6ba834 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_redis). @@ -50,19 +50,22 @@ values(Protocol, get) -> values("single", post) -> SpecificOpts = #{ server => <<"127.0.0.1:6379">>, + redis_type => single, database => 1 }, values(common, "single", SpecificOpts); values("sentinel", post) -> SpecificOpts = #{ servers => [<<"127.0.0.1:26379">>], + redis_type => sentinel, sentinel => <<"mymaster">>, database => 1 }, values(common, "sentinel", SpecificOpts); values("cluster", post) -> SpecificOpts = #{ - servers => [<<"127.0.0.1:6379">>] + servers => [<<"127.0.0.1:6379">>], + redis_type => cluster }, values(common, "cluster", SpecificOpts); values(Protocol, put) -> @@ -79,8 +82,7 @@ values(common, RedisType, SpecificOpts) -> auto_reconnect => true, command_template => [<<"LPUSH">>, <<"MSGS">>, <<"${payload}">>], resource_opts => #{ - enable_batch => false, - batch_size => 100, + batch_size => 1, batch_time => <<"20ms">> }, ssl => #{enable => false} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl new file mode 100644 index 000000000..20d940462 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_timescale.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_timescale). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"timescale">> => #{ + summary => <<"Timescale Bridge">>, + value => emqx_ee_bridge_pgsql:values(Method, timescale) + } + } + ]. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_timescale". + +roots() -> []. + +fields("post") -> + emqx_ee_bridge_pgsql:fields("post", timescale); +fields(Method) -> + emqx_ee_bridge_pgsql:fields(Method). + +desc(_) -> + undefined. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl index d1fad4765..747cd187d 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- %% Kafka connection configuration diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index 6145716f2..a7de100d6 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -1,8 +1,10 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_bridge_impl_kafka_producer). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + %% callbacks of behaviour emqx_resource -export([ callback_mode/0, @@ -33,7 +35,10 @@ on_start(InstId, Config) -> authentication := Auth, ssl := SSL } = Config, - _ = maybe_install_wolff_telemetry_handlers(InstId), + %% TODO: change this to `kafka_producer` after refactoring for kafka_consumer + BridgeType = kafka, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + _ = maybe_install_wolff_telemetry_handlers(ResourceID), %% it's a bug if producer config is not found %% the caller should not try to start a producer if %% there is no producer config @@ -76,7 +81,8 @@ on_start(InstId, Config) -> {ok, #{ message_template => compile_message_template(MessageTemplate), client_id => ClientId, - producers => Producers + producers => Producers, + resource_id => ResourceID }}; {error, Reason2} -> ?SLOG(error, #{ @@ -102,7 +108,7 @@ on_start(InstId, Config) -> throw(failed_to_start_kafka_producer) end. -on_stop(InstanceID, #{client_id := ClientID, producers := Producers}) -> +on_stop(_InstanceID, #{client_id := ClientID, producers := Producers, resource_id := ResourceID}) -> _ = with_log_at_error( fun() -> wolff:stop_and_delete_supervised_producers(Producers) end, #{ @@ -118,7 +124,7 @@ on_stop(InstanceID, #{client_id := ClientID, producers := Producers}) -> } ), with_log_at_error( - fun() -> uninstall_telemetry_handlers(InstanceID) end, + fun() -> uninstall_telemetry_handlers(ResourceID) end, #{ msg => "failed_to_uninstall_telemetry_handlers", client_id => ClientID @@ -137,7 +143,7 @@ on_query(_InstId, {send_message, Message}, #{message_template := Template, produ %% If the producer process is down when sending, this function would %% raise an error exception which is to be caught by the caller of this callback {_Partition, _Pid} = wolff:send(Producers, [KafkaMessage], {fun ?MODULE:on_kafka_ack/3, [#{}]}), - ok. + {async_return, ok}. compile_message_template(#{ key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate @@ -299,85 +305,100 @@ get_required(Field, Config, Throw) -> Value =:= none andalso throw(Throw), Value. +%% we *must* match the bridge id in the event metadata with that in +%% the handler config; otherwise, multiple kafka producer bridges will +%% install multiple handlers to the same wolff events, multiplying the handle_telemetry_event( [wolff, dropped], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> emqx_resource_metrics:dropped_inc(ID, Val); handle_telemetry_event( [wolff, dropped_queue_full], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> - emqx_resource_metrics:dropped_queue_full_inc(ID, Val); + %% When wolff emits a `dropped_queue_full' event due to replayq + %% overflow, it also emits a `dropped' event (at the time of + %% writing, wolff is 1.7.4). Since we already bump `dropped' when + %% `dropped.queue_full' occurs, we have to correct it here. This + %% correction will have to be dropped if wolff stops also emitting + %% `dropped'. + emqx_resource_metrics:dropped_queue_full_inc(ID, Val), + emqx_resource_metrics:dropped_inc(ID, -Val); handle_telemetry_event( [wolff, queuing], - #{counter_inc := Val}, - #{bridge_id := ID}, - _HandlerConfig + #{gauge_set := Val}, + #{bridge_id := ID, partition_id := PartitionID}, + #{bridge_id := ID} ) when is_integer(Val) -> - emqx_resource_metrics:queuing_change(ID, Val); + emqx_resource_metrics:queuing_set(ID, PartitionID, Val); handle_telemetry_event( [wolff, retried], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> emqx_resource_metrics:retried_inc(ID, Val); handle_telemetry_event( [wolff, failed], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> emqx_resource_metrics:failed_inc(ID, Val); handle_telemetry_event( [wolff, inflight], - #{counter_inc := Val}, - #{bridge_id := ID}, - _HandlerConfig + #{gauge_set := Val}, + #{bridge_id := ID, partition_id := PartitionID}, + #{bridge_id := ID} ) when is_integer(Val) -> - emqx_resource_metrics:inflight_change(ID, Val); + emqx_resource_metrics:inflight_set(ID, PartitionID, Val); handle_telemetry_event( [wolff, retried_failed], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> emqx_resource_metrics:retried_failed_inc(ID, Val); handle_telemetry_event( [wolff, retried_success], #{counter_inc := Val}, #{bridge_id := ID}, - _HandlerConfig + #{bridge_id := ID} ) when is_integer(Val) -> emqx_resource_metrics:retried_success_inc(ID, Val); +handle_telemetry_event( + [wolff, success], + #{counter_inc := Val}, + #{bridge_id := ID}, + #{bridge_id := ID} +) when is_integer(Val) -> + emqx_resource_metrics:success_inc(ID, Val); handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) -> %% Event that we do not handle ok. --spec telemetry_handler_id(emqx_resource:resource_id()) -> binary(). -telemetry_handler_id(InstanceID) -> - <<"emqx-bridge-kafka-producer-", InstanceID/binary>>. +%% Note: don't use the instance/manager ID, as that changes everytime +%% the bridge is recreated, and will lead to multiplication of +%% metrics. +-spec telemetry_handler_id(resource_id()) -> binary(). +telemetry_handler_id(ResourceID) -> + <<"emqx-bridge-kafka-producer-", ResourceID/binary>>. -uninstall_telemetry_handlers(InstanceID) -> - HandlerID = telemetry_handler_id(InstanceID), +uninstall_telemetry_handlers(ResourceID) -> + HandlerID = telemetry_handler_id(ResourceID), telemetry:detach(HandlerID). -maybe_install_wolff_telemetry_handlers(InstanceID) -> +maybe_install_wolff_telemetry_handlers(ResourceID) -> %% Attach event handlers for Kafka telemetry events. If a handler with the %% handler id already exists, the attach_many function does nothing telemetry:attach_many( %% unique handler id - telemetry_handler_id(InstanceID), - %% Note: we don't handle `[wolff, success]' because, - %% currently, we already increment the success counter for - %% this resource at `emqx_rule_runtime:handle_action' when - %% the response is `ok' and we would double increment it - %% here. + telemetry_handler_id(ResourceID), [ [wolff, dropped], [wolff, dropped_queue_full], @@ -386,8 +407,13 @@ maybe_install_wolff_telemetry_handlers(InstanceID) -> [wolff, failed], [wolff, inflight], [wolff, retried_failed], - [wolff, retried_success] + [wolff, retried_success], + [wolff, success] ], fun ?MODULE:handle_telemetry_event/4, - [] + %% we *must* keep track of the same id that is handed down to + %% wolff producers; otherwise, multiple kafka producer bridges + %% will install multiple handlers to the same wolff events, + %% multiplying the metric counts... + #{bridge_id => ResourceID} ). diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl index 429323ad7..867b09f32 100644 --- a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(ee_bridge_hstreamdb_SUITE). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 44149826d..bdde21c76 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_bridge_impl_kafka_producer_SUITE). @@ -287,11 +287,9 @@ kafka_bridge_rest_api_helper(Config) -> ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)), - ?assertEqual(0, emqx_resource_metrics:batching_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:queuing_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_other_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_queue_full_get(ResourceId)), - ?assertEqual(0, emqx_resource_metrics:dropped_queue_not_enabled_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_resource_not_found_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_resource_stopped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:retried_get(ResourceId)), @@ -314,11 +312,9 @@ kafka_bridge_rest_api_helper(Config) -> ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)), - ?assertEqual(0, emqx_resource_metrics:batching_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:queuing_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_other_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_queue_full_get(ResourceId)), - ?assertEqual(0, emqx_resource_metrics:dropped_queue_not_enabled_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_resource_not_found_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:dropped_resource_stopped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:retried_get(ResourceId)), @@ -390,7 +386,7 @@ t_failed_creation_then_fix(_Config) -> }, {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0), ct:pal("base offset before testing ~p", [Offset]), - ?assertEqual(ok, ?PRODUCER:on_query(ResourceId, {send_message, Msg}, State)), + ?assertEqual({async_return, ok}, ?PRODUCER:on_query(ResourceId, {send_message, Msg}, State)), {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset), ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg), %% TODO: refactor those into init/end per testcase @@ -455,7 +451,7 @@ publish_helper(#{ StartRes = ?PRODUCER:on_start(InstId, Conf), {ok, State} = StartRes, OnQueryRes = ?PRODUCER:on_query(InstId, {send_message, Msg}, State), - ok = OnQueryRes, + {async_return, ok} = OnQueryRes, {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset), ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg), ok = ?PRODUCER:on_stop(InstId, State), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index b84b7d74b..0eadf46ad 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_gcp_pubsub_SUITE). @@ -38,18 +38,12 @@ groups() -> {group, sync_query}, {group, async_query} ], - QueueGroups = [ - {group, queue_enabled}, - {group, queue_disabled} - ], ResourceGroups = [{group, gcp_pubsub}], [ {with_batch, SynchronyGroups}, {without_batch, SynchronyGroups}, - {sync_query, QueueGroups}, - {async_query, QueueGroups}, - {queue_enabled, ResourceGroups}, - {queue_disabled, ResourceGroups}, + {sync_query, ResourceGroups}, + {async_query, ResourceGroups}, {gcp_pubsub, MatrixTCs} ]. @@ -99,13 +93,9 @@ init_per_group(sync_query, Config) -> init_per_group(async_query, Config) -> [{query_mode, async} | Config]; init_per_group(with_batch, Config) -> - [{enable_batch, true} | Config]; + [{batch_size, 100} | Config]; init_per_group(without_batch, Config) -> - [{enable_batch, false} | Config]; -init_per_group(queue_enabled, Config) -> - [{enable_queue, true} | Config]; -init_per_group(queue_disabled, Config) -> - [{enable_queue, false} | Config]; + [{batch_size, 1} | Config]; init_per_group(_Group, Config) -> Config. @@ -118,27 +108,30 @@ end_per_group(_Group, _Config) -> init_per_testcase(TestCase, Config0) when TestCase =:= t_publish_success_batch -> - case ?config(enable_batch, Config0) of - true -> + case ?config(batch_size, Config0) of + 1 -> + [{skip_due_to_no_batching, true}]; + _ -> {ok, _} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), - [{telemetry_table, Tid} | Config]; - false -> - {skip, no_batching} + put(telemetry_table, Tid), + [{telemetry_table, Tid} | Config] end; init_per_testcase(TestCase, Config0) -> {ok, _} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), + put(telemetry_table, Tid), [{telemetry_table, Tid} | Config]. end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(), delete_all_bridges(), ok = emqx_connector_web_hook_server:stop(), + emqx_common_test_helpers:call_janitor(), ok. %%------------------------------------------------------------------------------ @@ -268,9 +261,7 @@ certs() -> ]. gcp_pubsub_config(Config) -> - EnableBatch = proplists:get_value(enable_batch, Config, true), QueryMode = proplists:get_value(query_mode, Config, sync), - EnableQueue = proplists:get_value(enable_queue, Config, false), BatchSize = proplists:get_value(batch_size, Config, 100), BatchTime = proplists:get_value(batch_time, Config, <<"20ms">>), PayloadTemplate = proplists:get_value(payload_template, Config, ""), @@ -293,9 +284,7 @@ gcp_pubsub_config(Config) -> " pipelining = ~b\n" " resource_opts = {\n" " worker_pool_size = 1\n" - " enable_batch = ~p\n" " query_mode = ~s\n" - " enable_queue = ~p\n" " batch_size = ~b\n" " batch_time = \"~s\"\n" " }\n" @@ -306,9 +295,7 @@ gcp_pubsub_config(Config) -> PayloadTemplate, PubSubTopic, PipelineSize, - EnableBatch, QueryMode, - EnableQueue, BatchSize, BatchTime ] @@ -355,11 +342,9 @@ service_account_json(PrivateKeyPEM) -> metrics_mapping() -> #{ - batching => fun emqx_resource_metrics:batching_get/1, dropped => fun emqx_resource_metrics:dropped_get/1, dropped_other => fun emqx_resource_metrics:dropped_other_get/1, dropped_queue_full => fun emqx_resource_metrics:dropped_queue_full_get/1, - dropped_queue_not_enabled => fun emqx_resource_metrics:dropped_queue_not_enabled_get/1, dropped_resource_not_found => fun emqx_resource_metrics:dropped_resource_not_found_get/1, dropped_resource_stopped => fun emqx_resource_metrics:dropped_resource_stopped_get/1, failed => fun emqx_resource_metrics:failed_get/1, @@ -392,7 +377,11 @@ assert_metrics(ExpectedMetrics, ResourceId) -> maps:keys(ExpectedMetrics) ), CurrentMetrics = current_metrics(ResourceId), - ?assertEqual(ExpectedMetrics, Metrics, #{current_metrics => CurrentMetrics}), + TelemetryTable = get(telemetry_table), + RecordedEvents = ets:tab2list(TelemetryTable), + ?assertEqual(ExpectedMetrics, Metrics, #{ + current_metrics => CurrentMetrics, recorded_events => RecordedEvents + }), ok. assert_empty_metrics(ResourceId) -> @@ -517,6 +506,29 @@ install_telemetry_handler(TestCase) -> end), Tid. +wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> + Events = receive_all_events(GaugeName, Timeout), + case lists:last(Events) of + #{measurements := #{gauge_set := ExpectedValue}} -> + ok; + #{measurements := #{gauge_set := Value}} -> + ct:fail( + "gauge ~p didn't reach expected value ~p; last value: ~p", + [GaugeName, ExpectedValue, Value] + ) + end. + +receive_all_events(EventName, Timeout) -> + receive_all_events(EventName, Timeout, []). + +receive_all_events(EventName, Timeout, Acc) -> + receive + {telemetry, #{name := [_, _, EventName]} = Event} -> + receive_all_events(EventName, Timeout, [Event | Acc]) + after Timeout -> + lists:reverse(Acc) + end. + wait_telemetry_event(TelemetryTable, EventName, ResourceId) -> wait_telemetry_event(TelemetryTable, EventName, ResourceId, #{timeout => 5_000, n_events => 1}). @@ -553,6 +565,7 @@ t_publish_success(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), + QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, ?check_trace( create_bridge(Config), @@ -581,9 +594,19 @@ t_publish_success(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), + ExpectedInflightEvents = + case QueryMode of + sync -> 1; + async -> 3 + end, + wait_telemetry_event( + TelemetryTable, + inflight, + ResourceId, + #{n_events => ExpectedInflightEvents, timeout => 5_000} + ), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, @@ -600,6 +623,7 @@ t_publish_success_local_topic(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), + QueryMode = ?config(query_mode, Config), LocalTopic = <<"local/topic">>, {ok, _} = create_bridge(Config, #{<<"local_topic">> => LocalTopic}), assert_empty_metrics(ResourceId), @@ -618,9 +642,19 @@ t_publish_success_local_topic(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), + ExpectedInflightEvents = + case QueryMode of + sync -> 1; + async -> 3 + end, + wait_telemetry_event( + TelemetryTable, + inflight, + ResourceId, + #{n_events => ExpectedInflightEvents, timeout => 5_000} + ), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, @@ -648,6 +682,7 @@ t_publish_templated(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), + QueryMode = ?config(query_mode, Config), Topic = <<"t/topic">>, PayloadTemplate = << "{\"payload\": \"${payload}\"," @@ -693,9 +728,19 @@ t_publish_templated(Config) -> ), %% to avoid test flakiness wait_telemetry_event(TelemetryTable, success, ResourceId), + ExpectedInflightEvents = + case QueryMode of + sync -> 1; + async -> 3 + end, + wait_telemetry_event( + TelemetryTable, + inflight, + ResourceId, + #{n_events => ExpectedInflightEvents, timeout => 5_000} + ), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, @@ -709,6 +754,15 @@ t_publish_templated(Config) -> ok. t_publish_success_batch(Config) -> + case proplists:get_bool(skip_due_to_no_batching, Config) of + true -> + ct:pal("this test case is skipped due to non-applicable config"), + ok; + false -> + test_publish_success_batch(Config) + end. + +test_publish_success_batch(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), @@ -760,13 +814,14 @@ t_publish_success_batch(Config) -> ResourceId, #{timeout => 15_000, n_events => NumMessages} ), + wait_until_gauge_is(queuing, 0, _Timeout = 400), + wait_until_gauge_is(inflight, 0, _Timeout = 400), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, - matched => NumMessages div BatchSize, + matched => NumMessages, queuing => 0, retried => 0, success => NumMessages @@ -777,15 +832,13 @@ t_publish_success_batch(Config) -> t_not_a_json(Config) -> ?assertMatch( - {error, - {_, [ - #{ - kind := validation_error, - reason := #{exception := {error, {badmap, "not a json"}}}, - %% should be censored as it contains secrets - value := <<"******">> - } - ]}}, + {error, #{ + discarded_errors_count := 0, + kind := validation_error, + reason := #{exception := {error, {badmap, "not a json"}}}, + %% should be censored as it contains secrets + value := <<"******">> + }}, create_bridge( Config, #{ @@ -797,15 +850,13 @@ t_not_a_json(Config) -> t_not_of_service_account_type(Config) -> ?assertMatch( - {error, - {_, [ - #{ - kind := validation_error, - reason := {wrong_type, <<"not a service account">>}, - %% should be censored as it contains secrets - value := <<"******">> - } - ]}}, + {error, #{ + discarded_errors_count := 0, + kind := validation_error, + reason := {wrong_type, <<"not a service account">>}, + %% should be censored as it contains secrets + value := <<"******">> + }}, create_bridge( Config, #{ @@ -818,22 +869,20 @@ t_not_of_service_account_type(Config) -> t_json_missing_fields(Config) -> GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), ?assertMatch( - {error, - {_, [ - #{ - kind := validation_error, - reason := - {missing_keys, [ - <<"client_email">>, - <<"private_key">>, - <<"private_key_id">>, - <<"project_id">>, - <<"type">> - ]}, - %% should be censored as it contains secrets - value := <<"******">> - } - ]}}, + {error, #{ + discarded_errors_count := 0, + kind := validation_error, + reason := + {missing_keys, [ + <<"client_email">>, + <<"private_key">>, + <<"private_key_id">>, + <<"project_id">>, + <<"type">> + ]}, + %% should be censored as it contains secrets + value := <<"******">> + }}, create_bridge([ {gcp_pubsub_config, GCPPubSubConfig0#{<<"service_account_json">> := #{}}} | Config @@ -951,7 +1000,6 @@ t_publish_timeout(Config) -> do_econnrefused_or_timeout_test(Config, timeout). do_econnrefused_or_timeout_test(Config, Error) -> - EnableQueue = ?config(enable_queue, Config), QueryMode = ?config(query_mode, Config), ResourceId = ?config(resource_id, Config), TelemetryTable = ?config(telemetry_table, Config), @@ -1027,39 +1075,22 @@ do_econnrefused_or_timeout_test(Config, Error) -> end ), - case {Error, QueryMode, EnableQueue} of - {_, sync, false} -> - wait_telemetry_event(TelemetryTable, dropped_queue_not_enabled, ResourceId, #{ - timeout => 10_000, - n_events => 1 - }), - assert_metrics( - #{ - batching => 0, - dropped => 1, - dropped_queue_not_enabled => 1, - failed => 0, - inflight => 0, - matched => 1, - queuing => 0, - retried => 0, - success => 0 - }, - ResourceId - ); + case {Error, QueryMode} of %% apparently, async with disabled queue doesn't mark the %% message as dropped; and since it never considers the %% response expired, this succeeds. - {econnrefused, async, _} -> + {econnrefused, async} -> wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{ - timeout => 10_000, n_events => 2 + timeout => 10_000, n_events => 1 }), + %% even waiting, hard to avoid flakiness... simpler to just sleep + %% a bit until stabilization. + ct:sleep(200), CurrentMetrics = current_metrics(ResourceId), RecordedEvents = ets:tab2list(TelemetryTable), ct:pal("telemetry events: ~p", [RecordedEvents]), ?assertMatch( #{ - batching := 0, dropped := Dropped, failed := 0, inflight := Inflight, @@ -1070,13 +1101,14 @@ do_econnrefused_or_timeout_test(Config, Error) -> } when Matched >= 1 andalso Inflight + Queueing + Dropped =< 2, CurrentMetrics ); - {timeout, async, _} -> + {timeout, async} -> wait_telemetry_event(TelemetryTable, success, ResourceId, #{ timeout => 10_000, n_events => 2 }), + wait_until_gauge_is(inflight, 0, _Timeout = 400), + wait_until_gauge_is(queuing, 0, _Timeout = 400), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, @@ -1087,13 +1119,15 @@ do_econnrefused_or_timeout_test(Config, Error) -> }, ResourceId ); - {_, sync, true} -> + {_, sync} -> wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{ timeout => 10_000, n_events => 2 }), + %% even waiting, hard to avoid flakiness... simpler to just sleep + %% a bit until stabilization. + ct:sleep(200), assert_metrics( #{ - batching => 0, dropped => 0, failed => 0, inflight => 0, @@ -1289,9 +1323,22 @@ t_unrecoverable_error(Config) -> end ), wait_telemetry_event(TelemetryTable, failed, ResourceId), + ExpectedInflightEvents = + case QueryMode of + sync -> 1; + async -> 3 + end, + wait_telemetry_event( + TelemetryTable, + inflight, + ResourceId, + #{n_events => ExpectedInflightEvents, timeout => 5_000} + ), + %% even waiting, hard to avoid flakiness... simpler to just sleep + %% a bit until stabilization. + ct:sleep(200), assert_metrics( #{ - batching => 0, dropped => 0, failed => 1, inflight => 0, @@ -1336,6 +1383,7 @@ t_stop(Config) -> fun(Res, Trace) -> ?assertMatch({ok, {ok, _}}, Res), ?assertMatch([_], ?of_kind(gcp_pubsub_stop, Trace)), + ?assertMatch([_ | _], ?of_kind(connector_jwt_deleted, Trace)), ok end ), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 80b3d1c09..6331611d0 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_influxdb_SUITE). @@ -204,9 +204,9 @@ init_per_group(sync_query, Config) -> init_per_group(async_query, Config) -> [{query_mode, async} | Config]; init_per_group(with_batch, Config) -> - [{enable_batch, true} | Config]; + [{batch_size, 100} | Config]; init_per_group(without_batch, Config) -> - [{enable_batch, false} | Config]; + [{batch_size, 1} | Config]; init_per_group(_Group, Config) -> Config. @@ -261,7 +261,6 @@ example_write_syntax() -> "${undef_key}=\"hard-coded-value\",", "bool=${payload.bool}">>. influxdb_config(apiv1 = Type, InfluxDBHost, InfluxDBPort, Config) -> - EnableBatch = proplists:get_value(enable_batch, Config, true), BatchSize = proplists:get_value(batch_size, Config, 100), QueryMode = proplists:get_value(query_mode, Config, sync), UseTLS = proplists:get_value(use_tls, Config, false), @@ -278,7 +277,6 @@ influxdb_config(apiv1 = Type, InfluxDBHost, InfluxDBPort, Config) -> " precision = ns\n" " write_syntax = \"~s\"\n" " resource_opts = {\n" - " enable_batch = ~p\n" " query_mode = ~s\n" " batch_size = ~b\n" " }\n" @@ -292,7 +290,6 @@ influxdb_config(apiv1 = Type, InfluxDBHost, InfluxDBPort, Config) -> InfluxDBHost, InfluxDBPort, WriteSyntax, - EnableBatch, QueryMode, BatchSize, UseTLS @@ -300,7 +297,6 @@ influxdb_config(apiv1 = Type, InfluxDBHost, InfluxDBPort, Config) -> ), {Name, ConfigString, parse_and_check(ConfigString, Type, Name)}; influxdb_config(apiv2 = Type, InfluxDBHost, InfluxDBPort, Config) -> - EnableBatch = proplists:get_value(enable_batch, Config, true), BatchSize = proplists:get_value(batch_size, Config, 100), QueryMode = proplists:get_value(query_mode, Config, sync), UseTLS = proplists:get_value(use_tls, Config, false), @@ -317,7 +313,6 @@ influxdb_config(apiv2 = Type, InfluxDBHost, InfluxDBPort, Config) -> " precision = ns\n" " write_syntax = \"~s\"\n" " resource_opts = {\n" - " enable_batch = ~p\n" " query_mode = ~s\n" " batch_size = ~b\n" " }\n" @@ -331,7 +326,6 @@ influxdb_config(apiv2 = Type, InfluxDBHost, InfluxDBPort, Config) -> InfluxDBHost, InfluxDBPort, WriteSyntax, - EnableBatch, QueryMode, BatchSize, UseTLS @@ -723,7 +717,7 @@ t_bad_timestamp(Config) -> InfluxDBType = ?config(influxdb_type, Config), InfluxDBName = ?config(influxdb_name, Config), QueryMode = ?config(query_mode, Config), - EnableBatch = ?config(enable_batch, Config), + BatchSize = ?config(batch_size, Config), InfluxDBConfigString0 = ?config(influxdb_config_string, Config), InfluxDBTypeCfg = case InfluxDBType of @@ -774,7 +768,8 @@ t_bad_timestamp(Config) -> fun(Result, Trace) -> ?assertMatch({_, {ok, _}}, Result), {Return, {ok, _}} = Result, - case {QueryMode, EnableBatch} of + IsBatch = BatchSize > 1, + case {QueryMode, IsBatch} of {async, true} -> ?assertEqual(ok, Return), ?assertMatch( @@ -921,12 +916,13 @@ t_write_failure(Config) -> t_missing_field(Config) -> QueryMode = ?config(query_mode, Config), - EnableBatch = ?config(enable_batch, Config), + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, {ok, _} = create_bridge( Config, #{ - <<"resource_opts">> => #{<<"batch_size">> => 1}, + <<"resource_opts">> => #{<<"worker_pool_size">> => 1}, <<"write_syntax">> => <<"${clientid} foo=${foo}i">> } ), @@ -943,9 +939,14 @@ t_missing_field(Config) -> begin emqx:publish(Msg0), emqx:publish(Msg1), + NEvents = + case IsBatch of + true -> 1; + false -> 2 + end, {ok, _} = snabbkaffe:block_until( - ?match_n_events(2, #{ + ?match_n_events(NEvents, #{ ?snk_kind := influxdb_connector_send_query_error, mode := QueryMode }), @@ -956,10 +957,10 @@ t_missing_field(Config) -> fun(Trace) -> PersistedData0 = query_by_clientid(ClientId0, Config), PersistedData1 = query_by_clientid(ClientId1, Config), - case EnableBatch of + case IsBatch of true -> ?assertMatch( - [#{error := points_trans_failed}, #{error := points_trans_failed} | _], + [#{error := points_trans_failed} | _], ?of_kind(influxdb_connector_send_query_error, Trace) ); false -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 35698f812..fb8f1fcc3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_mongodb_SUITE). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 78cb78f8d..812c4ee85 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_mysql_SUITE). @@ -78,10 +78,10 @@ init_per_group(tls, Config) -> | Config ]; init_per_group(with_batch, Config0) -> - Config = [{enable_batch, true} | Config0], + Config = [{batch_size, 100} | Config0], common_init(Config); init_per_group(without_batch, Config0) -> - Config = [{enable_batch, false} | Config0], + Config = [{batch_size, 1} | Config0], common_init(Config); init_per_group(_Group, Config) -> Config. @@ -157,7 +157,7 @@ mysql_config(BridgeType, Config) -> MysqlPort = integer_to_list(?config(mysql_port, Config)), Server = ?config(mysql_host, Config) ++ ":" ++ MysqlPort, Name = atom_to_binary(?MODULE), - EnableBatch = ?config(enable_batch, Config), + BatchSize = ?config(batch_size, Config), QueryMode = ?config(query_mode, Config), TlsEnabled = ?config(enable_tls, Config), ConfigString = @@ -170,7 +170,7 @@ mysql_config(BridgeType, Config) -> " password = ~p\n" " sql = ~p\n" " resource_opts = {\n" - " enable_batch = ~p\n" + " batch_size = ~b\n" " query_mode = ~s\n" " }\n" " ssl = {\n" @@ -185,7 +185,7 @@ mysql_config(BridgeType, Config) -> ?MYSQL_USERNAME, ?MYSQL_PASSWORD, ?SQL_BRIDGE, - EnableBatch, + BatchSize, QueryMode, TlsEnabled ] @@ -404,9 +404,13 @@ t_write_failure(Config) -> emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> send_message(Config, SentData) end), - fun(Result, _Trace) -> - ?assertMatch({error, {resource_error, _}}, Result), - ok + fun + ({error, {resource_error, _}}, _Trace) -> + ok; + ({error, {recoverable_error, disconnected}}, _Trace) -> + ok; + (_, _Trace) -> + ?assert(false) end ), ok. @@ -436,7 +440,9 @@ t_simple_sql_query(Config) -> ), Request = {sql, <<"SELECT count(1) AS T">>}, Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + case IsBatch of true -> ?assertEqual({error, batch_select_not_implemented}, Result); false -> ?assertEqual({ok, [<<"T">>], [[1]]}, Result) end, @@ -448,7 +454,9 @@ t_missing_data(Config) -> create_bridge(Config) ), Result = send_message(Config, #{}), - case ?config(enable_batch, Config) of + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + case IsBatch of true -> ?assertMatch( {error, {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}, Result @@ -465,7 +473,9 @@ t_bad_sql_parameter(Config) -> ), Request = {sql, <<"">>, [bad_parameter]}, Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + case IsBatch of true -> ?assertEqual({error, invalid_request}, Result); false -> ?assertEqual({error, {invalid_params, [bad_parameter]}}, Result) end, @@ -478,7 +488,9 @@ t_unprepared_statement_query(Config) -> ), Request = {prepared_query, unprepared_query, []}, Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + case IsBatch of true -> ?assertEqual({error, invalid_request}, Result); false -> ?assertEqual({error, prepared_statement_invalid}, Result) end, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl new file mode 100644 index 000000000..c2ff6fa8f --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -0,0 +1,493 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_pgsql_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(SQL_BRIDGE, + "INSERT INTO mqtt_test(payload, arrived) " + "VALUES (${payload}, TO_TIMESTAMP((${timestamp} :: bigint)/1000))" +). +-define(SQL_CREATE_TABLE, + "CREATE TABLE IF NOT EXISTS mqtt_test (payload text, arrived timestamp NOT NULL) " +). +-define(SQL_DROP_TABLE, "DROP TABLE mqtt_test"). +-define(SQL_DELETE, "DELETE from mqtt_test"). +-define(SQL_SELECT, "SELECT payload FROM mqtt_test"). + +% DB defaults +-define(PGSQL_DATABASE, "mqtt"). +-define(PGSQL_USERNAME, "root"). +-define(PGSQL_PASSWORD, "public"). +-define(BATCH_SIZE, 10). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + [ + {tcp, [ + {group, with_batch}, + {group, without_batch}, + {group, matrix}, + {group, timescale} + ]}, + {tls, [ + {group, with_batch}, + {group, without_batch}, + {group, matrix}, + {group, timescale} + ]}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs}, + {matrix, [t_setup_via_config_and_publish, t_setup_via_http_api_and_publish]}, + {timescale, [t_setup_via_config_and_publish, t_setup_via_http_api_and_publish]} + ]. + +init_per_group(tcp, Config) -> + Host = os:getenv("PGSQL_TCP_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("PGSQL_TCP_PORT", "5432")), + [ + {pgsql_host, Host}, + {pgsql_port, Port}, + {enable_tls, false}, + {query_mode, sync}, + {proxy_name, "pgsql_tcp"} + | Config + ]; +init_per_group(tls, Config) -> + Host = os:getenv("PGSQL_TLS_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("PGSQL_TLS_PORT", "5433")), + [ + {pgsql_host, Host}, + {pgsql_port, Port}, + {enable_tls, true}, + {query_mode, sync}, + {proxy_name, "pgsql_tls"} + | Config + ]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(matrix, Config0) -> + Config = [{bridge_type, <<"matrix">>}, {enable_batch, true} | Config0], + common_init(Config); +init_per_group(timescale, Config0) -> + Config = [{bridge_type, <<"timescale">>}, {enable_batch, true} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_drop_table(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + connect_and_clear_table(Config), + delete_bridge(Config), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + connect_and_clear_table(Config), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(Config0) -> + BridgeType = proplists:get_value(bridge_type, Config0, <<"pgsql">>), + Host = ?config(pgsql_host, Config0), + Port = ?config(pgsql_port, Config0), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to pgsql directly and create the table + connect_and_create_table(Config0), + {Name, PGConf} = pgsql_config(BridgeType, Config0), + Config = + [ + {pgsql_config, PGConf}, + {pgsql_bridge_type, BridgeType}, + {pgsql_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_pgsql); + _ -> + {skip, no_pgsql} + end + end. + +pgsql_config(BridgeType, Config) -> + Port = integer_to_list(?config(pgsql_port, Config)), + Server = ?config(pgsql_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end, + QueryMode = ?config(query_mode, Config), + TlsEnabled = ?config(enable_tls, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " database = ~p\n" + " username = ~p\n" + " password = ~p\n" + " sql = ~p\n" + " resource_opts = {\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " }\n" + " ssl = {\n" + " enable = ~w\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?PGSQL_DATABASE, + ?PGSQL_USERNAME, + ?PGSQL_PASSWORD, + ?SQL_BRIDGE, + BatchSize, + QueryMode, + TlsEnabled + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + BridgeType = ?config(pgsql_bridge_type, Config), + Name = ?config(pgsql_name, Config), + PGConfig = ?config(pgsql_config, Config), + emqx_bridge:create(BridgeType, Name, PGConfig). + +delete_bridge(Config) -> + BridgeType = ?config(pgsql_bridge_type, Config), + Name = ?config(pgsql_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(pgsql_name, Config), + BridgeType = ?config(pgsql_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?config(pgsql_name, Config), + BridgeType = ?config(pgsql_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request). + +connect_direct_pgsql(Config) -> + Opts = #{ + host => ?config(pgsql_host, Config), + port => ?config(pgsql_port, Config), + username => ?PGSQL_USERNAME, + password => ?PGSQL_PASSWORD, + database => ?PGSQL_DATABASE + }, + + SslOpts = + case ?config(enable_tls, Config) of + true -> + Opts#{ + ssl => true, + ssl_opts => emqx_tls_lib:to_client_opts(#{enable => true}) + }; + false -> + Opts + end, + {ok, Con} = epgsql:connect(SslOpts), + Con. + +% These funs connect and then stop the pgsql connection +connect_and_create_table(Config) -> + Con = connect_direct_pgsql(Config), + {ok, _, _} = epgsql:squery(Con, ?SQL_CREATE_TABLE), + ok = epgsql:close(Con). + +connect_and_drop_table(Config) -> + Con = connect_direct_pgsql(Config), + {ok, _, _} = epgsql:squery(Con, ?SQL_DROP_TABLE), + ok = epgsql:close(Con). + +connect_and_clear_table(Config) -> + Con = connect_direct_pgsql(Config), + {ok, _} = epgsql:squery(Con, ?SQL_DELETE), + ok = epgsql:close(Con). + +connect_and_get_payload(Config) -> + Con = connect_direct_pgsql(Config), + {ok, _, [{Result}]} = epgsql:squery(Con, ?SQL_SELECT), + ok = epgsql:close(Con), + Result. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{payload => Val, timestamp => 1668602148000}, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual({ok, 1}, send_message(Config, SentData)), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), + ?assertMatch( + Val, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(pgsql_connector_query_return, Trace0), + case ?config(enable_batch, Config) of + true -> + ?assertMatch([#{result := {_, [{ok, 1}]}}], Trace); + false -> + ?assertMatch([#{result := {ok, 1}}], Trace) + end, + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(pgsql_bridge_type, Config), + Name = ?config(pgsql_name, Config), + PgsqlConfig0 = ?config(pgsql_config, Config), + PgsqlConfig = PgsqlConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(PgsqlConfig) + ), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{payload => Val, timestamp => 1668602148000}, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual({ok, 1}, send_message(Config, SentData)), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), + ?assertMatch( + Val, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(pgsql_connector_query_return, Trace0), + case ?config(enable_batch, Config) of + true -> + ?assertMatch([#{result := {_, [{ok, 1}]}}], Trace); + false -> + ?assertMatch([#{result := {ok, 1}}], Trace) + end, + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + Name = ?config(pgsql_name, Config), + BridgeType = ?config(pgsql_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {ok, Status} when Status =:= disconnected orelse Status =:= connecting, + emqx_resource_manager:health_check(ResourceID) + ) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := {start_pool_failed, _, _}}], + ?of_kind(pgsql_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge(Config), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{payload => Val, timestamp => 1668602148000}, + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + send_message(Config, SentData) + end), + fun + ({error, {resource_error, _}}, _Trace) -> + ok; + ({error, {recoverable_error, disconnected}}, _Trace) -> + ok; + (_, _Trace) -> + ?assert(false) + end + ), + ok. + +% This test doesn't work with batch enabled since it is not possible +% to set the timeout directly for batch queries +t_write_timeout(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge(Config), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{payload => Val, timestamp => 1668602148000}, + Timeout = 10, + emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {error, {resource_error, _}}, + query_resource(Config, {send_message, SentData, [], Timeout}) + ) + end), + ok. + +t_simple_sql_query(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {sql, <<"SELECT count(1) AS T">>}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> ?assertEqual({error, batch_prepare_not_implemented}, Result); + false -> ?assertMatch({ok, _, [{1}]}, Result) + end, + ok. + +t_missing_data(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + ?assertMatch( + {error, {error, error, <<"23502">>, not_null_violation, _, _}}, Result + ), + ok. + +t_bad_sql_parameter(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {sql, <<"">>, [bad_parameter]}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> + ?assertEqual({error, invalid_request}, Result); + false -> + ?assertMatch( + {error, {resource_error, _}}, Result + ) + end, + ok. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index cd6a2d212..c3529ddeb 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_bridge_redis_SUITE). @@ -17,7 +17,8 @@ %%------------------------------------------------------------------------------ -define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{ - <<"server">> => <<"toxiproxy:6379">> + <<"server">> => <<"toxiproxy:6379">>, + <<"redis_type">> => <<"single">> }). -define(COMMON_REDIS_OPTS, #{ @@ -31,7 +32,7 @@ -define(PROXY_HOST, "toxiproxy"). -define(PROXY_PORT, "8474"). -all() -> [{group, redis_types}, {group, rest}]. +all() -> [{group, transport_types}, {group, rest}]. groups() -> ResourceSpecificTCs = [t_create_delete_bridge], @@ -47,7 +48,7 @@ groups() -> ], [ {rest, TCs}, - {redis_types, [ + {transport_types, [ {group, tcp}, {group, tls} ]}, @@ -63,7 +64,7 @@ groups() -> init_per_group(Group, Config) when Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster -> - [{redis_type, Group} | Config]; + [{transport_type, Group} | Config]; init_per_group(Group, Config) when Group =:= tcp; Group =:= tls -> @@ -79,6 +80,12 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> + wait_for_ci_redis(redis_checks(), Config). + +wait_for_ci_redis(0, _Config) -> + throw(no_redis); +wait_for_ci_redis(Checks, Config) -> + timer:sleep(1000), TestHosts = all_test_hosts(), case emqx_common_test_helpers:is_all_tcp_servers_available(TestHosts) of true -> @@ -96,7 +103,15 @@ init_per_suite(Config) -> | Config ]; false -> - {skip, no_redis} + wait_for_ci_redis(Checks - 1, Config) + end. + +redis_checks() -> + case os:getenv("IS_CI") of + "yes" -> + 10; + _ -> + 1 end. end_per_suite(_Config) -> @@ -108,7 +123,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> ok = delete_all_bridges(), - case ?config(redis_type, Config) of + case ?config(transport_type, Config) of undefined -> Config; RedisType -> @@ -131,7 +146,7 @@ end_per_testcase(_Testcase, Config) -> t_create_delete_bridge(Config) -> Name = <<"mybridge">>, - Type = ?config(redis_type, Config), + Type = ?config(transport_type, Config), BridgeConfig = ?config(bridge_config, Config), IsBatch = ?config(is_batch, Config), ?assertMatch( @@ -220,16 +235,24 @@ t_check_replay(Config) -> begin ?wait_async_action( with_down_failure(Config, ProxyName, fun() -> - ct:sleep(100), - lists:foreach( - fun(_) -> - _ = publish_message(Topic, <<"test_payload">>) - end, - lists:seq(1, ?BATCH_SIZE) - ) + {_, {ok, _}} = + ?wait_async_action( + lists:foreach( + fun(_) -> + _ = publish_message(Topic, <<"test_payload">>) + end, + lists:seq(1, ?BATCH_SIZE) + ), + #{ + ?snk_kind := redis_ee_connector_send_done, + batch := true, + result := {error, _} + }, + 10_000 + ) end), #{?snk_kind := redis_ee_connector_send_done, batch := true, result := {ok, _}}, - 10000 + 10_000 ) end, fun(Trace) -> @@ -374,21 +397,15 @@ all_test_hosts() -> lists:flatmap( fun (#{<<"servers">> := ServersRaw}) -> - lists:map( - fun(Server) -> - parse_server(Server) - end, - string:tokens(binary_to_list(ServersRaw), ", ") - ); + parse_servers(ServersRaw); (#{<<"server">> := ServerRaw}) -> - [parse_server(ServerRaw)] + parse_servers(ServerRaw) end, Confs ). -parse_server(Server) -> - emqx_connector_schema_lib:parse_server(Server, #{ - host_type => hostname, +parse_servers(Servers) -> + emqx_schema:parse_servers(Servers, #{ default_port => 6379 }). @@ -415,31 +432,37 @@ redis_connect_configs() -> #{ redis_single => #{ tcp => #{ - <<"server">> => <<"redis:6379">> + <<"server">> => <<"redis:6379">>, + <<"redis_type">> => <<"single">> }, tls => #{ <<"server">> => <<"redis-tls:6380">>, - <<"ssl">> => redis_connect_ssl_opts(redis_single) + <<"ssl">> => redis_connect_ssl_opts(redis_single), + <<"redis_type">> => <<"single">> } }, redis_sentinel => #{ tcp => #{ <<"servers">> => <<"redis-sentinel:26379">>, + <<"redis_type">> => <<"sentinel">>, <<"sentinel">> => <<"mymaster">> }, tls => #{ <<"servers">> => <<"redis-sentinel-tls:26380">>, + <<"redis_type">> => <<"sentinel">>, <<"sentinel">> => <<"mymaster">>, <<"ssl">> => redis_connect_ssl_opts(redis_sentinel) } }, redis_cluster => #{ tcp => #{ - <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">> + <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>, + <<"redis_type">> => <<"cluster">> }, tls => #{ <<"servers">> => <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>, + <<"redis_type">> => <<"cluster">>, <<"ssl">> => redis_connect_ssl_opts(redis_cluster) } } @@ -449,8 +472,6 @@ toxiproxy_redis_bridge_config() -> Conf0 = ?REDIS_TOXYPROXY_CONNECT_CONFIG#{ <<"resource_opts">> => #{ <<"query_mode">> => <<"async">>, - <<"enable_batch">> => <<"true">>, - <<"enable_queue">> => <<"true">>, <<"worker_pool_size">> => <<"1">>, <<"batch_size">> => integer_to_binary(?BATCH_SIZE), <<"health_check_interval">> => <<"1s">> @@ -463,8 +484,7 @@ invalid_command_bridge_config() -> Conf1 = maps:merge(Conf0, ?COMMON_REDIS_OPTS), Conf1#{ <<"resource_opts">> => #{ - <<"enable_batch">> => <<"false">>, - <<"enable_queue">> => <<"false">>, + <<"batch_size">> => <<"1">>, <<"worker_pool_size">> => <<"1">> }, <<"command_template">> => [<<"BAD">>, <<"COMMAND">>, <<"${payload}">>] @@ -474,13 +494,10 @@ resource_configs() -> #{ batch_off => #{ <<"query_mode">> => <<"sync">>, - <<"enable_batch">> => <<"false">>, - <<"enable_queue">> => <<"false">> + <<"batch_size">> => <<"1">> }, batch_on => #{ <<"query_mode">> => <<"async">>, - <<"enable_batch">> => <<"true">>, - <<"enable_queue">> => <<"true">>, <<"worker_pool_size">> => <<"1">>, <<"batch_size">> => integer_to_binary(?BATCH_SIZE) } diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 38f6689c5..5137574e3 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_conf_schema). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl index 0d6d4f061..941c3e4d1 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl +++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_conf_schema_SUITE). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl index b4bf0de3d..5e1d4e551 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl +++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_conf_schema_tests). diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl index 73807d13a..4b6fbbd92 100644 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%------------------------------------------------------------------- -define(INFLUXDB_DEFAULT_PORT, 8086). diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 1419c2070..ab4c88396 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.5"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 84f9bec8b..dfebd75f5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index 139cb89e9..37d193edf 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_gcp_pubsub). @@ -83,9 +83,7 @@ on_start( %% emulating the emulator behavior %% https://cloud.google.com/pubsub/docs/emulator HostPort = os:getenv("PUBSUB_EMULATOR_HOST", "pubsub.googleapis.com:443"), - {Host, Port} = emqx_connector_schema_lib:parse_server( - HostPort, #{host_type => hostname, default_port => 443} - ), + {Host, Port} = emqx_schema:parse_server(HostPort, #{default_port => 443}), PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), @@ -154,6 +152,7 @@ on_stop( connector => InstanceId }), emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), + emqx_connector_jwt:delete_jwt(?JWT_TABLE, InstanceId), ehttpc_sup:stop_pool(PoolName). -spec on_query( diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl index e4bbe8425..0ae9bb2dc 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_hstreamdb). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 3ef9fc352..7974bf028 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_influxdb). @@ -35,7 +35,6 @@ %% influxdb servers don't need parse -define(INFLUXDB_HOST_OPTIONS, #{ - host_type => hostname, default_port => ?INFLUXDB_DEFAULT_PORT }). @@ -141,7 +140,7 @@ namespace() -> connector_influxdb. fields(common) -> [ - {server, fun server/1}, + {server, server()}, {precision, mk(enum([ns, us, ms, s, m, h]), #{ required => false, default => ms, desc => ?DESC("precision") @@ -164,13 +163,14 @@ fields(influxdb_api_v2) -> {token, mk(binary(), #{required => true, desc => ?DESC("token")})} ] ++ emqx_connector_schema_lib:ssl_fields(). -server(type) -> emqx_schema:ip_port(); -server(required) -> true; -server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; -server(converter) -> fun to_server_raw/1; -server(default) -> <<"127.0.0.1:8086">>; -server(desc) -> ?DESC("server"); -server(_) -> undefined. +server() -> + Meta = #{ + required => false, + default => <<"127.0.0.1:8086">>, + desc => ?DESC("server"), + converter => fun convert_server/2 + }, + emqx_schema:servers_sc(Meta, ?INFLUXDB_HOST_OPTIONS). desc(common) -> ?DESC("common"); @@ -261,9 +261,10 @@ do_start_client( client_config( InstId, Config = #{ - server := {Host, Port} + server := Server } ) -> + {Host, Port} = emqx_schema:parse_server(Server, ?INFLUXDB_HOST_OPTIONS), [ {host, str(Host)}, {port, Port}, @@ -542,17 +543,12 @@ log_error_points(InstId, Errs) -> Errs ). -%% =================================================================== -%% typereflt funcs - --spec to_server_raw(string() | binary()) -> - {string(), pos_integer()}. -to_server_raw(<<"http://", Server/binary>>) -> - emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS); -to_server_raw(<<"https://", Server/binary>>) -> - emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS); -to_server_raw(Server) -> - emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS). +convert_server(<<"http://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(<<"https://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(Server, HoconOpts) -> + emqx_schema:convert_servers(Server, HoconOpts). str(A) when is_atom(A) -> atom_to_list(A); @@ -568,22 +564,6 @@ str(S) when is_list(S) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -to_server_raw_test_() -> - [ - ?_assertEqual( - {"foobar", 1234}, - to_server_raw(<<"http://foobar:1234">>) - ), - ?_assertEqual( - {"foobar", 1234}, - to_server_raw(<<"https://foobar:1234">>) - ), - ?_assertEqual( - {"foobar", 1234}, - to_server_raw(<<"foobar:1234">>) - ) - ]. - %% for coverage desc_test_() -> [ @@ -605,7 +585,7 @@ desc_test_() -> ), ?_assertMatch( {desc, _, _}, - server(desc) + hocon_schema:field_schema(server(), desc) ), ?_assertMatch( connector_influxdb, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl index 39579c737..4ce96d5c7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_redis). diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl index 4de456b2b..8acabbef4 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(ee_connector_hstreamdb_SUITE). diff --git a/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf b/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf index cbe27da16..379cb3358 100644 --- a/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf +++ b/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf @@ -3,14 +3,14 @@ emqx_license_schema { desc { en: "Defines the EMQX Enterprise license. \n\n" "\n" - "The default license has 1000 connections limit, it is " - "issued on 2022-04-19 and valid for 5 years (1825 days).\n" + "The default license has 100 connections limit, it is " + "issued on 2023-01-09 and valid for 5 years (1825 days).\n" "\n" "EMQX comes with a default trial license. For production use, please \n" "visit https://www.emqx.com/apply-licenses/emqx to apply." zh: "EMQX企业许可证。\n" "EMQX 自带一个默认的试用许可证，" - "默认试用许可允许最多接入 1000 个连接，签发时间是 2022年4月19日，有效期是 5 年（1825 天）。" + "默认试用许可允许最多接入 100 个连接，签发时间是 2023年1月9日，有效期是 5 年（1825 天）。" "若需要在生产环境部署，\n" "请访问 https://www.emqx.com/apply-licenses/emqx 来申请。\n" } diff --git a/lib-ee/emqx_license/include/emqx_license.hrl b/lib-ee/emqx_license/include/emqx_license.hrl index f67c63577..b2a0bb40b 100644 --- a/lib-ee/emqx_license/include/emqx_license.hrl +++ b/lib-ee/emqx_license/include/emqx_license.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% @doc EMQX License Management CLI. %%-------------------------------------------------------------------- diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 2ef4ba405..2df1b3797 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib]}, diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index 776a0f239..ef285b937 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license). diff --git a/lib-ee/emqx_license/src/emqx_license_app.erl b/lib-ee/emqx_license/src/emqx_license_app.erl index 3a7c33cf3..6c32389e1 100644 --- a/lib-ee/emqx_license/src/emqx_license_app.erl +++ b/lib-ee/emqx_license/src/emqx_license_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% @doc EMQX License Management Application. %%-------------------------------------------------------------------- diff --git a/lib-ee/emqx_license/src/emqx_license_checker.erl b/lib-ee/emqx_license/src/emqx_license_checker.erl index 5840d8917..2ebb96004 100644 --- a/lib-ee/emqx_license/src/emqx_license_checker.erl +++ b/lib-ee/emqx_license/src/emqx_license_checker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_checker). diff --git a/lib-ee/emqx_license/src/emqx_license_cli.erl b/lib-ee/emqx_license/src/emqx_license_cli.erl index 87f2ac702..26c3a074e 100644 --- a/lib-ee/emqx_license/src/emqx_license_cli.erl +++ b/lib-ee/emqx_license/src/emqx_license_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_cli). diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index 17e85b88a..0133cd637 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_http_api). diff --git a/lib-ee/emqx_license/src/emqx_license_installer.erl b/lib-ee/emqx_license/src/emqx_license_installer.erl index 9a24c811b..7a6ea5339 100644 --- a/lib-ee/emqx_license/src/emqx_license_installer.erl +++ b/lib-ee/emqx_license/src/emqx_license_installer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_installer). diff --git a/lib-ee/emqx_license/src/emqx_license_parser.erl b/lib-ee/emqx_license/src/emqx_license_parser.erl index 727ef2760..05625902a 100644 --- a/lib-ee/emqx_license/src/emqx_license_parser.erl +++ b/lib-ee/emqx_license/src/emqx_license_parser.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% @doc EMQX License Management. %%-------------------------------------------------------------------- diff --git a/lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl b/lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl index d721e3de0..4b2d6dccd 100644 --- a/lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl +++ b/lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_parser_v20220101). diff --git a/lib-ee/emqx_license/src/emqx_license_resources.erl b/lib-ee/emqx_license/src/emqx_license_resources.erl index 4aaf853a3..3f353064b 100644 --- a/lib-ee/emqx_license/src/emqx_license_resources.erl +++ b/lib-ee/emqx_license/src/emqx_license_resources.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_resources). diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index 4300963fd..bde4ed076 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_schema). @@ -78,11 +78,11 @@ check_license_watermark(Conf) -> %% @doc The default license key. %% This default license has 1000 connections limit. -%% It is issued on 2022-04-19 and valid for 5 years (1825 days) +%% It is issued on 2023-01-09 and valid for 5 years (1825 days) %% NOTE: when updating a new key, the schema doc in emqx_license_schema_i18n.conf %% should be updated accordingly default_license() -> "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ" - "GVmYXVsdAoyMDIyMDQxOQoxODI1CjEwMDAK.MEQCICbgRVijCQov2" - "hrvZXR1mk9Oa+tyV1F5oJ6iOZeSHjnQAiB9dUiVeaZekDOjztk+NC" - "Wjhk4PG8tWfw2uFZWruSzD6g==". + "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f" + "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn" + "8mgRDtiT3R8RZooqy6dEsOmDI=". diff --git a/lib-ee/emqx_license/src/emqx_license_sup.erl b/lib-ee/emqx_license/src/emqx_license_sup.erl index ce1a6a636..6b8f73953 100644 --- a/lib-ee/emqx_license/src/emqx_license_sup.erl +++ b/lib-ee/emqx_license/src/emqx_license_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% @doc EMQX License Management Supervisor. %%-------------------------------------------------------------------- diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl b/lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl index 81e2d3ca7..842dcb77c 100644 --- a/lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl +++ b/lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_proto_v1). diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl index b2d99d5d4..1254115bc 100644 --- a/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl +++ b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_proto_v2). diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index b403d763d..4a0f6d91b 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl index d3e585d30..a4ef1af6e 100644 --- a/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_checker_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl index 84d694ca5..630880f01 100644 --- a/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_cli_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 082a236d8..11ac1ca89 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_http_api_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl index dfc64672c..5d5e27489 100644 --- a/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_installer_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_installer_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl index 44c83d27b..f2f24dc54 100644 --- a/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_parser_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl index 529eedf24..66e5d4a61 100644 --- a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_resources_SUITE). diff --git a/lib-ee/emqx_license/test/emqx_license_test_lib.erl b/lib-ee/emqx_license/test/emqx_license_test_lib.erl index 008db1b41..644b6959d 100644 --- a/lib-ee/emqx_license/test/emqx_license_test_lib.erl +++ b/lib-ee/emqx_license/test/emqx_license_test_lib.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_license_test_lib). diff --git a/mix.exs b/mix.exs index 1bf30486e..45bc820e0 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do {:lc, github: "emqx/lc", tag: "0.3.2", override: true}, {:redbug, "2.0.8"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, - {:ehttpc, github: "emqx/ehttpc", tag: "0.4.2", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.4.3", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.35.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -132,10 +132,10 @@ defmodule EMQXUmbrella.MixProject do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true}, - {:wolff, github: "kafka4beam/wolff", tag: "1.7.0"}, - {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.0", override: true}, + {:wolff, github: "kafka4beam/wolff", tag: "1.7.4"}, + {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"}, - {:brod, github: "kafka4beam/brod", tag: "3.16.4"}, + {:brod, github: "kafka4beam/brod", tag: "3.16.7"}, {:snappyer, "1.2.8", override: true}, {:supervisor3, "1.1.11", override: true} ] @@ -640,7 +640,7 @@ defmodule EMQXUmbrella.MixProject do defp jq_dep() do if enable_jq?(), - do: [{:jq, github: "emqx/jq", tag: "v0.3.8", override: true}], + do: [{:jq, github: "emqx/jq", tag: "v0.3.9", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index 861046ad9..a61bfe16a 100644 --- a/rebar.config +++ b/rebar.config @@ -49,7 +49,7 @@ , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.2"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.3"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} @@ -68,7 +68,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} @@ -87,4 +87,9 @@ emqx_exproto_pb % generated code for protobuf ]}. -{project_plugins, [erlfmt, {rebar3_hex, "7.0.2"}]}. +{project_plugins, + [ erlfmt, + {rebar3_hex, "7.0.2"}, + {rebar3_sbom, + {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} +]}. diff --git a/rebar.config.erl b/rebar.config.erl index 1770fc66f..e8b1271f2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -42,7 +42,7 @@ quicer() -> {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. jq() -> - {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.8"}}}. + {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 95e71f128..3432c757c 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*') +latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*' --exclude '*docker*') +echo "Compare base: $latest_release" bad_app_count=0 diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index d86ee4f6a..3a7b40317 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -11,19 +11,23 @@ help() { echo echo "-h|--help: To display this usage info" echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests" - echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl" - echo "--console: Start EMQX in console mode" + echo "--console: Start EMQX in console mode but do not run test cases" echo "--attach: Attach to the Erlang docker container without running any test case" + echo "--stop: Stop running containers for the given app" echo "--only-up: Only start the testbed but do not run CT" echo "--keep-up: Keep the testbed running after CT" + echo "--ci: Set this flag in GitHub action to enforce no tests are skipped" + echo "--" If any, all args after '--' are passed to rebar3 ct + echo " otherwise it runs the entire app's CT" } WHICH_APP='novalue' CONSOLE='no' KEEP_UP='no' ONLY_UP='no' -SUITES='' ATTACH='no' +STOP='no' +IS_CI='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -46,13 +50,22 @@ while [ "$#" -gt 0 ]; do ATTACH='yes' shift 1 ;; + --stop) + STOP='yes' + shift 1 + ;; --console) CONSOLE='yes' shift 1 ;; - --suites) - SUITES="$2" - shift 2 + --ci) + IS_CI='yes' + shift 1 + ;; + --) + shift 1 + REBAR3CT="$*" + shift $# ;; *) echo "unknown option $1" @@ -63,6 +76,7 @@ done if [ "${WHICH_APP}" = 'novalue' ]; then echo "must provide --app arg" + help exit 1 fi @@ -143,7 +157,7 @@ F_OPTIONS="" for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done - +ORIG_UID_GID="$UID:$UID" if [[ "${NEED_ROOT:-}" == 'yes' ]]; then export UID_GID='root:root' else @@ -152,11 +166,13 @@ else # Permissions issue happens because we are mounting local filesystem # where files are owned by $UID to docker container where it's using # root (UID=0) by default, and git is not happy about it. - export UID_GID="$UID:$UID" + export UID_GID="$ORIG_UID_GID" fi -# shellcheck disable=2086 # no quotes for F_OPTIONS -docker-compose $F_OPTIONS up -d --build --remove-orphans +if [ "$STOP" = 'no' ]; then + # shellcheck disable=2086 # no quotes for F_OPTIONS + docker-compose $F_OPTIONS up -d --build --remove-orphans +fi # /emqx is where the source dir is mounted to the Erlang container # in .ci/docker-compose-file/docker-compose.yaml @@ -171,23 +187,44 @@ docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache & # need to initialize .erlang.cookie manually here because / is not writable by $UID docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie" +restore_ownership() { + if [[ "$ORIG_UID_GID" != "$UID_GID" ]]; then + docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $ORIG_UID_GID /emqx" + fi +} + if [ "$ONLY_UP" = 'yes' ]; then exit 0 fi -if [ "$ATTACH" = 'yes' ]; then +set +e + +if [ "$STOP" = 'yes' ]; then + # shellcheck disable=2086 # no quotes for F_OPTIONS + docker-compose $F_OPTIONS down --remove-orphans +elif [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash + restore_ownership elif [ "$CONSOLE" = 'yes' ]; then docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run" + restore_ownership else - set +e - docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" - RESULT=$? - if [ "$KEEP_UP" = 'yes' ]; then - exit $RESULT + if [ -z "${REBAR3CT:-}" ]; then + docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" else + docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT" + fi + RESULT=$? + restore_ownership + if [ $RESULT -ne 0 ]; then + LOG='_build/test/logs/docker-compose.log' + echo "Dumping docker-compose log to $LOG" + # shellcheck disable=2086 # no quotes for F_OPTIONS + docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG" + fi + if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS docker-compose $F_OPTIONS down - exit $RESULT fi + exit $RESULT fi diff --git a/scripts/format-changelog.sh b/scripts/format-changelog.sh new file mode 100755 index 000000000..862b42c17 --- /dev/null +++ b/scripts/format-changelog.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail +shopt -s nullglob +export LANG=C.UTF-8 + +[ "$#" -ne 2 ] && { + echo "Usage $0 " 1>&2; + exit 1 +} + +version="${1}" +language="${2}" + +changes_dir="$(git rev-parse --show-toplevel)/changes/${version}" + +item() { + local filename pr indent + filename="${1}" + pr="$(echo "${filename}" | sed -E 's/.*-([0-9]+)\.(en|zh)\.md$/\1/')" + indent="- [#${pr}](https://github.com/emqx/emqx/pull/${pr}) " + while read -r line; do + echo "${indent}${line}" + indent=" " + done < "${filename}" + echo +} + +section() { + local section_name=$1 + for i in "${changes_dir}"/"${section_name}"-*."${language}".md; do + item "${i}" + done +} + +if [ "${language}" = "en" ]; then + cat <&2; + exit 1 +fi diff --git a/scripts/git-hook-pre-commit.sh b/scripts/git-hook-pre-commit.sh index a5e441d05..b0e0699b9 100755 --- a/scripts/git-hook-pre-commit.sh +++ b/scripts/git-hook-pre-commit.sh @@ -15,3 +15,4 @@ if ! (./scripts/erlfmt $OPT $files); then echo "EXECUTE 'make fmt' to fix" >&2 exit 1 fi +./scripts/apps-version-check.sh diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 751ca841c..816cbe182 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -10,8 +10,9 @@ main(_) -> Conf = [merge(Conf0, Cfgs1), io_lib:nl() ], - ok = filelib:ensure_dir("apps/emqx_dashboard/priv/i18n.conf"), - ok = file:write_file("apps/emqx_dashboard/priv/i18n.conf", Conf). + OutputFile = "apps/emqx_dashboard/priv/i18n.conf", + ok = filelib:ensure_dir(OutputFile), + ok = file:write_file(OutputFile, Conf). merge(BaseConf, Cfgs) -> lists:foldl( diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 0fe99c6b2..56b7d47b4 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -21,3 +21,4 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." ./scripts/get-dashboard.sh "$dashboard_version" ./scripts/merge-config.escript ./scripts/merge-i18n.escript +./scripts/update-bom.sh "$PROFILE_STR" ./rel diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index d28955bca..d35176600 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -206,9 +206,18 @@ if [ -d "${CHECKS_DIR}" ]; then done fi +generate_changelog () { + local CHANGES_EN_MD="changes/${TAG}-en.md" CHANGES_ZH_MD="changes/${TAG}-zh.md" + ./scripts/format-changelog.sh "${TAG}" "en" > "$CHANGES_EN_MD" + ./scripts/format-changelog.sh "${TAG}" "zh" > "$CHANGES_ZH_MD" + git add "$CHANGES_EN_MD" "$CHANGES_ZH_MD" + [ -n "$(git status -s)" ] && git commit -m "chore: Generate changelog for ${TAG}" +} + if [ "$DRYRUN" = 'yes' ]; then logmsg "Release tag is ready to be created with command: git tag $TAG" else + generate_changelog git tag "$TAG" logmsg "$TAG is created OK." fi diff --git a/scripts/update-bom.sh b/scripts/update-bom.sh new file mode 100755 index 000000000..20ab45e22 --- /dev/null +++ b/scripts/update-bom.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +[[ -n "$WITHOUT_UPDATE_BOM" ]] && exit 0 + +set -euo pipefail + +PROFILE="$1" +REL_DIR="$2" + +./rebar3 as "$PROFILE" sbom -f -o "$REL_DIR/bom.json" +