Merge pull request #9720 from zhongwencool/release-v5.0.14

Release v5.0.14

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -54,7 +54,7 @@ services:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
-      KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+      KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
       KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
@@ -66,8 +66,8 @@ services:
     volumes:
       - emqx-shared-secret:/var/lib/secret
       - ./kafka/jaas.conf:/etc/kafka/jaas.conf
-      - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+      - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh
       - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
       - ./kerberos/krb5.conf:/etc/krb5.conf
-    command: run_add_scram_users.sh
+    command: kafka-entrypoint.sh
 

.ci/docker-compose-file/docker-compose-toxiproxy.yaml

@@ -15,6 +15,8 @@ services:
       - 8087:8087
       - 13306:3306
       - 13307:3307
+      - 15432:5432
+      - 15433:5433
     command:
       - "-host=0.0.0.0"
       - "-config=/config/toxiproxy.json"

.ci/docker-compose-file/kafka/kafka-entrypoint.sh

@@ -22,6 +22,7 @@ sleep 3
 
 echo "+++++++ Starting Kafka ++++++++"
 
+# fork start Kafka
 start-kafka.sh &
 
 SERVER=localhost
@@ -41,6 +42,12 @@ echo "+++++++ Run config commands ++++++++"
 
 kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=password],SCRAM-SHA-512=[password=password]' --entity-type users --entity-name emqxuser
 
+echo "+++++++ Creating Kafka Topics ++++++++"
+
+# create topics after re-configuration
+# there seem to be a race condition when creating the topics (too early)
+env KAFKA_CREATE_TOPICS="$KAFKA_CREATE_TOPICS_NG" KAFKA_PORT="$PORT1" create-topics.sh
+
 echo "+++++++ Wait until Kafka ports are down ++++++++"
 
 bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1

.ci/docker-compose-file/toxiproxy.json

@@ -29,5 +29,17 @@
     "listen": "0.0.0.0:6379",
     "upstream": "redis:6379",
     "enabled": true
+  },
+  {
+    "name": "pgsql_tcp",
+    "listen": "0.0.0.0:5432",
+    "upstream": "pgsql:5432",
+    "enabled": true
+  },
+  {
+    "name": "pgsql_tls",
+    "listen": "0.0.0.0:5433",
+    "upstream": "pgsql-tls:5432",
+    "enabled": true
   }
 ]

.editorconfig

@@ -20,8 +20,3 @@ indent_size = 4
 # Tab indentation (no size specified)
 [Makefile]
 indent_style = tab
-
-# Matches the exact files either package.json or .travis.yml
-[{.travis.yml}]
-indent_style = space
-indent_size = 2

.github/CODEOWNERS

@@ -1,42 +1,38 @@
-## MQTT
+## Default
-/apps/emqx_connector/src/mqtt/ @qzhuyan
+* @emqx/emqx-review-board
-/apps/emqx/*/*mqtt* @qzhuyan
 
 ## apps
-/apps/emqx/                @lafirest @thalesmg @HJianBo @ieQu1
+/apps/emqx/                @emqx/emqx-review-board @lafirest @thalesmg
-/apps/emqx_authn/          @savonarola @JimMoen @HJianBo
+/apps/emqx_authn/          @emqx/emqx-review-board @JimMoen @savonarola
-/apps/emqx_authz/          @savonarola @JimMoen @HJianBo
+/apps/emqx_authz/          @emqx/emqx-review-board @JimMoen @savonarola
-/apps/emqx_auto_subscribe/ @thalesmg @HJianBo
+/apps/emqx_auto_subscribe/ @emqx/emqx-review-board @thalesmg
-/apps/emqx_bridge/         @terry-xiaoyu @thalesmg
+/apps/emqx_bridge/         @emqx/emqx-review-board @thalesmg
-/apps/emqx_conf/           @ieQu1 @thalesmg
+/apps/emqx_conf/           @emqx/emqx-review-board @thalesmg
-/apps/emqx_connector/      @terry-xiaoyu @JimMoen
+/apps/emqx_connector/      @emqx/emqx-review-board @JimMoen
-/apps/emqx_dashboard/      @lafirest @JimMoen
+/apps/emqx_dashboard/      @emqx/emqx-review-board @JimMoen @lafirest
-/apps/emqx_exhook/         @lafirest @HJianBo @JimMoen
+/apps/emqx_exhook/         @emqx/emqx-review-board @JimMoen @lafirest
-/apps/emqx_gateway/        @HJianBo @lafirest
+/apps/emqx_gateway/        @emqx/emqx-review-board @lafirest
-/apps/emqx_machine/        @thalesmg @terry-xiaoyu @ieQu1
+/apps/emqx_machine/        @emqx/emqx-review-board @thalesmg
-/apps/emqx_management/     @HJianBo @lafirest @sstrigler
+/apps/emqx_management/     @emqx/emqx-review-board @lafirest @sstrigler
-/apps/emqx_modules/        @thalesmg @terry-xiaoyu @HJianBo
+/apps/emqx_modules/        @emqx/emqx-review-board @thalesmg
-/apps/emqx_plugin_libs/    @terry-xiaoyu @lafirest
+/apps/emqx_plugin_libs/    @emqx/emqx-review-board @lafirest
-/apps/emqx_plugins/        @thalesmg @JimMoen @ieQu1
+/apps/emqx_plugins/        @emqx/emqx-review-board @JimMoen @thalesmg
-/apps/emqx_prometheus/     @JimMoen @ieQu1
+/apps/emqx_prometheus/     @emqx/emqx-review-board @JimMoen
-/apps/emqx_psk/            @lafirest @thalesmg @terry-xiaoyu
+/apps/emqx_psk/            @emqx/emqx-review-board @lafirest @thalesmg
-/apps/emqx_resource/       @terry-xiaoyu @thalesmg
+/apps/emqx_resource/       @emqx/emqx-review-board @thalesmg
-/apps/emqx_retainer/       @lafirest @ieQu1 @thalesmg
+/apps/emqx_retainer/       @emqx/emqx-review-board @lafirest @thalesmg
-/apps/emqx_rule_engine/    @terry-xiaoyu @HJianBo @kjellwinblad
+/apps/emqx_rule_engine/    @emqx/emqx-review-board @kjellwinblad
-/apps/emqx_slow_subs/      @lafirest @HJianBo
+/apps/emqx_slow_subs/      @emqx/emqx-review-board @lafirest
-/apps/emqx_statsd/         @JimMoen @HJianBo
+/apps/emqx_statsd/         @emqx/emqx-review-board @JimMoen
 
 ## other
-/lib-ee/                   @thalesmg
+/lib-ee/                   @emqx/emqx-review-board @thalesmg
-/bin/                      @zmstone @thalesmg @terry-xiaoyu @id
+/bin/                      @emqx/emqx-review-board @thalesmg @id
-/rel/                      @zmstone @thalesmg @id
+/rel/                      @emqx/emqx-review-board @thalesmg @id
 
 ## CI
-/.github/ @id
+/.github/ @emqx/emqx-review-board @id
-/.ci/     @id
+/.ci/     @emqx/emqx-review-board @id
-/scripts/ @id
+/scripts/ @emqx/emqx-review-board @id
-/build    @id
+/build    @emqx/emqx-review-board @id
-/deploy/  @id
+/deploy/  @emqx/emqx-review-board @id
-
-## Default
-* @zmstone

.github/PULL_REQUEST_TEMPLATE/v5.md

@@ -5,7 +5,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev
 
 - [ ] Added tests for the changes
 - [ ] Changed lines covered in coverage report
-- [ ] Change log has been added to `changes/` dir
+- [ ] Change log has been added to `changes/<version>/(feat|fix)-<PR-id>.en.md` and `.zh.md` files
 - [ ] For internal contributor: there is a jira ticket to track this change
 - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up
 - [ ] Schema changes are backward compatible

.github/workflows/build_packages.yaml

@@ -127,9 +127,18 @@ jobs:
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
         echo "EMQX uninstalled"
     - uses: actions/upload-artifact@v3
+      if: success()
       with:
         name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
+    - name: Send notification to Slack
+      uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.event_name == 'schedule'
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      with:
+        payload: |
+          {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
 
   mac:
     needs: prepare
@@ -165,9 +174,18 @@ jobs:
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
     - uses: actions/upload-artifact@v3
+      if: success()
       with:
         name: ${{ matrix.profile }}
         path: _packages/${{ matrix.profile }}/
+    - name: Send notification to Slack
+      uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.event_name == 'schedule'
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      with:
+        payload: |
+          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
 
   linux:
     needs: prepare
@@ -215,7 +233,7 @@ jobs:
             elixir: 1.13.4
             release_with: elixir
           - profile: emqx
-            otp: 24.3.4.2-1 # TODO: 25.1.2-2
+            otp: 25.1.2-2
             arch: amd64
             os: amzn2
             build_machine: ubuntu-20.04
@@ -271,9 +289,18 @@ jobs:
             --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}"
         done
     - uses: actions/upload-artifact@v3
+      if: success()
       with:
         name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
+    - name: Send notification to Slack
+      uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.event_name == 'schedule'
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      with:
+        payload: |
+          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
 
   publish_artifacts:
     runs-on: ubuntu-20.04

.github/workflows/run_test_cases.yaml

@@ -162,7 +162,7 @@ jobs:
             INFLUXDB_TAG: 2.5.0
             PROFILE: ${{ matrix.profile }}
             CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
-          run: ./scripts/ct/run.sh --app ${{ matrix.app }}
+          run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
         - uses: actions/upload-artifact@v3
           with:
             name: coverdata
@@ -170,7 +170,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
+            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
             path: source/_build/test/logs
 
     ct:
@@ -213,7 +213,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
+            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
             path: source/_build/test/logs
 
     make_cover:

.gitignore

@@ -69,3 +69,4 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi
 *.conf.rendered
 lux_logs/
 /.prepare
+bom.json

CONTRIBUTING.md

@@ -55,7 +55,7 @@ Must be one of the following:
 - **chore**: Updating grunt tasks etc; no production code change
 - **perf**: A code change that improves performance
 - **test**: Adding missing tests, refactoring tests; no production code change
-- **build**: Changes that affect the CI/CD pipeline or build system or external dependencies (example scopes: travis, jenkins, makefile)
+- **build**: Changes that affect the CI/CD pipeline or build system or external dependencies (example scopes: jenkins, makefile)
 - **ci**: Changes provided by DevOps for CI purposes.
 - **revert**: Reverts a previous commit.
 

Makefile

@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.4
+export EMQX_DASHBOARD_VERSION ?= v1.1.5
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.9
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
@@ -88,6 +88,7 @@ define gen-app-ct-target
 $1-ct: $(REBAR)
 	@$(SCRIPTS)/pre-compile.sh $(PROFILE)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \
+	        --readable=$(CT_READABLE) \
 		--name $(CT_NODE_NAME) \
 		--cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \
 		--suite $(shell $(SCRIPTS)/find-suites.sh $1)

NOTICE

@@ -1,5 +1,5 @@
 EMQX, highly scalable, highly available distributed MQTT messaging platform for IoT.
-Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 
 This product contains code developed at EMQ Technologies Co., Ltd.
 Visit https://www.emqx.come to learn more.

README-CN.md

@@ -1,7 +1,7 @@
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

README-RU.md

@@ -1,7 +1,7 @@
 # Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

README.md

@@ -1,7 +1,7 @@
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

apps/emqx/NOTICE

@@ -1,5 +1,5 @@
 EMQX, highly scalable, highly available distributed MQTT messaging platform for IoT.
-Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 
 This product contains code developed at EMQ Technologies Co., Ltd.
 Visit https://www.emqx.come to learn more.

apps/emqx/include/bpapi.hrl

@@ -1,3 +1,19 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
 -ifndef(EMQX_BPAPI_HRL).
 -define(EMQX_BPAPI_HRL, true).
 

apps/emqx/include/emqx.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_access_control.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_authentication.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_hooks.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_mqtt.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_placeholder.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/emqx_release.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.13").
+-define(EMQX_RELEASE_CE, "5.0.14").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-beta.6").

apps/emqx/include/http_api.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/logger.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/include/types.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

apps/emqx/src/bpapi/emqx_bpapi.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/bpapi/emqx_bpapi.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/bpapi/emqx_bpapi_trans.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/config/emqx_config_logger.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx.app.src

@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.13"},
+    {vsn, "5.0.14"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_access_control.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_alarm.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_alarm_handler.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_authentication.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_authentication_config.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_authentication_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_authn_authz_metrics_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_authz_cache.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_banned.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_base62.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_batch.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_boot.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_broker.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_broker_bench.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_broker_helper.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_broker_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_channel.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_cm.erl

@@ -1,5 +1,5 @@
 %%-------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_cm.hrl

@@ -1,5 +1,5 @@
 %%-------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_cm_locker.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_cm_registry.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_cm_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_config.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -362,8 +362,8 @@ schema_default(Schema) ->
             [];
         ?LAZY(?ARRAY(_)) ->
             [];
-        ?LAZY(?UNION(Unions)) ->
+        ?LAZY(?UNION(Members)) ->
-            case [A || ?ARRAY(A) <- Unions] of
+            case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
                 [_ | _] -> [];
                 _ -> #{}
             end;
@@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) ->
         required => false,
         format => map,
         apply_override_envs => true,
-        remove_env_meta => true,
         check_lazy => true
     },
     hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).
@@ -413,6 +412,31 @@ check_config(SchemaMod, RawConf) ->
     check_config(SchemaMod, RawConf, #{}).
 
 check_config(SchemaMod, RawConf, Opts0) ->
+    try
+        do_check_config(SchemaMod, RawConf, Opts0)
+    catch
+        throw:{Schema, Errors} ->
+            compact_errors(Schema, Errors)
+    end.
+
+%% HOCON tries to be very informative about all the detailed errors
+%% it's maybe too much when reporting to the user
+-spec compact_errors(any(), any()) -> no_return().
+compact_errors(Schema, [Error0 | More]) when is_map(Error0) ->
+    Error1 = Error0#{discarded_errors_count => length(More)},
+    Error =
+        case is_atom(Schema) of
+            true ->
+                Error1#{schema_module => Schema};
+            false ->
+                Error1
+        end,
+    throw(Error);
+compact_errors(Schema, Errors) ->
+    %% unexpected, we need the stacktrace reported, hence error
+    error({Schema, Errors}).
+
+do_check_config(SchemaMod, RawConf, Opts0) ->
     Opts1 = #{
         return_plain => true,
         format => map,

apps/emqx/src/emqx_config_handler.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -245,7 +245,7 @@ process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
             BinKeyPath = bin_path(ConfKeyPath),
             case check_permissions(update, BinKeyPath, NewRawConf, Opts) of
                 allow ->
-                    OverrideConf = update_override_config(NewRawConf, Opts),
+                    OverrideConf = merge_to_override_config(NewRawConf, Opts),
                     {ok, NewRawConf, OverrideConf, Opts};
                 {deny, Reason} ->
                     {error, {permission_denied, Reason}}
@@ -447,9 +447,10 @@ remove_from_override_config(BinKeyPath, Opts) ->
     OldConf = emqx_config:read_override_conf(Opts),
     emqx_map_lib:deep_remove(BinKeyPath, OldConf).
 
-update_override_config(_RawConf, #{persistent := false}) ->
+%% apply new config on top of override config
+merge_to_override_config(_RawConf, #{persistent := false}) ->
     undefined;
-update_override_config(RawConf, Opts) ->
+merge_to_override_config(RawConf, Opts) ->
     OldConf = emqx_config:read_override_conf(Opts),
     maps:merge(OldConf, RawConf).
 

apps/emqx/src/emqx_congestion.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_connection.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_ctl.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_datetime.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_exclusive_subscription.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_flapping.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_frame.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_gc.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_guid.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_hocon.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_hooks.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_inflight.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_json.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_keepalive.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_kernel_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_correction.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_listeners.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_logger.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -221,7 +221,7 @@ best_effort_json_obj(Map, Config) ->
     end.
 
 json([], _) ->
-    "[]";
+    "";
 json(<<"">>, _) ->
     "\"\"";
 json(A, _) when is_atom(A) -> atom_to_binary(A, utf8);
@@ -376,4 +376,19 @@ p_config() ->
         ]
     ).
 
+best_effort_json_test() ->
+    ?assertEqual(
+        <<"{}">>,
+        emqx_logger_jsonfmt:best_effort_json([])
+    ),
+    ?assertEqual(
+        <<"{\n    \"key\": []\n}">>,
+        emqx_logger_jsonfmt:best_effort_json(#{key => []})
+    ),
+    ?assertEqual(
+        <<"[\n    {\n        \"key\": []\n    }\n]">>,
+        emqx_logger_jsonfmt:best_effort_json([#{key => []}])
+    ),
+    ok.
+
 -endif.

apps/emqx/src/emqx_logger_textfmt.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_map_lib.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_message.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_metrics.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_metrics_worker.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@
 
 -behaviour(gen_server).
 
+-include_lib("stdlib/include/ms_transform.hrl").
+
 %% API functions
 -export([
     start_link/1,
@@ -29,8 +31,16 @@
 -export([
     inc/3,
     inc/4,
+    observe/4,
     get/3,
+    get_gauge/3,
+    set_gauge/5,
+    shift_gauge/5,
+    get_gauges/2,
+    delete_gauges/2,
     get_rate/2,
+    get_slide/2,
+    get_slide/3,
     get_counters/2,
     create_metrics/3,
     create_metrics/4,
@@ -60,7 +70,16 @@
 -define(SAMPLING, 1).
 -endif.
 
--export_type([metrics/0, handler_name/0, metric_id/0]).
+-export_type([metrics/0, handler_name/0, metric_id/0, metric_spec/0]).
+
+% Default
+-type metric_type() ::
+    %% Simple counter
+    counter
+    %% Sliding window average
+    | slide.
+
+-type metric_spec() :: {metric_type(), atom()}.
 
 -type rate() :: #{
     current := float(),
@@ -68,14 +87,22 @@
     last5m := float()
 }.
 -type metrics() :: #{
-    counters := #{atom() => integer()},
+    counters := #{metric_name() => integer()},
-    rate := #{atom() => rate()}
+    gauges := #{metric_name() => integer()},
+    slides := #{metric_name() => number()},
+    rate := #{metric_name() => rate()}
 }.
 -type handler_name() :: atom().
+%% metric_id() is actually a resource id
 -type metric_id() :: binary() | atom().
+-type metric_name() :: atom().
+-type worker_id() :: term().
 
 -define(CntrRef(Name), {?MODULE, Name}).
 -define(SAMPCOUNT_5M, (?SECS_5M div ?SAMPLING)).
+-define(GAUGE_TABLE(NAME),
+    list_to_atom(atom_to_list(?MODULE) ++ "_" ++ atom_to_list(NAME) ++ "_gauge")
+).
 
 -record(rate, {
     max = 0 :: number(),
@@ -89,9 +116,22 @@
     last5m_smpl = [] :: list()
 }).
 
+-record(slide_datapoint, {
+    sum :: non_neg_integer(),
+    samples :: non_neg_integer(),
+    time :: non_neg_integer()
+}).
+
+-record(slide, {
+    %% Total number of samples through the history
+    n_samples = 0 :: non_neg_integer(),
+    datapoints = [] :: [#slide_datapoint{}]
+}).
+
 -record(state, {
     metric_ids = sets:new(),
-    rates :: undefined | #{metric_id() => #rate{}}
+    rates :: #{metric_id() => #{metric_name() => #rate{}}} | undefined,
+    slides = #{} :: #{metric_id() => #{metric_name() => #slide{}}}
 }).
 
 %%------------------------------------------------------------------------------
@@ -112,13 +152,18 @@ child_spec(ChldName, Name) ->
         modules => [emqx_metrics_worker]
     }.
 
--spec create_metrics(handler_name(), metric_id(), [atom()]) -> ok | {error, term()}.
+-spec create_metrics(handler_name(), metric_id(), [metric_spec() | metric_name()]) ->
+    ok | {error, term()}.
 create_metrics(Name, Id, Metrics) ->
-    create_metrics(Name, Id, Metrics, Metrics).
+    Metrics1 = desugar(Metrics),
+    Counters = filter_counters(Metrics1),
+    create_metrics(Name, Id, Metrics1, Counters).
 
--spec create_metrics(handler_name(), metric_id(), [atom()], [atom()]) -> ok | {error, term()}.
+-spec create_metrics(handler_name(), metric_id(), [metric_spec() | metric_name()], [atom()]) ->
+    ok | {error, term()}.
 create_metrics(Name, Id, Metrics, RateMetrics) ->
-    gen_server:call(Name, {create_metrics, Id, Metrics, RateMetrics}).
+    Metrics1 = desugar(Metrics),
+    gen_server:call(Name, {create_metrics, Id, Metrics1, RateMetrics}).
 
 -spec clear_metrics(handler_name(), metric_id()) -> ok.
 clear_metrics(Name, Id) ->
@@ -135,13 +180,13 @@ has_metrics(Name, Id) ->
         _ -> true
     end.
 
--spec get(handler_name(), metric_id(), atom() | integer()) -> number().
+-spec get(handler_name(), metric_id(), metric_name() | integer()) -> number().
 get(Name, Id, Metric) ->
     case get_ref(Name, Id) of
         not_found ->
             0;
         Ref when is_atom(Metric) ->
-            counters:get(Ref, idx_metric(Name, Id, Metric));
+            counters:get(Ref, idx_metric(Name, Id, counter, Metric));
         Ref when is_integer(Metric) ->
             counters:get(Ref, Metric)
     end.
@@ -156,26 +201,158 @@ get_counters(Name, Id) ->
         fun(_Metric, Index) ->
             get(Name, Id, Index)
         end,
-        get_indexes(Name, Id)
+        get_indexes(Name, counter, Id)
     ).
 
+-spec get_slide(handler_name(), metric_id()) -> map().
+get_slide(Name, Id) ->
+    gen_server:call(Name, {get_slide, Id}).
+
+%% Get the average for a specified sliding window period.
+%%
+%% It will only account for the samples recorded in the past `Window' seconds.
+-spec get_slide(handler_name(), metric_id(), non_neg_integer()) -> number().
+get_slide(Name, Id, Window) ->
+    gen_server:call(Name, {get_slide, Id, Window}).
+
 -spec reset_counters(handler_name(), metric_id()) -> ok.
 reset_counters(Name, Id) ->
-    Indexes = maps:values(get_indexes(Name, Id)),
+    case get_ref(Name, Id) of
-    Ref = get_ref(Name, Id),
+        not_found ->
-    lists:foreach(fun(Idx) -> counters:put(Ref, Idx, 0) end, Indexes).
+            ok;
+        Ref ->
+            #{size := Size} = counters:info(Ref),
+            lists:foreach(fun(Idx) -> counters:put(Ref, Idx, 0) end, lists:seq(1, Size))
+    end.
 
 -spec get_metrics(handler_name(), metric_id()) -> metrics().
 get_metrics(Name, Id) ->
-    #{rate => get_rate(Name, Id), counters => get_counters(Name, Id)}.
+    #{
+        rate => get_rate(Name, Id),
+        counters => get_counters(Name, Id),
+        gauges => get_gauges(Name, Id),
+        slides => get_slide(Name, Id)
+    }.
 
 -spec inc(handler_name(), metric_id(), atom()) -> ok.
 inc(Name, Id, Metric) ->
     inc(Name, Id, Metric, 1).
 
--spec inc(handler_name(), metric_id(), atom(), integer()) -> ok.
+-spec inc(handler_name(), metric_id(), metric_name(), integer()) -> ok.
 inc(Name, Id, Metric, Val) ->
-    counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val).
+    counters:add(get_ref(Name, Id), idx_metric(Name, Id, counter, Metric), Val).
+
+%% Add a sample to the slide.
+%%
+%% Slide is short for "sliding window average" type of metric.
+%%
+%% It allows to monitor an average of some observed values in time,
+%% and it's mainly used for performance analysis. For example, it can
+%% be used to report run time of operations.
+%%
+%% Consider an example:
+%%
+%% ```
+%% emqx_metrics_worker:create_metrics(Name, Id, [{slide, a}]),
+%% emqx_metrics_worker:observe(Name, Id, a, 10),
+%% emqx_metrics_worker:observe(Name, Id, a, 30),
+%% #{a := 20} = emqx_metrics_worker:get_slide(Name, Id, _Window = 1).
+%% '''
+%%
+%% After recording 2 samples, this metric becomes 20 (the average of 10 and 30).
+%%
+%% But after 1 second it becomes 0 again, unless new samples are recorded.
+%%
+-spec observe(handler_name(), metric_id(), atom(), integer()) -> ok.
+observe(Name, Id, Metric, Val) ->
+    #{ref := CRef, slide := Idx} = maps:get(Id, get_pterm(Name)),
+    Index = maps:get(Metric, Idx),
+    %% Update sum:
+    counters:add(CRef, Index, Val),
+    %% Update number of samples:
+    counters:add(CRef, Index + 1, 1).
+
+-spec set_gauge(handler_name(), metric_id(), worker_id(), metric_name(), integer()) -> ok.
+set_gauge(Name, Id, WorkerId, Metric, Val) ->
+    Table = ?GAUGE_TABLE(Name),
+    try
+        true = ets:insert(Table, {{Id, Metric, WorkerId}, Val}),
+        ok
+    catch
+        error:badarg ->
+            ok
+    end.
+
+-spec shift_gauge(handler_name(), metric_id(), worker_id(), metric_name(), integer()) -> ok.
+shift_gauge(Name, Id, WorkerId, Metric, Val) ->
+    Table = ?GAUGE_TABLE(Name),
+    try
+        _ = ets:update_counter(
+            Table,
+            {Id, Metric, WorkerId},
+            Val,
+            {{Id, Metric, WorkerId}, 0}
+        ),
+        ok
+    catch
+        error:badarg ->
+            ok
+    end.
+
+-spec get_gauge(handler_name(), metric_id(), metric_name()) -> integer().
+get_gauge(Name, Id, Metric) ->
+    Table = ?GAUGE_TABLE(Name),
+    MatchSpec =
+        ets:fun2ms(
+            fun({{Id0, Metric0, _WorkerId}, Val}) when Id0 =:= Id, Metric0 =:= Metric ->
+                Val
+            end
+        ),
+    try
+        lists:sum(ets:select(Table, MatchSpec))
+    catch
+        error:badarg ->
+            0
+    end.
+
+-spec get_gauges(handler_name(), metric_id()) -> map().
+get_gauges(Name, Id) ->
+    Table = ?GAUGE_TABLE(Name),
+    MatchSpec =
+        ets:fun2ms(
+            fun({{Id0, Metric, _WorkerId}, Val}) when Id0 =:= Id ->
+                {Metric, Val}
+            end
+        ),
+    try
+        lists:foldr(
+            fun({Metric, Val}, Acc) ->
+                maps:update_with(Metric, fun(X) -> X + Val end, Val, Acc)
+            end,
+            #{},
+            ets:select(Table, MatchSpec)
+        )
+    catch
+        error:badarg ->
+            #{}
+    end.
+
+-spec delete_gauges(handler_name(), metric_id()) -> ok.
+delete_gauges(Name, Id) ->
+    Table = ?GAUGE_TABLE(Name),
+    MatchSpec =
+        ets:fun2ms(
+            fun({{Id0, _Metric, _WorkerId}, _Val}) when Id0 =:= Id ->
+                true
+            end
+        ),
+    try
+        _ = ets:select_delete(Table, MatchSpec),
+        ok
+    catch
+        error:badarg ->
+            ok
+    end.
 
 start_link(Name) ->
     gen_server:start_link({local, Name}, ?MODULE, Name, []).
@@ -185,6 +362,7 @@ init(Name) ->
     %% the rate metrics
     erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
     persistent_term:put(?CntrRef(Name), #{}),
+    _ = ets:new(?GAUGE_TABLE(Name), [named_table, ordered_set, public, {write_concurrency, true}]),
     {ok, #state{}}.
 
 handle_call({get_rate, _Id}, _From, State = #state{rates = undefined}) ->
@@ -198,9 +376,9 @@ handle_call({get_rate, Id}, _From, State = #state{rates = Rates}) ->
 handle_call(
     {create_metrics, Id, Metrics, RateMetrics},
     _From,
-    State = #state{metric_ids = MIDs, rates = Rates}
+    State = #state{metric_ids = MIDs, rates = Rates, slides = Slides}
 ) ->
-    case RateMetrics -- Metrics of
+    case RateMetrics -- filter_counters(Metrics) of
         [] ->
             RatePerId = maps:from_list([{M, #rate{}} || M <- RateMetrics]),
             Rate1 =
@@ -208,9 +386,11 @@ handle_call(
                     undefined -> #{Id => RatePerId};
                     _ -> Rates#{Id => RatePerId}
                 end,
+            Slides1 = Slides#{Id => create_slides(Metrics)},
             {reply, create_counters(get_self_name(), Id, Metrics), State#state{
                 metric_ids = sets:add_element(Id, MIDs),
-                rates = Rate1
+                rates = Rate1,
+                slides = Slides1
             }};
         _ ->
             {reply, {error, not_super_set_of, {RateMetrics, Metrics}}, State}
@@ -218,35 +398,54 @@ handle_call(
 handle_call(
     {delete_metrics, Id},
     _From,
-    State = #state{metric_ids = MIDs, rates = Rates}
+    State = #state{metric_ids = MIDs, rates = Rates, slides = Slides}
 ) ->
-    {reply, delete_counters(get_self_name(), Id), State#state{
+    Name = get_self_name(),
+    delete_counters(Name, Id),
+    delete_gauges(Name, Id),
+    {reply, ok, State#state{
         metric_ids = sets:del_element(Id, MIDs),
         rates =
             case Rates of
                 undefined -> undefined;
                 _ -> maps:remove(Id, Rates)
-            end
+            end,
+        slides = maps:remove(Id, Slides)
     }};
 handle_call(
     {reset_metrics, Id},
     _From,
-    State = #state{rates = Rates}
+    State = #state{rates = Rates, slides = Slides}
 ) ->
+    delete_gauges(get_self_name(), Id),
+    NewRates =
+        case Rates of
+            undefined ->
+                undefined;
+            _ ->
+                ResetRate =
+                    maps:map(
+                        fun(_Key, _Value) -> #rate{} end,
+                        maps:get(Id, Rates, #{})
+                    ),
+                maps:put(Id, ResetRate, Rates)
+        end,
+    SlideSpecs = [{slide, I} || I <- maps:keys(maps:get(Id, Slides, #{}))],
+    NewSlides = Slides#{Id => create_slides(SlideSpecs)},
     {reply, reset_counters(get_self_name(), Id), State#state{
         rates =
-            case Rates of
+            NewRates,
-                undefined ->
+        slides = NewSlides
-                    undefined;
-                _ ->
-                    ResetRate =
-                        maps:map(
-                            fun(_Key, _Value) -> #rate{} end,
-                            maps:get(Id, Rates, #{})
-                        ),
-                    maps:put(Id, ResetRate, Rates)
-            end
     }};
+handle_call({get_slide, Id}, _From, State = #state{slides = Slides}) ->
+    SlidesForID = maps:get(Id, Slides, #{}),
+    {reply, maps:map(fun(Metric, Slide) -> do_get_slide(Id, Metric, Slide) end, SlidesForID),
+        State};
+handle_call({get_slide, Id, Window}, _From, State = #state{slides = Slides}) ->
+    SlidesForID = maps:get(Id, Slides, #{}),
+    {reply,
+        maps:map(fun(Metric, Slide) -> do_get_slide(Window, Id, Metric, Slide) end, SlidesForID),
+        State};
 handle_call(_Request, _From, State) ->
     {reply, ok, State}.
 
@@ -256,7 +455,7 @@ handle_cast(_Msg, State) ->
 handle_info(ticking, State = #state{rates = undefined}) ->
     erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
     {noreply, State};
-handle_info(ticking, State = #state{rates = Rates0}) ->
+handle_info(ticking, State = #state{rates = Rates0, slides = Slides0}) ->
     Rates =
         maps:map(
             fun(Id, RatesPerID) ->
@@ -269,8 +468,20 @@ handle_info(ticking, State = #state{rates = Rates0}) ->
             end,
             Rates0
         ),
+    Slides =
+        maps:map(
+            fun(Id, SlidesPerID) ->
+                maps:map(
+                    fun(Metric, Slide) ->
+                        update_slide(Id, Metric, Slide)
+                    end,
+                    SlidesPerID
+                )
+            end,
+            Slides0
+        ),
     erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
-    {noreply, State#state{rates = Rates}};
+    {noreply, State#state{rates = Rates, slides = Slides}};
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -301,17 +512,18 @@ create_counters(_Name, _Id, []) ->
     error({create_counter_error, must_provide_a_list_of_metrics});
 create_counters(Name, Id, Metrics) ->
     %% backup the old counters
-    OlderCounters = maps:with(Metrics, get_counters(Name, Id)),
+    OlderCounters = maps:with(filter_counters(Metrics), get_counters(Name, Id)),
     %% create the new counter
-    Size = length(Metrics),
+    {Size, Indexes} = create_metric_indexes(Metrics),
-    Indexes = maps:from_list(lists:zip(Metrics, lists:seq(1, Size))),
     Counters = get_pterm(Name),
     CntrRef = counters:new(Size, [write_concurrency]),
     persistent_term:put(
         ?CntrRef(Name),
-        Counters#{Id => #{ref => CntrRef, indexes => Indexes}}
+        Counters#{Id => Indexes#{ref => CntrRef}}
     ),
-    %% restore the old counters
+    %% Restore the old counters. Slides are not restored, since they
+    %% are periodically zeroed anyway. We do lose some samples in the
+    %% current interval, but that's acceptable for now.
     lists:foreach(
         fun({Metric, N}) ->
             inc(Name, Id, Metric, N)
@@ -319,6 +531,16 @@ create_counters(Name, Id, Metrics) ->
         maps:to_list(OlderCounters)
     ).
 
+create_metric_indexes(Metrics) ->
+    create_metric_indexes(Metrics, 1, [], []).
+
+create_metric_indexes([], Size, Counters, Slides) ->
+    {Size, #{counter => maps:from_list(Counters), slide => maps:from_list(Slides)}};
+create_metric_indexes([{counter, Id} | Rest], Index, Counters, Slides) ->
+    create_metric_indexes(Rest, Index + 1, [{Id, Index} | Counters], Slides);
+create_metric_indexes([{slide, Id} | Rest], Index, Counters, Slides) ->
+    create_metric_indexes(Rest, Index + 2, Counters, [{Id, Index} | Slides]).
+
 delete_counters(Name, Id) ->
     persistent_term:put(?CntrRef(Name), maps:remove(Id, get_pterm(Name))).
 
@@ -328,12 +550,12 @@ get_ref(Name, Id) ->
         error -> not_found
     end.
 
-idx_metric(Name, Id, Metric) ->
+idx_metric(Name, Id, Type, Metric) ->
-    maps:get(Metric, get_indexes(Name, Id)).
+    maps:get(Metric, get_indexes(Name, Type, Id)).
 
-get_indexes(Name, Id) ->
+get_indexes(Name, Type, Id) ->
     case maps:find(Id, get_pterm(Name)) of
-        {ok, #{indexes := Indexes}} -> Indexes;
+        {ok, #{Type := Indexes}} -> Indexes;
         error -> #{}
     end.
 
@@ -381,6 +603,53 @@ calculate_rate(CurrVal, #rate{
         tick = Tick + 1
     }.
 
+do_get_slide(Id, Metric, S = #slide{n_samples = NSamples}) ->
+    #{
+        n_samples => NSamples,
+        current => do_get_slide(2, Id, Metric, S),
+        last5m => do_get_slide(?SECS_5M, Id, Metric, S)
+    }.
+
+do_get_slide(Window, Id, Metric, #slide{datapoints = DP0}) ->
+    Datapoint = get_slide_datapoint(Id, Metric),
+    {N, Sum} = get_slide_window(os:system_time(second) - Window, [Datapoint | DP0], 0, 0),
+    case N > 0 of
+        true -> Sum div N;
+        false -> 0
+    end.
+
+get_slide_window(_StartTime, [], N, S) ->
+    {N, S};
+get_slide_window(StartTime, [#slide_datapoint{time = T} | _], N, S) when T < StartTime ->
+    {N, S};
+get_slide_window(StartTime, [#slide_datapoint{samples = N, sum = S} | Rest], AccN, AccS) ->
+    get_slide_window(StartTime, Rest, AccN + N, AccS + S).
+
+get_slide_datapoint(Id, Metric) ->
+    Name = get_self_name(),
+    CRef = get_ref(Name, Id),
+    Index = idx_metric(Name, Id, slide, Metric),
+    Total = counters:get(CRef, Index),
+    N = counters:get(CRef, Index + 1),
+    #slide_datapoint{
+        sum = Total,
+        samples = N,
+        time = os:system_time(second)
+    }.
+
+update_slide(Id, Metric, Slide0 = #slide{n_samples = NSamples, datapoints = DPs}) ->
+    Datapoint = get_slide_datapoint(Id, Metric),
+    %% Reset counters:
+    Name = get_self_name(),
+    CRef = get_ref(Name, Id),
+    Index = idx_metric(Name, Id, slide, Metric),
+    counters:put(CRef, Index, 0),
+    counters:put(CRef, Index + 1, 0),
+    Slide0#slide{
+        datapoints = [Datapoint | lists:droplast(DPs)],
+        n_samples = Datapoint#slide_datapoint.samples + NSamples
+    }.
+
 format_rates_of_id(RatesPerId) ->
     maps:map(
         fun(_Metric, Rates) ->
@@ -403,6 +672,27 @@ precision(Float, N) ->
     Base = math:pow(10, N),
     round(Float * Base) / Base.
 
+desugar(Metrics) ->
+    lists:map(
+        fun
+            (Atom) when is_atom(Atom) ->
+                {counter, Atom};
+            (Spec = {_, _}) ->
+                Spec
+        end,
+        Metrics
+    ).
+
+filter_counters(Metrics) ->
+    [K || {counter, K} <- Metrics].
+
+create_slides(Metrics) ->
+    EmptyDatapoints = [
+        #slide_datapoint{sum = 0, samples = 0, time = 0}
+     || _ <- lists:seq(1, ?SECS_5M div ?SAMPLING)
+    ],
+    maps:from_list([{K, #slide{datapoints = EmptyDatapoints}} || {slide, K} <- Metrics]).
+
 get_self_name() ->
     {registered_name, Name} = process_info(self(), registered_name),
     Name.

apps/emqx/src/emqx_misc.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -68,7 +68,7 @@
     nolink_apply/2
 ]).
 
--export([clamp/3]).
+-export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
 
 -dialyzer({nowarn_function, [nolink_apply/2]}).
 
@@ -556,6 +556,75 @@ try_to_existing_atom(Convert, Data, Encoding) ->
         _:Reason -> {error, Reason}
     end.
 
+is_sensitive_key(token) -> true;
+is_sensitive_key("token") -> true;
+is_sensitive_key(<<"token">>) -> true;
+is_sensitive_key(password) -> true;
+is_sensitive_key("password") -> true;
+is_sensitive_key(<<"password">>) -> true;
+is_sensitive_key(secret) -> true;
+is_sensitive_key("secret") -> true;
+is_sensitive_key(<<"secret">>) -> true;
+is_sensitive_key(_) -> false.
+
+redact(Term) ->
+    do_redact(Term, fun is_sensitive_key/1).
+
+redact(Term, Checker) ->
+    do_redact(Term, fun(V) ->
+        is_sensitive_key(V) orelse Checker(V)
+    end).
+
+do_redact(L, Checker) when is_list(L) ->
+    lists:map(fun(E) -> do_redact(E, Checker) end, L);
+do_redact(M, Checker) when is_map(M) ->
+    maps:map(
+        fun(K, V) ->
+            do_redact(K, V, Checker)
+        end,
+        M
+    );
+do_redact({Key, Value}, Checker) ->
+    case Checker(Key) of
+        true ->
+            {Key, redact_v(Value)};
+        false ->
+            {do_redact(Key, Checker), do_redact(Value, Checker)}
+    end;
+do_redact(T, Checker) when is_tuple(T) ->
+    Elements = erlang:tuple_to_list(T),
+    Redact = do_redact(Elements, Checker),
+    erlang:list_to_tuple(Redact);
+do_redact(Any, _Checker) ->
+    Any.
+
+do_redact(K, V, Checker) ->
+    case Checker(K) of
+        true ->
+            redact_v(V);
+        false ->
+            do_redact(V, Checker)
+    end.
+
+-define(REDACT_VAL, "******").
+redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
+redact_v(_V) -> ?REDACT_VAL.
+
+is_redacted(K, V) ->
+    do_is_redacted(K, V, fun is_sensitive_key/1).
+
+is_redacted(K, V, Fun) ->
+    do_is_redacted(K, V, fun(E) ->
+        is_sensitive_key(E) orelse Fun(E)
+    end).
+
+do_is_redacted(K, ?REDACT_VAL, Fun) ->
+    Fun(K);
+do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
+    Fun(K);
+do_is_redacted(_K, _V, _Fun) ->
+    false.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
@@ -568,6 +637,62 @@ ipv6_probe_test() ->
             ok
     end.
 
+redact_test_() ->
+    Case = fun(Type, KeyT) ->
+        Key =
+            case Type of
+                atom -> KeyT;
+                string -> erlang:atom_to_list(KeyT);
+                binary -> erlang:atom_to_binary(KeyT)
+            end,
+
+        ?assert(is_sensitive_key(Key)),
+
+        %% direct
+        ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
+        ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
+        ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
+        ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
+
+        %% 1 level nested
+        ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
+        ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
+
+        %% 2 level nested
+        ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
+        ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
+        ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
+
+        %% 3 level nested
+        ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
+        ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
+        ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
+    end,
+
+    Types = [atom, string, binary],
+    Keys = [
+        token,
+        password,
+        secret
+    ],
+    [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
+
+redact2_test_() ->
+    Case = fun(Key, Checker) ->
+        ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
+        ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
+        ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
+        ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
+    end,
+
+    Checker = fun(E) -> E =:= passcode end,
+
+    Keys = [secret, passcode],
+    [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
+
+case_name(Type, Key) ->
+    lists:concat([Type, "-", Key]).
+
 -endif.
 
 pub_props_to_packet(Properties) ->

apps/emqx/src/emqx_mountpoint.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_mqtt_caps.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_mqtt_props.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx/src/emqx_mqueue.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.