Merge branch 'master' into release-50

.github/actions/docker-meta/action.yaml

@@ -1,81 +0,0 @@
-name: 'Docker meta'
-inputs:
-  profile:
-    required: true
-    type: string
-  registry:
-    required: true
-    type: string
-  arch:
-    required: true
-    type: string
-  otp:
-    required: true
-    type: string
-  elixir:
-    required: false
-    type: string
-    default: ''
-  builder_base:
-    required: true
-    type: string
-  owner:
-    required: true
-    type: string
-  docker_tags:
-    required: true
-    type: string
-
-outputs:
-  emqx_name:
-    description: "EMQX name"
-    value: ${{ steps.pre-meta.outputs.emqx_name }}
-  version:
-    description: "docker image version"
-    value: ${{ steps.meta.outputs.version }}
-  tags:
-    description: "docker image tags"
-    value: ${{ steps.meta.outputs.tags }}
-  labels:
-    description: "docker image labels"
-    value: ${{ steps.meta.outputs.labels }}
-
-runs:
-  using: composite
-  steps:
-    - name: prepare for docker/metadata-action
-      id: pre-meta
-      shell: bash
-      run: |
-        emqx_name=${{ inputs.profile }}
-        img_suffix=${{ inputs.arch }}
-        img_labels="org.opencontainers.image.otp.version=${{ inputs.otp }}"
-        if [ -n "${{ inputs.elixir }}" ]; then
-          emqx_name="emqx-elixir"
-          img_suffix="elixir-${{ inputs.arch }}"
-          img_labels="org.opencontainers.image.elixir.version=${{ inputs.elixir }}\n${img_labels}"
-        fi
-        if [ "${{ inputs.profile }}" = "emqx" ]; then
-          img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}"
-        fi
-        if [ "${{ inputs.profile }}" = "emqx-enterprise" ]; then
-          img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}"
-        fi
-        if [[ "${{ inputs.builder_base }}" =~ "alpine" ]]; then
-          img_suffix="${img_suffix}-alpine"
-        fi
-        echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT
-        echo "img_suffix=${img_suffix}" >> $GITHUB_OUTPUT
-        echo "img_labels=${img_labels}" >> $GITHUB_OUTPUT
-        echo "img_name=${{ inputs.registry }}/${{ inputs.owner }}/${{ inputs.profile }}" >> $GITHUB_OUTPUT
-    - uses: docker/metadata-action@v4
-      id: meta
-      with:
-        images:
-          ${{ steps.pre-meta.outputs.img_name }}
-        flavor: |
-          suffix=-${{ steps.pre-meta.outputs.img_suffix }}
-        tags: |
-          type=raw,value=${{ inputs.docker_tags }}
-        labels:
-          ${{ steps.pre-meta.outputs.img_labels }}

.github/workflows/build_and_push_docker_images.yaml

@@ -9,15 +9,17 @@ on:
     tags:
     - v*
     - e*
-  release:
-    types:
-      - published
+    - docker-latest-*
   workflow_dispatch:
     inputs:
       branch_or_tag:
         required: false
       profile:
         required: false
+        default: 'emqx'
+      is_latest:
+        required: false
+        default: false
 
 jobs:
   prepare:
@@ -26,10 +28,11 @@ jobs:
     container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
 
     outputs:
-      BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
-      IS_DOCKER_LATEST: ${{ steps.get_profile.outputs.IS_DOCKER_LATEST }}
+      PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
+      EDITION: ${{ steps.get_profile.outputs.EDITION }}
+      IS_LATEST: ${{ steps.get_profile.outputs.IS_LATEST }}
       IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
-      DOCKER_TAG_VERSION: ${{ steps.get_profile.outputs.DOCKER_TAG_VERSION }}
+      VERSION: ${{ steps.get_profile.outputs.VERSION }}
 
     steps:
       - uses: actions/checkout@v3
@@ -45,14 +48,14 @@ jobs:
           tag=${{ github.ref }}
           # tag docker-latest-ce or docker-latest-ee
           if git describe --tags --exact --match 'docker-latest-*' 2>/dev/null; then
-            echo 'docker_latest=true due to docker-latest-* tag'
-            docker_latest=true
-          elif [ "${{ github.event_name }}" = "release" ]; then
-            echo 'docker_latest=true due to release'
-            docker_latest=true
+            echo 'is_latest=true due to docker-latest-* tag'
+            is_latest=true
+          elif [ "${{ inputs.is_latest }}" = "true" ]; then
+            echo 'is_latest=true due to manual input from workflow_dispatch'
+            is_latest=true
           else
-            echo 'docker_latest=false'
-            docker_latest=false
+            echo 'is_latest=false'
+            is_latest=false
           fi
           if git describe --tags --match "[v|e]*" --exact; then
             echo "This is an exact git tag, will publish images"
@@ -64,18 +67,20 @@ jobs:
           case $tag in
             refs/tags/v*)
               PROFILE='emqx'
+              EDITION='Opensource'
               ;;
             refs/tags/e*)
               PROFILE=emqx-enterprise
+              EDITION='Enterprise'
               ;;
             *)
               PROFILE=${{ github.event.inputs.profile }}
               case "$PROFILE" in
                 emqx)
-                  true
+                  EDITION='Opensource'
                   ;;
                 emqx-enterprise)
-                  true
+                  EDITION='Enterprise'
                   ;;
                 *)
                   echo "ERROR: Failed to resolve build profile"
@@ -85,14 +90,18 @@ jobs:
               ;;
           esac
           VSN="$(./pkg-vsn.sh "$PROFILE")"
-          echo "Building $PROFILE image with tag $VSN (latest=$docker_latest)"
-          echo "IS_DOCKER_LATEST=$docker_latest" >> $GITHUB_OUTPUT
+          echo "Building emqx/$PROFILE:$VSN image (latest=$is_latest)"
+          echo "Push = $is_exact"
+          echo "IS_LATEST=$is_latest" >> $GITHUB_OUTPUT
           echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT
-          echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT
-          echo "DOCKER_TAG_VERSION=$VSN" >> $GITHUB_OUTPUT
+          echo "PROFILE=$PROFILE" >> $GITHUB_OUTPUT
+          echo "EDITION=$EDITION" >> $GITHUB_OUTPUT
+          echo "VERSION=$VSN" >> $GITHUB_OUTPUT
       - name: get_all_deps
+        env:
+          PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
         run: |
-          make -C source deps-all
+          PROFILE=$PROFILE make -C source deps-$PROFILE
           zip -ryq source.zip source/* source/.[^.]*
       - uses: actions/upload-artifact@v3
         with:
@@ -100,17 +109,17 @@ jobs:
           path: source.zip
 
   docker:
-    runs-on: ${{ matrix.arch[1] }}
+    runs-on: ubuntu-20.04
     needs: prepare
 
     strategy:
       fail-fast: false
       matrix:
-        arch:
-          - [amd64, ubuntu-20.04]
-          - [arm64, aws-arm64]
         profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
+          - "${{ needs.prepare.outputs.PROFILE }}"
+        flavor:
+          - ''
+          - '-elixir'
         registry:
           - 'docker.io'
           - 'public.ecr.aws'
@@ -128,9 +137,10 @@ jobs:
         exclude: # TODO: publish enterprise to ecr too?
           - registry: 'public.ecr.aws'
             profile: emqx-enterprise
+          - flavor: '-elixir'
+            os: [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
+
     steps:
-    - uses: AutoModality/action-clean@v1
-      if: matrix.arch[1] == 'aws-arm64'
     - uses: actions/download-artifact@v3
       with:
         name: source
@@ -138,16 +148,17 @@ jobs:
     - name: unzip source code
       run: unzip -q source.zip
 
+    - uses: docker/setup-qemu-action@v2
     - uses: docker/setup-buildx-action@v2
 
-    - name: Login for docker.
+    - name: Login to hub.docker.com
       uses: docker/login-action@v2
       if: matrix.registry == 'docker.io'
       with:
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
-    - name: Login for AWS ECR
+    - name: Login to AWS ECR
       uses: docker/login-action@v2
       if: matrix.registry == 'public.ecr.aws'
       with:
@@ -155,230 +166,48 @@ jobs:
         username: ${{ secrets.AWS_ACCESS_KEY_ID }}
         password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         ecr: true
+    - name: prepare for docker/metadata-action
+      id: pre-meta
+      shell: bash
+      run: |
+        extra_labels=
+        img_suffix=
+        flavor="${{ matrix.flavor }}"
+        if [ "${{ matrix.flavor }}" = '-elixir' ]; then
+          img_suffix="-elixir"
+          extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}"
+        fi
+        if [[ "${{ matrix.os[0] }}" =~ "alpine" ]]; then
+          img_suffix="${img_suffix}-alpine"
+        fi
 
-    - uses: ./source/.github/actions/docker-meta
+        echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT
+        echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT
+
+    - uses: docker/metadata-action@v4
       id: meta
       with:
-        profile: ${{ matrix.profile }}
-        registry: ${{ matrix.registry }}
-        arch: ${{ matrix.arch[0] }}
-        otp: ${{ matrix.otp }}
-        builder_base: ${{ matrix.os[0] }}
-        owner: ${{ github.repository_owner }}
-        docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
+        images: |
+          ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
+        flavor: |
+          suffix=${{ steps.pre-meta.outputs.img_suffix }}
+        tags: |
+          type=raw,value=${{ needs.prepare.outputs.VERSION }}
+          type=raw,value=latest,enable=${{ needs.prepare.outputs.IS_LATEST }}
+        labels: |
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+          org.opencontainers.image.edition=${{ needs.prepare.outputs.EDITION }}
+          ${{ steps.pre-meta.outputs.extra_labels }}
 
     - uses: docker/build-push-action@v3
       with:
         push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
         pull: true
         no-cache: true
-        platforms: linux/${{ matrix.arch[0] }}
+        platforms: linux/amd64,linux/arm64
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
-          RUN_FROM=${{ matrix.os[1] }}
-          EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
+          EMQX_NAME=${{ matrix.profile }}${{ matrix.flavor }}
         file: source/${{ matrix.os[2] }}
         context: source
-
-    - name: Docker Hub Description
-      if: matrix.registry == 'docker.io'
-      uses: peter-evans/dockerhub-description@v3
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_TOKEN }}
-        repository: "emqx/${{ needs.prepare.outputs.BUILD_PROFILE }}"
-        readme-filepath: ./source/deploy/docker/README.md
-        short-description: "The most scalable open-source MQTT broker for IoT, IIoT, connected vehicles, and more."
-
-  docker-elixir:
-    runs-on: ${{ matrix.arch[1] }}
-    needs: prepare
-    # do not build elixir images for ee for now
-    if: needs.prepare.outputs.BUILD_PROFILE == 'emqx'
-
-    strategy:
-      fail-fast: false
-      matrix:
-        arch:
-          - [amd64, ubuntu-20.04]
-          - [arm64, aws-arm64]
-        profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-        registry:
-          - 'docker.io'
-        os:
-          - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
-        builder:
-          - 5.0-26 # update to latest
-        otp:
-          - 25.1.2-2 # update to latest
-        elixir:
-          - 1.13.4 # update to latest
-
-    steps:
-    - uses: AutoModality/action-clean@v1
-      if: matrix.arch[1] == 'aws-arm64'
-    - uses: actions/download-artifact@v3
-      with:
-        name: source
-        path: .
-    - name: unzip source code
-      run: unzip -q source.zip
-
-    - uses: docker/setup-buildx-action@v2
-
-    - name: Login for docker.
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
-    - uses: ./source/.github/actions/docker-meta
-      id: meta
-      with:
-        profile: ${{ matrix.profile }}
-        registry: ${{ matrix.registry }}
-        arch: ${{ matrix.arch[0] }}
-        otp: ${{ matrix.otp }}
-        elixir: ${{ matrix.elixir }}
-        builder_base: ${{ matrix.os[0] }}
-        owner: ${{ github.repository_owner }}
-        docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
-    - uses: docker/build-push-action@v3
-      with:
-        push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
-        pull: true
-        no-cache: true
-        platforms: linux/${{ matrix.arch[0] }}
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-        build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
-          RUN_FROM=${{ matrix.os[1] }}
-          EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
-        file: source/${{ matrix.os[2] }}
-        context: source
-
-  docker-push-multi-arch-manifest:
-    # note, we only run on amd64
-    if: needs.prepare.outputs.IS_EXACT_TAG
-    needs:
-      - prepare
-      - docker
-    runs-on: ${{ matrix.arch[1] }}
-    strategy:
-      fail-fast: false
-      matrix:
-        arch:
-          - [amd64, ubuntu-20.04]
-        profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-        os:
-          - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
-          - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
-        # NOTE: only support latest otp version, not a matrix
-        otp:
-          - 24.3.4.2-1 # switch to 25 once ready to release 5.1
-        registry:
-          - 'docker.io'
-          - 'public.ecr.aws'
-        exclude:
-          - registry: 'public.ecr.aws'
-            profile: emqx-enterprise
-
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: source
-          path: .
-
-      - name: unzip source code
-        run: unzip -q source.zip
-
-      - uses: docker/login-action@v2
-        if: matrix.registry == 'docker.io'
-        with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
-      - uses: docker/login-action@v2
-        if: matrix.registry == 'public.ecr.aws'
-        with:
-          registry: public.ecr.aws
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          ecr: true
-
-      - uses: ./source/.github/actions/docker-meta
-        id: meta
-        with:
-          profile: ${{ matrix.profile }}
-          registry: ${{ matrix.registry }}
-          arch: ${{ matrix.arch[0] }}
-          otp: ${{ matrix.otp }}
-          builder_base: ${{ matrix.os[0] }}
-          owner: ${{ github.repository_owner }}
-          docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
-      - name: update manifest for multiarch image
-        working-directory: source
-        run: |
-          is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}"
-          scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$is_latest"
-
-  docker-elixir-push-multi-arch-manifest:
-    # note, we only run on amd64
-    # do not build enterprise elixir images for now
-    if: needs.prepare.outputs.IS_EXACT_TAG == 'true' && needs.prepare.outputs.BUILD_PROFILE == 'emqx'
-    needs:
-      - prepare
-      - docker-elixir
-    runs-on: ${{ matrix.arch[1] }}
-    strategy:
-      fail-fast: false
-      matrix:
-        arch:
-          - [amd64, ubuntu-20.04]
-        profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-        # NOTE: for docker, only support latest otp version, not a matrix
-        otp:
-          - 25.1.2-2 # update to latest
-        elixir:
-          - 1.13.4 # update to latest
-        registry:
-          - 'docker.io'
-
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: source
-          path: .
-
-      - name: unzip source code
-        run: unzip -q source.zip
-
-      - uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
-      - uses: ./source/.github/actions/docker-meta
-        id: meta
-        with:
-          profile: ${{ matrix.profile }}
-          registry: ${{ matrix.registry }}
-          arch: ${{ matrix.arch[0] }}
-          otp: ${{ matrix.otp }}
-          elixir: ${{ matrix.elixir }}
-          builder_base: ${{ matrix.os[0] }}
-          owner: ${{ github.repository_owner }}
-          docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
-      - name: update manifest for multiarch image
-        working-directory: source
-        run: |
-          scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" false

.github/workflows/run_fvt_tests.yaml

@@ -201,12 +201,25 @@ jobs:
           echo "waiting emqx started";
           sleep 10;
         done
+    - name: Get Token
+      timeout-minutes: 1
+      run: |
+        kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &
+
+        while
+          [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ]
+        do
+          echo "waiting emqx"
+          sleep 1
+        done
+
+        echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
+
     - name: Check cluster
       timeout-minutes: 10
       run: |
-        kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &
         while
-          [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
+          [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
         do
           echo "waiting ${{ matrix.profile }} cluster scale"
           sleep 1

.github/workflows/run_jmeter_tests.yaml

@@ -92,7 +92,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         repository: emqx/emqx-fvt
-        ref: broker-autotest-v2
+        ref: broker-autotest-v4
         path: scripts
     - uses: actions/setup-java@v3
       with:
@@ -191,7 +191,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         repository: emqx/emqx-fvt
-        ref: broker-autotest-v2
+        ref: broker-autotest-v4
         path: scripts
     - uses: actions/setup-java@v3
       with:
@@ -297,7 +297,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         repository: emqx/emqx-fvt
-        ref: broker-autotest-v2
+        ref: broker-autotest-v4
         path: scripts
     - uses: actions/setup-java@v3
       with:
@@ -396,7 +396,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         repository: emqx/emqx-fvt
-        ref: broker-autotest-v2
+        ref: broker-autotest-v4
         path: scripts
     - name: run jwks_server
       timeout-minutes: 10
@@ -496,7 +496,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         repository: emqx/emqx-fvt
-        ref: broker-autotest-v2
+        ref: broker-autotest-v4
         path: scripts
     - uses: actions/setup-java@v3
       with:

Makefile

@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.1.5
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.9
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.12
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

apps/emqx/i18n/emqx_schema_i18n.conf

@@ -2050,7 +2050,7 @@ base_listener_enable_authn {
 Set <code>true</code> (default) to enable client authentication on this listener, the authentication
 process goes through the configured authentication chain.
 When set to <code>false</code> to allow any clients with or without authentication information such as username or password to log in.
-When set to <code>quick_deny_anonymous<code>, it behaves like when set to <code>true</code> but clients will be
+When set to <code>quick_deny_anonymous</code>, it behaves like when set to <code>true</code>, but clients will be
 denied immediately without going through any authenticators if <code>username</code> is not provided. This is useful to fence off
 anonymous clients early.
 """

apps/emqx/include/http_api.hrl

@@ -15,10 +15,8 @@
 %%--------------------------------------------------------------------
 
 %% HTTP API Auth
--define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD').
--define(WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET,
-    'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET'
-).
+-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
+-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
 
 %% Bad Request
 -define(BAD_REQUEST, 'BAD_REQUEST').
@@ -57,8 +55,8 @@
 
 %% All codes
 -define(ERROR_CODES, [
-    {'WRONG_USERNAME_OR_PWD', <<"Wrong username or pwd">>},
-    {'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET', <<"Wrong username & pwd or key & secret">>},
+    {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>},
+    {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>},
     {'BAD_REQUEST', <<"Request parameters are not legal">>},
     {'NOT_MATCH', <<"Conditions are not matched">>},
     {'ALREADY_EXISTS', <<"Resource already existed">>},

apps/emqx/src/emqx_hocon.erl

@@ -21,7 +21,8 @@
     format_path/1,
     check/2,
     format_error/1,
-    format_error/2
+    format_error/2,
+    make_schema/1
 ]).
 
 %% @doc Format hocon config field path to dot-separated string in iolist format.
@@ -79,6 +80,9 @@ format_error({_Schema, [#{kind := K} = First | Rest] = All}, Opts) when
 format_error(_Other, _) ->
     false.
 
+make_schema(Fields) ->
+    #{roots => Fields, fields => #{}}.
+
 %% Ensure iolist()
 iol(B) when is_binary(B) -> B;
 iol(A) when is_atom(A) -> atom_to_binary(A, utf8);

apps/emqx/src/emqx_schema.erl

@@ -111,15 +111,19 @@
     comma_separated_atoms/0
 ]).
 
--export([namespace/0, roots/0, roots/1, fields/1, desc/1]).
+-export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 -export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]).
+-export([authz_fields/0]).
 -export([sc/2, map/2]).
 
 -elvis([{elvis_style, god_modules, disable}]).
 
 namespace() -> broker.
 
+tags() ->
+    [<<"EMQX">>].
+
 roots() ->
     %% TODO change config importance to a field metadata
     roots(high) ++ roots(medium) ++ roots(low).
@@ -323,31 +327,7 @@ fields("stats") ->
             )}
     ];
 fields("authorization") ->
-    [
-        {"no_match",
-            sc(
-                hoconsc:enum([allow, deny]),
-                #{
-                    default => allow,
-                    required => true,
-                    desc => ?DESC(fields_authorization_no_match)
-                }
-            )},
-        {"deny_action",
-            sc(
-                hoconsc:enum([ignore, disconnect]),
-                #{
-                    default => ignore,
-                    required => true,
-                    desc => ?DESC(fields_authorization_deny_action)
-                }
-            )},
-        {"cache",
-            sc(
-                ref(?MODULE, "cache"),
-                #{}
-            )}
-    ];
+    authz_fields();
 fields("cache") ->
     [
         {"enable",
@@ -2088,6 +2068,33 @@ do_default_ciphers(_) ->
     %% otherwise resolve default ciphers list at runtime
     [].
 
+authz_fields() ->
+    [
+        {"no_match",
+            sc(
+                hoconsc:enum([allow, deny]),
+                #{
+                    default => allow,
+                    required => true,
+                    desc => ?DESC(fields_authorization_no_match)
+                }
+            )},
+        {"deny_action",
+            sc(
+                hoconsc:enum([ignore, disconnect]),
+                #{
+                    default => ignore,
+                    required => true,
+                    desc => ?DESC(fields_authorization_deny_action)
+                }
+            )},
+        {"cache",
+            sc(
+                ref(?MODULE, "cache"),
+                #{}
+            )}
+    ].
+
 %% @private return a list of keys in a parent field
 -spec keys(string(), hocon:config()) -> [string()].
 keys(Parent, Conf) ->
@@ -2342,7 +2349,7 @@ authentication(Which) ->
             undefined -> hoconsc:array(typerefl:map());
             Module -> Module:root_type()
         end,
-    %% It is a lazy type because when handing runtime update requests
+    %% It is a lazy type because when handling runtime update requests
     %% the config is not checked by emqx_schema, but by the injected schema
     Type = hoconsc:lazy(Type0),
     #{

apps/emqx/test/emqx_common_test_http.erl

@@ -29,6 +29,9 @@
     auth_header/2
 ]).
 
+-define(DEFAULT_APP_ID, <<"default_appid">>).
+-define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
+
 request_api(Method, Url, Auth) ->
     request_api(Method, Url, [], Auth, []).
 
@@ -74,12 +77,18 @@ auth_header(User, Pass) ->
     {"Authorization", "Basic " ++ Encoded}.
 
 default_auth_header() ->
-    AppId = <<"myappid">>,
-    AppSecret = emqx_mgmt_auth:get_appsecret(AppId),
-    auth_header(erlang:binary_to_list(AppId), erlang:binary_to_list(AppSecret)).
+    {ok, #{api_key := APIKey}} = emqx_mgmt_auth:read(?DEFAULT_APP_ID),
+    auth_header(
+        erlang:binary_to_list(APIKey), erlang:binary_to_list(?DEFAULT_APP_SECRET)
+    ).
 
 create_default_app() ->
-    emqx_mgmt_auth:add_app(<<"myappid">>, <<"test">>).
+    Now = erlang:system_time(second),
+    ExpiredAt = Now + timer:minutes(10),
+    emqx_mgmt_auth:create(
+        ?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">>
+    ),
+    ok.
 
 delete_default_app() ->
-    emqx_mgmt_auth:del_app(<<"myappid">>).
+    emqx_mgmt_auth:delete(?DEFAULT_APP_ID).

apps/emqx_authn/src/emqx_authn.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.11"},
+    {vsn, "0.1.12"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -22,6 +22,7 @@
 -export([
     common_fields/0,
     roots/0,
+    tags/0,
     fields/1,
     authenticator_type/0,
     authenticator_type_without_scram/0,
@@ -32,6 +33,9 @@
 
 roots() -> [].
 
+tags() ->
+    [<<"Authentication">>].
+
 common_fields() ->
     [{enable, fun enable/1}].
 

apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl

@@ -25,6 +25,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -105,6 +106,9 @@ mnesia(boot) ->
 
 namespace() -> "authn-scram-builtin_db".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -26,6 +26,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1,
@@ -51,6 +52,9 @@
 
 namespace() -> "authn-http".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() ->
     [
         {?CONF_NS,

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -25,6 +25,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -44,6 +45,9 @@
 
 namespace() -> "authn-jwt".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() ->
     [
         {?CONF_NS,

apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl

@@ -26,6 +26,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -107,6 +108,9 @@ mnesia(boot) ->
 
 namespace() -> "authn-builtin_db".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -25,6 +25,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -44,6 +45,9 @@
 
 namespace() -> "authn-mongodb".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() ->
     [
         {?CONF_NS,

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -27,6 +27,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -46,6 +47,9 @@
 
 namespace() -> "authn-mysql".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -26,6 +26,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -50,6 +51,9 @@
 
 namespace() -> "authn-postgresql".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -25,6 +25,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1
@@ -44,6 +45,9 @@
 
 namespace() -> "authn-redis".
 
+tags() ->
+    [<<"Authentication">>].
+
 roots() ->
     [
         {?CONF_NS,

apps/emqx_authn/test/emqx_authn_api_SUITE.erl

@@ -18,7 +18,8 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1, multipart_formdata_request/3]).
+-import(emqx_dashboard_api_test_helpers, [multipart_formdata_request/3]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 -include("emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -65,9 +66,8 @@ end_per_testcase(_, Config) ->
 init_per_suite(Config) ->
     emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY),
     _ = application:load(emqx_conf),
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_authn, emqx_dashboard],
-        fun set_special_configs/1
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_authn]
     ),
 
     ?AUTHN:delete_chain(?GLOBAL),
@@ -76,12 +76,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_authn]),
     ok.
 
 %%------------------------------------------------------------------------------

apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl

@@ -197,7 +197,7 @@ t_list_users(_) ->
             #{is_superuser := false, user_id := _},
             #{is_superuser := false, user_id := _}
         ],
-        meta := #{page := 1, limit := 2, count := 3}
+        meta := #{page := 1, limit := 2, count := 3, hasnext := true}
     } = emqx_authn_mnesia:list_users(
         #{<<"page">> => 1, <<"limit">> => 2},
         State
@@ -205,7 +205,7 @@ t_list_users(_) ->
 
     #{
         data := [#{is_superuser := false, user_id := _}],
-        meta := #{page := 2, limit := 2, count := 3}
+        meta := #{page := 2, limit := 2, count := 3, hasnext := false}
     } = emqx_authn_mnesia:list_users(
         #{<<"page">> => 2, <<"limit">> => 2},
         State
@@ -213,7 +213,7 @@ t_list_users(_) ->
 
     #{
         data := [#{is_superuser := false, user_id := <<"u3">>}],
-        meta := #{page := 1, limit := 20, count := 0}
+        meta := #{page := 1, limit := 20, hasnext := false}
     } = emqx_authn_mnesia:list_users(
         #{
             <<"page">> => 1,

apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl

@@ -300,14 +300,14 @@ t_list_users(_) ->
 
     #{
         data := [?USER_MAP, ?USER_MAP],
-        meta := #{page := 1, limit := 2, count := 3}
+        meta := #{page := 1, limit := 2, count := 3, hasnext := true}
     } = emqx_enhanced_authn_scram_mnesia:list_users(
         #{<<"page">> => 1, <<"limit">> => 2},
         State
     ),
     #{
         data := [?USER_MAP],
-        meta := #{page := 2, limit := 2, count := 3}
+        meta := #{page := 2, limit := 2, count := 3, hasnext := false}
     } = emqx_enhanced_authn_scram_mnesia:list_users(
         #{<<"page">> => 2, <<"limit">> => 2},
         State
@@ -319,7 +319,7 @@ t_list_users(_) ->
                 is_superuser := _
             }
         ],
-        meta := #{page := 1, limit := 3, count := 0}
+        meta := #{page := 1, limit := 3, hasnext := false}
     } = emqx_enhanced_authn_scram_mnesia:list_users(
         #{
             <<"page">> => 1,

apps/emqx_authz/README.md

@@ -15,7 +15,6 @@ authz:{
               pool_size: 1
               username: root
               password: public
-              auto_reconnect: true
               ssl: {
                 enable: true
                 cacertfile:  "etc/certs/cacert.pem"
@@ -33,7 +32,6 @@ authz:{
               pool_size: 1
               username: root
               password: public
-              auto_reconnect: true
               ssl: {enable: false}
            }
            sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}"
@@ -45,7 +43,6 @@ authz:{
               database: 0
               pool_size: 1
               password: public
-              auto_reconnect: true
               ssl: {enable: false}
            }
            cmd: "HGETALL mqtt_authz:${username}"

apps/emqx_authz/etc/emqx_authz.conf

@@ -1,6 +1,7 @@
 authorization {
   deny_action = ignore
   no_match = allow
+  cache = { enable = true }
   sources =  [
     {
       type = file

apps/emqx_authz/src/emqx_authz_api_settings.erl

@@ -64,7 +64,7 @@ schema("/authorization/settings") ->
     }.
 
 ref_authz_schema() ->
-    proplists:delete(sources, emqx_conf_schema:fields("authorization")).
+    emqx_schema:authz_fields().
 
 settings(get, _Params) ->
     {200, authorization_settings()};
@@ -83,4 +83,6 @@ settings(put, #{
     {200, authorization_settings()}.
 
 authorization_settings() ->
-    maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})).
+    C = maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})),
+    Schema = emqx_hocon:make_schema(emqx_schema:authz_fields()),
+    hocon_tconf:make_serializable(Schema, C, #{}).

apps/emqx_authz/src/emqx_authz_api_sources.erl

@@ -449,7 +449,7 @@ is_ok(ResL) ->
 
 get_raw_sources() ->
     RawSources = emqx:get_raw_config([authorization, sources], []),
-    Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
+    Schema = emqx_hocon:make_schema(emqx_authz_schema:authz_fields()),
     Conf = #{<<"sources">> => RawSources},
     #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}),
     merge_default_headers(Sources).

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -33,9 +33,11 @@
 -export([
     namespace/0,
     roots/0,
+    tags/0,
     fields/1,
     validations/0,
-    desc/1
+    desc/1,
+    authz_fields/0
 ]).
 
 -export([
@@ -65,28 +67,15 @@ type_names() ->
 
 namespace() -> authz.
 
+tags() ->
+    [<<"Authorization">>].
+
 %% @doc authorization schema is not exported
 %% but directly used by emqx_schema
 roots() -> [].
 
 fields("authorization") ->
-    Types = [?R_REF(Type) || Type <- type_names()],
-    UnionMemberSelector =
-        fun
-            (all_union_members) -> Types;
-            %% must return list
-            ({value, Value}) -> [select_union_member(Value)]
-        end,
-    [
-        {sources,
-            ?HOCON(
-                ?ARRAY(?UNION(UnionMemberSelector)),
-                #{
-                    default => [],
-                    desc => ?DESC(sources)
-                }
-            )}
-    ];
+    authz_fields();
 fields(file) ->
     authz_common_fields(file) ++
         [{path, ?HOCON(string(), #{required => true, desc => ?DESC(path)})}];
@@ -488,3 +477,22 @@ select_union_member_loop(TypeValue, [Type | Types]) ->
         false ->
             select_union_member_loop(TypeValue, Types)
     end.
+
+authz_fields() ->
+    Types = [?R_REF(Type) || Type <- type_names()],
+    UnionMemberSelector =
+        fun
+            (all_union_members) -> Types;
+            %% must return list
+            ({value, Value}) -> [select_union_member(Value)]
+        end,
+    [
+        {sources,
+            ?HOCON(
+                ?ARRAY(?UNION(UnionMemberSelector)),
+                #{
+                    default => [],
+                    desc => ?DESC(sources)
+                }
+            )}
+    ].

apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/2, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -32,8 +32,8 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_authz, emqx_dashboard, emqx_management],
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_authz],
         fun set_special_configs/1
     ),
     Config.
@@ -47,7 +47,7 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]),
+    emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
     ok.
 
 set_special_configs(emqx_dashboard) ->
@@ -67,12 +67,12 @@ t_clean_cahce(_) ->
     ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0),
 
     {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
-    ?assertEqual(2, length(jsx:decode(Result3))),
+    ?assertEqual(2, length(emqx_json:decode(Result3))),
 
     request(delete, uri(["authorization", "cache"])),
 
     {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
-    ?assertEqual(0, length(jsx:decode(Result4))),
+    ?assertEqual(0, length(emqx_json:decode(Result4))),
 
     ok.
 

apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl

@@ -22,7 +22,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -31,8 +31,8 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_authz, emqx_dashboard],
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_authz],
         fun set_special_configs/1
     ),
     Config.
@@ -46,7 +46,7 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+    emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
     ok.
 
 set_special_configs(emqx_dashboard) ->
@@ -92,7 +92,8 @@ t_api(_) ->
         <<"meta">> := #{
             <<"count">> := 1,
             <<"limit">> := 100,
-            <<"page">> := 1
+            <<"page">> := 1,
+            <<"hasnext">> := false
         }
     } = jsx:decode(Request1),
     ?assertEqual(3, length(Rules1)),
@@ -111,9 +112,9 @@ t_api(_) ->
     #{
         <<"data">> := [],
         <<"meta">> := #{
-            <<"count">> := 0,
             <<"limit">> := 20,
-            <<"page">> := 1
+            <<"page">> := 1,
+            <<"hasnext">> := false
         }
     } = jsx:decode(Request1_1),
 

apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -30,7 +30,7 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = emqx_common_test_helpers:start_apps(
+    ok = emqx_mgmt_api_test_util:init_suite(
         [emqx_conf, emqx_authz, emqx_dashboard],
         fun set_special_configs/1
     ),
@@ -46,7 +46,7 @@ end_per_suite(_Config) ->
         }
     ),
     ok = stop_apps([emqx_resource]),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+    emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
     ok.
 
 set_special_configs(emqx_dashboard) ->

apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -115,8 +115,8 @@ init_per_suite(Config) ->
         end
     ),
 
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_authz, emqx_dashboard],
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_authz],
         fun set_special_configs/1
     ),
     ok = start_apps([emqx_resource]),
@@ -134,7 +134,7 @@ end_per_suite(_Config) ->
     %% resource and connector should be stop first,
     %% or authz_[mysql|pgsql|redis..]_SUITE would be failed
     ok = stop_apps([emqx_resource]),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+    emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
     meck:unload(emqx_resource),
     ok.
 

apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl

@@ -93,9 +93,8 @@ init_per_suite(Config) ->
             "        }"
         >>
     ),
-    emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_dashboard, ?APP],
-        fun set_special_configs/1
+    emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, ?APP]
     ),
     Config.
 
@@ -111,12 +110,6 @@ end_per_testcase(t_get_basic_usage_info, _Config) ->
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config(),
-    ok;
-set_special_configs(_) ->
-    ok.
-
 topic_config(T) ->
     #{
         topic => T,
@@ -132,7 +125,7 @@ end_per_suite(_) ->
     application:unload(?APP),
     meck:unload(emqx_resource),
     meck:unload(emqx_schema),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]).
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, ?APP]).
 
 t_auto_subscribe(_) ->
     emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),

apps/emqx_bridge/src/schema/emqx_bridge_schema.erl

@@ -20,7 +20,7 @@
 
 -import(hoconsc, [mk/2, ref/2]).
 
--export([roots/0, fields/1, desc/1, namespace/0]).
+-export([roots/0, fields/1, desc/1, namespace/0, tags/0]).
 
 -export([
     get_response/0,
@@ -104,6 +104,9 @@ metrics_status_fields() ->
 
 namespace() -> "bridge".
 
+tags() ->
+    [<<"Bridge">>].
+
 roots() -> [bridges].
 
 fields(bridges) ->

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -60,9 +60,8 @@ init_per_suite(Config) ->
     %% some testcases (may from other app) already get emqx_connector started
     _ = application:stop(emqx_resource),
     _ = application:stop(emqx_connector),
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_rule_engine, emqx_bridge, emqx_dashboard],
-        fun set_special_configs/1
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_rule_engine, emqx_bridge]
     ),
     ok = emqx_common_test_helpers:load_config(
         emqx_rule_engine_schema,
@@ -72,12 +71,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_bridge, emqx_dashboard]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config(<<"bridge_admin">>);
-set_special_configs(_) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge]),
     ok.
 
 init_per_testcase(_, Config) ->
@@ -605,9 +599,6 @@ t_with_redact_update(_Config) ->
     ?assertEqual(Password, Value),
     ok.
 
-request(Method, Url, Body) ->
-    request(<<"bridge_admin">>, Method, Url, Body).
-
 operation_path(node, Oper, BridgeID) ->
     uri(["nodes", node(), "bridges", BridgeID, "operation", Oper]);
 operation_path(cluster, Oper, BridgeID) ->

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -38,7 +38,9 @@
     cipher/0
 ]).
 
--export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]).
+-export([
+    namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0
+]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 
 %% Static apps which merge their configs into the merged emqx.conf
@@ -67,6 +69,9 @@
 %% root config should not have a namespace
 namespace() -> undefined.
 
+tags() ->
+    [<<"EMQX">>].
+
 roots() ->
     PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY,
     case persistent_term:get(PtKey, undefined) of
@@ -942,8 +947,8 @@ fields("log_burst_limit") ->
             )}
     ];
 fields("authorization") ->
-    emqx_schema:fields("authorization") ++
-        emqx_authz_schema:fields("authorization").
+    emqx_schema:authz_fields() ++
+        emqx_authz_schema:authz_fields().
 
 desc("cluster") ->
     ?DESC("desc_cluster");

apps/emqx_connector/README.md

@@ -14,7 +14,7 @@ An MySQL connector can be used as following:
 ```
 (emqx@127.0.0.1)5> emqx_resource:list_instances_verbose().
 [#{config =>
-       #{auto_reconnect => true,cacertfile => [],certfile => [],
+       #{cacertfile => [],certfile => [],
          database => "mqtt",keyfile => [],password => "public",
          pool_size => 1,
          server => {{127,0,0,1},3306},

apps/emqx_connector/i18n/emqx_connector_schema_lib.conf

@@ -68,12 +68,12 @@ emqx_connector_schema_lib {
 
     auto_reconnect {
         desc {
-          en: "Enable automatic reconnect to the database."
-          zh: "自动重连数据库。"
+          en: "Deprecated. Enable automatic reconnect to the database."
+          zh: "已弃用。自动重连数据库。"
         }
         label: {
-              en: "Auto Reconnect Database"
-              zh: "自动重连数据库"
+          en: "Deprecated. Auto Reconnect Database"
+          zh: "已弃用。自动重连数据库"
         }
     }
 

apps/emqx_connector/include/emqx_connector.hrl

@@ -24,6 +24,8 @@
 -define(REDIS_DEFAULT_PORT, 6379).
 -define(PGSQL_DEFAULT_PORT, 5432).
 
+-define(AUTO_RECONNECT_INTERVAL, 2).
+
 -define(SERVERS_DESC,
     "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`<br/>"
     "For each Node should be: "

apps/emqx_connector/rebar.config

@@ -12,7 +12,7 @@
     {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
     {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}},
     %% NOTE: mind poolboy version when updating mongodb-erlang version
-    {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.18"}}},
+    {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}},
     %% NOTE: mind poolboy version when updating eredis_cluster version
     {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.5"}}},
     %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git

apps/emqx_connector/src/emqx_connector_http.erl

@@ -209,7 +209,7 @@ on_start(
     ?SLOG(info, #{
         msg => "starting_http_connector",
         connector => InstId,
-        config => Config
+        config => emqx_misc:redact(Config)
     }),
     {Transport, TransportOpts} =
         case Scheme of

apps/emqx_connector/src/emqx_connector_ldap.erl

@@ -59,14 +59,13 @@ on_start(
         bind_password := BindPassword,
         timeout := Timeout,
         pool_size := PoolSize,
-        auto_reconnect := AutoReconn,
         ssl := SSL
     } = Config
 ) ->
     ?SLOG(info, #{
         msg => "starting_ldap_connector",
         connector => InstId,
-        config => Config
+        config => emqx_misc:redact(Config)
     }),
     Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS),
     SslOpts =
@@ -86,11 +85,11 @@ on_start(
         {bind_password, BindPassword},
         {timeout, Timeout},
         {pool_size, PoolSize},
-        {auto_reconnect, reconn_interval(AutoReconn)}
+        {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of
-        ok -> {ok, #{poolname => PoolName, auto_reconnect => AutoReconn}};
+        ok -> {ok, #{poolname => PoolName}};
         {error, Reason} -> {error, Reason}
     end.
 
@@ -129,9 +128,6 @@ on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = S
 
 on_get_status(_InstId, _State) -> connected.
 
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
-
 search(Conn, Base, Filter, Attributes) ->
     eldap2:search(Conn, [
         {base, Base},

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -155,7 +155,7 @@ on_start(
             rs -> "starting_mongodb_replica_set_connector";
             sharded -> "starting_mongodb_sharded_connector"
         end,
-    ?SLOG(info, #{msg => Msg, connector => InstId, config => Config}),
+    ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}),
     NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config),
     SslOpts =
         case maps:get(enable, SSL) of

apps/emqx_connector/src/emqx_connector_mqtt.erl

@@ -15,6 +15,8 @@
 %%--------------------------------------------------------------------
 -module(emqx_connector_mqtt).
 
+-include("emqx_connector.hrl").
+
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
@@ -147,7 +149,7 @@ on_start(InstId, Conf) ->
     ?SLOG(info, #{
         msg => "starting_mqtt_connector",
         connector => InstanceId,
-        config => Conf
+        config => emqx_misc:redact(Conf)
     }),
     BasicConf = basic_config(Conf),
     BridgeConf = BasicConf#{
@@ -198,12 +200,10 @@ on_query_async(
     ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
     emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplyFun, Args}).
 
-on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) ->
-    AutoReconn = maps:get(auto_reconnect, Conf, true),
+on_get_status(_InstId, #{name := InstanceId}) ->
     case emqx_connector_mqtt_worker:status(InstanceId) of
         connected -> connected;
-        _ when AutoReconn == true -> connecting;
-        _ when AutoReconn == false -> disconnected
+        _ -> connecting
     end.
 
 ensure_mqtt_worker_started(InstanceId, BridgeConf) ->
@@ -236,7 +236,6 @@ make_forward_confs(FrowardConf) ->
 basic_config(
     #{
         server := Server,
-        reconnect_interval := ReconnIntv,
         proto_ver := ProtoVer,
         bridge_mode := BridgeMode,
         clean_start := CleanStart,
@@ -252,7 +251,7 @@ basic_config(
         %% 30s
         connect_timeout => 30,
         auto_reconnect => true,
-        reconnect_interval => ReconnIntv,
+        reconnect_interval => ?AUTO_RECONNECT_INTERVAL,
         proto_ver => ProtoVer,
         %% Opening bridge_mode will form a non-standard mqtt connection message.
         %% A load balancing server (such as haproxy) is often set up before the emqx broker server.

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -52,7 +52,6 @@
 -type state() ::
     #{
         poolname := atom(),
-        auto_reconnect := boolean(),
         prepare_statement := prepares(),
         params_tokens := params_tokens(),
         batch_inserts := sqls(),
@@ -84,8 +83,6 @@ on_start(
         server := Server,
         database := DB,
         username := User,
-        password := Password,
-        auto_reconnect := AutoReconn,
         pool_size := PoolSize,
         ssl := SSL
     } = Config
@@ -94,7 +91,7 @@ on_start(
     ?SLOG(info, #{
         msg => "starting_mysql_connector",
         connector => InstId,
-        config => Config
+        config => emqx_misc:redact(Config)
     }),
     SslOpts =
         case maps:get(enable, SSL) of
@@ -107,14 +104,14 @@ on_start(
         {host, Host},
         {port, Port},
         {user, User},
-        {password, Password},
+        {password, maps:get(password, Config, <<>>)},
         {database, DB},
-        {auto_reconnect, reconn_interval(AutoReconn)},
+        {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
         {pool_size, PoolSize}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     Prepares = parse_prepare_sql(Config),
-    State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
+    State = maps:merge(#{poolname => PoolName}, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok ->
             {ok, init_prepare(State)};
@@ -194,7 +191,7 @@ mysql_function(prepared_query) ->
 mysql_function(_) ->
     mysql_function(prepared_query).
 
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
+on_get_status(_InstId, #{poolname := Pool} = State) ->
     case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
         true ->
             case do_check_prepares(State) of
@@ -205,10 +202,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State
                     {connected, NState};
                 {error, _Reason} ->
                     %% do not log error, it is logged in prepare_sql_to_conn
-                    conn_status(AutoReconn)
+                    connecting
             end;
         false ->
-            conn_status(AutoReconn)
+            connecting
     end.
 
 do_get_status(Conn) ->
@@ -227,11 +224,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, P
     end.
 
 %% ===================================================================
-conn_status(_AutoReconn = true) -> connecting;
-conn_status(_AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
 
 connect(Options) ->
     mysql:start_link(Options).

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -56,7 +56,6 @@
 -type state() ::
     #{
         poolname := atom(),
-        auto_reconnect := boolean(),
         prepare_sql := prepares(),
         params_tokens := params_tokens(),
         prepare_statement := epgsql:statement()
@@ -87,8 +86,6 @@ on_start(
         server := Server,
         database := DB,
         username := User,
-        password := Password,
-        auto_reconnect := AutoReconn,
         pool_size := PoolSize,
         ssl := SSL
     } = Config
@@ -97,7 +94,7 @@ on_start(
     ?SLOG(info, #{
         msg => "starting_postgresql_connector",
         connector => InstId,
-        config => Config
+        config => emqx_misc:redact(Config)
     }),
     SslOpts =
         case maps:get(enable, SSL) of
@@ -113,14 +110,14 @@ on_start(
         {host, Host},
         {port, Port},
         {username, User},
-        {password, emqx_secret:wrap(Password)},
+        {password, emqx_secret:wrap(maps:get(password, Config, ""))},
         {database, DB},
-        {auto_reconnect, reconn_interval(AutoReconn)},
+        {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
         {pool_size, PoolSize}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     Prepares = parse_prepare_sql(Config),
-    InitState = #{poolname => PoolName, auto_reconnect => AutoReconn, prepare_statement => #{}},
+    InitState = #{poolname => PoolName, prepare_statement => #{}},
     State = maps:merge(InitState, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok ->
@@ -247,7 +244,7 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) ->
     end,
     Result.
 
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
+on_get_status(_InstId, #{poolname := Pool} = State) ->
     case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
         true ->
             case do_check_prepares(State) of
@@ -258,10 +255,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State
                     {connected, NState};
                 false ->
                     %% do not log error, it is logged in prepare_sql_to_conn
-                    conn_status(AutoReconn)
+                    connecting
             end;
         false ->
-            conn_status(AutoReconn)
+            connecting
     end.
 
 do_get_status(Conn) ->
@@ -280,11 +277,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepare
     end.
 
 %% ===================================================================
-conn_status(_AutoReconn = true) -> connecting;
-conn_status(_AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
 
 connect(Opts) ->
     Host = proplists:get_value(host, Opts),

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -117,14 +117,13 @@ on_start(
     #{
         redis_type := Type,
         pool_size := PoolSize,
-        auto_reconnect := AutoReconn,
         ssl := SSL
     } = Config
 ) ->
     ?SLOG(info, #{
         msg => "starting_redis_connector",
         connector => InstId,
-        config => Config
+        config => emqx_misc:redact(Config)
     }),
     ConfKey =
         case Type of
@@ -142,7 +141,7 @@ on_start(
         [
             {pool_size, PoolSize},
             {password, maps:get(password, Config, "")},
-            {auto_reconnect, reconn_interval(AutoReconn)}
+            {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
         ] ++ Database ++ Servers,
     Options =
         case maps:get(enable, SSL) of
@@ -155,7 +154,7 @@ on_start(
                 [{ssl, false}]
         end ++ [{sentinel, maps:get(sentinel, Config, undefined)}],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
-    State = #{poolname => PoolName, type => Type, auto_reconnect => AutoReconn},
+    State = #{poolname => PoolName, type => Type},
     case Type of
         cluster ->
             case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of
@@ -229,18 +228,18 @@ eredis_cluster_workers_exist_and_are_connected(Workers) ->
             Workers
         ).
 
-on_get_status(_InstId, #{type := cluster, poolname := PoolName, auto_reconnect := AutoReconn}) ->
+on_get_status(_InstId, #{type := cluster, poolname := PoolName}) ->
     case eredis_cluster:pool_exists(PoolName) of
         true ->
             Workers = extract_eredis_cluster_workers(PoolName),
             Health = eredis_cluster_workers_exist_and_are_connected(Workers),
-            status_result(Health, AutoReconn);
+            status_result(Health);
         false ->
             disconnected
     end;
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn}) ->
+on_get_status(_InstId, #{poolname := Pool}) ->
     Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1),
-    status_result(Health, AutoReconn).
+    status_result(Health).
 
 do_get_status(Conn) ->
     case eredis:q(Conn, ["PING"]) of
@@ -248,12 +247,8 @@ do_get_status(Conn) ->
         _ -> false
     end.
 
-status_result(_Status = true, _AutoReconn) -> connected;
-status_result(_Status = false, _AutoReconn = true) -> connecting;
-status_result(_Status = false, _AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
+status_result(_Status = true) -> connected;
+status_result(_Status = false) -> connecting.
 
 do_cmd(PoolName, cluster, {cmd, Command}) ->
     eredis_cluster:q(PoolName, Command);

apps/emqx_connector/src/emqx_connector_schema_lib.erl

@@ -106,4 +106,5 @@ password(_) -> undefined.
 auto_reconnect(type) -> boolean();
 auto_reconnect(desc) -> ?DESC("auto_reconnect");
 auto_reconnect(default) -> true;
+auto_reconnect(deprecated) -> {since, "v5.0.15"};
 auto_reconnect(_) -> undefined.

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -65,8 +65,12 @@ start_listeners(Listeners) ->
         components => #{
             schemas => #{},
             'securitySchemes' => #{
-                'basicAuth' => #{type => http, scheme => basic},
-                'bearerAuth' => #{type => http, scheme => bearer}
+                'basicAuth' => #{
+                    type => http,
+                    scheme => basic,
+                    description =>
+                        <<"Authorize with [API Keys](https://www.emqx.io/docs/en/v5.0/admin/api.html#api-keys)">>
+                }
             }
         }
     },
@@ -215,28 +219,7 @@ listener_name(Protocol) ->
 authorize(Req) ->
     case cowboy_req:parse_header(<<"authorization">>, Req) of
         {basic, Username, Password} ->
-            case emqx_dashboard_admin:check(Username, Password) of
-                ok ->
-                    ok;
-                {error, <<"username_not_found">>} ->
-                    Path = cowboy_req:path(Req),
-                    case emqx_mgmt_auth:authorize(Path, Username, Password) of
-                        ok ->
-                            ok;
-                        {error, <<"not_allowed">>} ->
-                            return_unauthorized(
-                                ?WRONG_USERNAME_OR_PWD,
-                                <<"Check username/password">>
-                            );
-                        {error, _} ->
-                            return_unauthorized(
-                                ?WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET,
-                                <<"Check username/password or api_key/api_secret">>
-                            )
-                    end;
-                {error, _} ->
-                    return_unauthorized(?WRONG_USERNAME_OR_PWD, <<"Check username/password">>)
-            end;
+            api_key_authorize(Req, Username, Password);
         {bearer, Token} ->
             case emqx_dashboard_admin:verify_token(Token) of
                 ok ->
@@ -269,3 +252,20 @@ i18n_file() ->
 
 listeners() ->
     emqx_conf:get([dashboard, listeners], []).
+
+api_key_authorize(Req, Key, Secret) ->
+    Path = cowboy_req:path(Req),
+    case emqx_mgmt_auth:authorize(Path, Key, Secret) of
+        ok ->
+            ok;
+        {error, <<"not_allowed">>} ->
+            return_unauthorized(
+                ?BAD_API_KEY_OR_SECRET,
+                <<"Not allowed, Check api_key/api_secret">>
+            );
+        {error, _} ->
+            return_unauthorized(
+                ?BAD_API_KEY_OR_SECRET,
+                <<"Check api_key/api_secret">>
+            )
+    end.

apps/emqx_dashboard/src/emqx_dashboard_api.erl

@@ -47,7 +47,7 @@
 
 -define(EMPTY(V), (V == undefined orelse V == <<>>)).
 
--define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD').
+-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
 -define(WRONG_TOKEN_OR_USERNAME, 'WRONG_TOKEN_OR_USERNAME').
 -define(USER_NOT_FOUND, 'USER_NOT_FOUND').
 -define(ERROR_PWD_NOT_MATCH, 'ERROR_PWD_NOT_MATCH').
@@ -164,7 +164,7 @@ schema("/users/:username/change_pwd") ->
     }.
 
 response_schema(401) ->
-    emqx_dashboard_swagger:error_codes([?WRONG_USERNAME_OR_PWD], ?DESC(login_failed401));
+    emqx_dashboard_swagger:error_codes([?BAD_USERNAME_OR_PWD], ?DESC(login_failed401));
 response_schema(404) ->
     emqx_dashboard_swagger:error_codes([?USER_NOT_FOUND], ?DESC(users_api404)).
 
@@ -223,7 +223,7 @@ login(post, #{body := Params}) ->
             }};
         {error, R} ->
             ?SLOG(info, #{msg => "Dashboard login failed", username => Username, reason => R}),
-            {401, ?WRONG_USERNAME_OR_PWD, <<"Auth failed">>}
+            {401, ?BAD_USERNAME_OR_PWD, <<"Auth failed">>}
     end.
 
 logout(_, #{

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -139,14 +139,20 @@ fields(limit) ->
     [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}];
 fields(count) ->
     Desc = <<
-        "Total number of records counted.<br/>"
-        "Note: this field is <code>0</code> when the queryed table is empty, "
-        "or if the query can not be optimized and requires a full table scan."
+        "Total number of records matching the query.<br/>"
+        "Note: this field is present only if the query can be optimized and does "
+        "not require a full table scan."
+    >>,
+    Meta = #{desc => Desc, required => false},
+    [{count, hoconsc:mk(non_neg_integer(), Meta)}];
+fields(hasnext) ->
+    Desc = <<
+        "Flag indicating whether there are more results available on next pages."
     >>,
     Meta = #{desc => Desc, required => true},
-    [{count, hoconsc:mk(non_neg_integer(), Meta)}];
+    [{hasnext, hoconsc:mk(boolean(), Meta)}];
 fields(meta) ->
-    fields(page) ++ fields(limit) ++ fields(count).
+    fields(page) ++ fields(limit) ++ fields(count) ++ fields(hasnext).
 
 -spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map().
 schema_with_example(Type, Example) ->
@@ -708,6 +714,8 @@ typename_to_spec("qos()", _Mod) ->
     #{type => integer, minimum => 0, maximum => 2, example => 0};
 typename_to_spec("{binary(), binary()}", _Mod) ->
     #{type => object, example => #{}};
+typename_to_spec("{string(), string()}", _Mod) ->
+    #{type => object, example => #{}};
 typename_to_spec("comma_separated_list()", _Mod) ->
     #{type => string, example => <<"item1,item2">>};
 typename_to_spec("comma_separated_binary()", _Mod) ->

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -114,9 +114,9 @@ t_admin_delete_self_failed(_) ->
     ?assertEqual(1, length(Admins)),
     Header = auth_header_(<<"username1">>, <<"password">>),
     {error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header),
-    Token = erlang:iolist_to_binary(["Basic ", base64:encode("username1:password")]),
+    Token = ["Basic ", base64:encode("username1:password")],
     Header2 = {"Authorization", Token},
-    {error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2),
+    {error, {_, 401, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2),
     mnesia:clear_table(?ADMIN).
 
 t_rest_api(_Config) ->

apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl

@@ -25,43 +25,24 @@
 
 -define(SERVER, "http://127.0.0.1:18083/api/v5").
 
+-import(emqx_mgmt_api_test_util, [request/2]).
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
     mria:start(),
-    application:load(emqx_dashboard),
-    emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
+    emqx_mgmt_api_test_util:init_suite([emqx_conf]),
     Config.
 
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config(),
-    ok;
-set_special_configs(_) ->
-    ok.
-
 end_per_suite(Config) ->
     end_suite(),
     Config.
 
 end_suite() ->
-    application:unload(emqx_management),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard]).
+    emqx_mgmt_api_test_util:end_suite([emqx_conf]).
 
 t_bad_api_path(_) ->
     Url = ?SERVER ++ "/for/test/some/path/not/exist",
-    {error, {"HTTP/1.1", 404, "Not Found"}} = request(Url),
+    {ok, 404, _} = request(get, Url),
     ok.
-
-request(Url) ->
-    Request = {Url, []},
-    case httpc:request(get, Request, [], []) of
-        {error, Reason} ->
-            {error, Reason};
-        {ok, {{"HTTP/1.1", Code, _}, _, Return}} when
-            Code >= 200 andalso Code =< 299
-        ->
-            {ok, emqx_json:decode(Return, [return_maps])};
-        {ok, {Reason, _, _}} ->
-            {error, Reason}
-    end.

apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl

@@ -19,6 +19,8 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
+-import(emqx_dashboard_SUITE, [auth_header_/0]).
+
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx.hrl").
@@ -153,10 +155,6 @@ do_request_api(Method, Request) ->
             {error, Reason}
     end.
 
-auth_header_() ->
-    Basic = binary_to_list(base64:encode(<<"admin:public">>)),
-    {"Authorization", "Basic " ++ Basic}.
-
 restart_monitor() ->
     OldMonitor = erlang:whereis(emqx_dashboard_monitor),
     erlang:exit(OldMonitor, kill),

apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl

@@ -347,13 +347,7 @@ do_request_api(Method, Request) ->
     end.
 
 auth_header_() ->
-    AppId = <<"admin">>,
-    AppSecret = <<"public">>,
-    auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
-
-auth_header_(User, Pass) ->
-    Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
-    {"Authorization", "Basic " ++ Encoded}.
+    emqx_mgmt_api_test_util:auth_header_().
 
 api_path(Parts) ->
     ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -49,12 +49,15 @@
 ]).
 -elvis([{elvis_style, dont_repeat_yourself, disable}]).
 
--export([namespace/0, roots/0, fields/1, desc/1]).
+-export([namespace/0, roots/0, fields/1, desc/1, tags/0]).
 
 -export([proxy_protocol_opts/0]).
 
 namespace() -> gateway.
 
+tags() ->
+    [<<"Gateway">>].
+
 roots() -> [gateway].
 
 fields(gateway) ->

apps/emqx_gateway/test/emqx_gateway_test_utils.erl

@@ -106,8 +106,6 @@ assert_fields_exist(Ks, Map) ->
 %% http
 
 -define(http_api_host, "http://127.0.0.1:18083/api/v5").
--define(default_user, "admin").
--define(default_pass, "public").
 
 request(delete = Mth, Path) ->
     do_request(Mth, req(Path, []));
@@ -176,5 +174,4 @@ url(Path, Qs) ->
     lists:concat([?http_api_host, Path, "?", binary_to_list(cow_qs:qs(Qs))]).
 
 auth(Headers) ->
-    Token = base64:encode(?default_user ++ ":" ++ ?default_pass),
-    [{"Authorization", "Basic " ++ binary_to_list(Token)}] ++ Headers.
+    [emqx_mgmt_api_test_util:auth_header_() | Headers].

apps/emqx_management/src/emqx_mgmt_api.erl

@@ -20,7 +20,6 @@
 
 -elvis([{elvis_style, dont_repeat_yourself, #{min_complexity => 100}}]).
 
--define(FRESH_SELECT, fresh_select).
 -define(LONG_QUERY_TIMEOUT, 50000).
 
 -export([
@@ -174,13 +173,12 @@ do_node_query(
     case do_query(Node, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
-        {Rows, NQueryState = #{continuation := ?FRESH_SELECT}} ->
-            {_, NResultAcc} = accumulate_query_rows(Node, Rows, NQueryState, ResultAcc),
-            NResultAcc;
-        {Rows, NQueryState} ->
+        {Rows, NQueryState = #{complete := Complete}} ->
             case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
-                    NResultAcc;
+                    finalize_query(NResultAcc, NQueryState);
+                {_, NResultAcc} when Complete ->
+                    finalize_query(NResultAcc, NQueryState);
                 {more, NResultAcc} ->
                     do_node_query(Node, NQueryState, NResultAcc)
             end
@@ -212,8 +210,8 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
     end.
 
 %% @private
-do_cluster_query([], _QueryState, ResultAcc) ->
-    ResultAcc;
+do_cluster_query([], QueryState, ResultAcc) ->
+    finalize_query(ResultAcc, mark_complete(QueryState));
 do_cluster_query(
     [Node | Tail] = Nodes,
     QueryState,
@@ -222,31 +220,29 @@ do_cluster_query(
     case do_query(Node, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
-        {Rows, NQueryState} ->
+        {Rows, NQueryState = #{complete := Complete}} ->
             case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
-                    maybe_collect_total_from_tail_nodes(Tail, NQueryState, NResultAcc);
+                    FQueryState = maybe_collect_total_from_tail_nodes(Tail, NQueryState),
+                    FComplete = Complete andalso Tail =:= [],
+                    finalize_query(NResultAcc, mark_complete(FQueryState, FComplete));
+                {more, NResultAcc} when not Complete ->
+                    do_cluster_query(Nodes, NQueryState, NResultAcc);
+                {more, NResultAcc} when Tail =/= [] ->
+                    do_cluster_query(Tail, reset_query_state(NQueryState), NResultAcc);
                 {more, NResultAcc} ->
-                    NextNodes =
-                        case NQueryState of
-                            #{continuation := ?FRESH_SELECT} -> Tail;
-                            _ -> Nodes
-                        end,
-                    do_cluster_query(NextNodes, NQueryState, NResultAcc)
+                    finalize_query(NResultAcc, NQueryState)
             end
     end.
 
-maybe_collect_total_from_tail_nodes([], _QueryState, ResultAcc) ->
-    ResultAcc;
-maybe_collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) ->
-    case counting_total_fun(QueryState) of
-        false ->
-            ResultAcc;
-        _Fun ->
-            collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc)
-    end.
+maybe_collect_total_from_tail_nodes([], QueryState) ->
+    QueryState;
+maybe_collect_total_from_tail_nodes(Nodes, QueryState = #{total := _}) ->
+    collect_total_from_tail_nodes(Nodes, QueryState);
+maybe_collect_total_from_tail_nodes(_Nodes, QueryState) ->
+    QueryState.
 
-collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc}) ->
+collect_total_from_tail_nodes(Nodes, QueryState = #{total := TotalAcc}) ->
     %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
     case rpc:multicall(Nodes, ?MODULE, apply_total_query, [QueryState], ?LONG_QUERY_TIMEOUT) of
         {_, [Node | _]} ->
@@ -257,7 +253,8 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
                 [{Node, {badrpc, Reason}} | _] ->
                     {error, Node, {badrpc, Reason}};
                 [] ->
-                    ResultAcc#{total => ResL ++ TotalAcc}
+                    NTotalAcc = maps:merge(TotalAcc, maps:from_list(ResL)),
+                    QueryState#{total := NTotalAcc}
             end
     end.
 
@@ -266,13 +263,14 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
 %%--------------------------------------------------------------------
 
 %% QueryState ::
-%%  #{continuation := ets:continuation(),
+%%  #{continuation => ets:continuation(),
 %%    page := pos_integer(),
 %%    limit := pos_integer(),
-%%    total := [{node(), non_neg_integer()}],
+%%    total => #{node() => non_neg_integer()},
 %%    table := atom(),
-%%    qs := {Qs, Fuzzy}   %% parsed query params
-%%    msfun := query_to_match_spec_fun()
+%%    qs := {Qs, Fuzzy},  %% parsed query params
+%%    msfun := query_to_match_spec_fun(),
+%%    complete := boolean()
 %%    }
 init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
     #{match_spec := Ms, fuzzy_fun := FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
@@ -285,17 +283,31 @@ init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) -
                 true = is_list(Args),
                 {type, external} = erlang:fun_info(NamedFun, type)
         end,
-    #{
+    QueryState = #{
         page => Page,
         limit => Limit,
         table => Tab,
         qs => QString,
         msfun => MsFun,
-        mactch_spec => Ms,
+        match_spec => Ms,
         fuzzy_fun => FuzzyFun,
-        total => [],
-        continuation => ?FRESH_SELECT
-    }.
+        complete => false
+    },
+    case counting_total_fun(QueryState) of
+        false ->
+            QueryState;
+        Fun when is_function(Fun) ->
+            QueryState#{total => #{}}
+    end.
+
+reset_query_state(QueryState) ->
+    maps:remove(continuation, mark_complete(QueryState, false)).
+
+mark_complete(QueryState) ->
+    mark_complete(QueryState, true).
+
+mark_complete(QueryState, Complete) ->
+    QueryState#{complete => Complete}.
 
 %% @private This function is exempt from BPAPI
 do_query(Node, QueryState) when Node =:= node() ->
@@ -318,47 +330,50 @@ do_select(
     Node,
     QueryState0 = #{
         table := Tab,
-        mactch_spec := Ms,
-        fuzzy_fun := FuzzyFun,
-        continuation := Continuation,
-        limit := Limit
+        match_spec := Ms,
+        limit := Limit,
+        complete := false
     }
 ) ->
     QueryState = maybe_apply_total_query(Node, QueryState0),
     Result =
-        case Continuation of
-            ?FRESH_SELECT ->
+        case maps:get(continuation, QueryState, undefined) of
+            undefined ->
                 ets:select(Tab, Ms, Limit);
-            _ ->
+            Continuation ->
                 %% XXX: Repair is necessary because we pass Continuation back
                 %% and forth through the nodes in the `do_cluster_query`
                 ets:select(ets:repair_continuation(Continuation, Ms))
         end,
     case Result of
-        '$end_of_table' ->
-            {[], QueryState#{continuation => ?FRESH_SELECT}};
+        {Rows, '$end_of_table'} ->
+            NRows = maybe_apply_fuzzy_filter(Rows, QueryState),
+            {NRows, mark_complete(QueryState)};
         {Rows, NContinuation} ->
-            NRows =
-                case FuzzyFun of
-                    undefined ->
-                        Rows;
-                    {FilterFun, Args0} when is_function(FilterFun), is_list(Args0) ->
-                        lists:filter(
-                            fun(E) -> erlang:apply(FilterFun, [E | Args0]) end,
-                            Rows
-                        )
-                end,
-            {NRows, QueryState#{continuation => NContinuation}}
+            NRows = maybe_apply_fuzzy_filter(Rows, QueryState),
+            {NRows, QueryState#{continuation => NContinuation}};
+        '$end_of_table' ->
+            {[], mark_complete(QueryState)}
     end.
 
-maybe_apply_total_query(Node, QueryState = #{total := TotalAcc}) ->
-    case proplists:get_value(Node, TotalAcc, undefined) of
-        undefined ->
-            Total = apply_total_query(QueryState),
-            QueryState#{total := [{Node, Total} | TotalAcc]};
-        _ ->
-            QueryState
-    end.
+maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := undefined}) ->
+    Rows;
+maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := {FilterFun, Args}}) ->
+    lists:filter(
+        fun(E) -> erlang:apply(FilterFun, [E | Args]) end,
+        Rows
+    ).
+
+maybe_apply_total_query(Node, QueryState = #{total := Acc}) ->
+    case Acc of
+        #{Node := _} ->
+            QueryState;
+        #{} ->
+            NodeTotal = apply_total_query(QueryState),
+            QueryState#{total := Acc#{Node => NodeTotal}}
+    end;
+maybe_apply_total_query(_Node, QueryState = #{}) ->
+    QueryState.
 
 apply_total_query(QueryState = #{table := Tab}) ->
     case counting_total_fun(QueryState) of
@@ -371,7 +386,7 @@ apply_total_query(QueryState = #{table := Tab}) ->
 
 counting_total_fun(_QueryState = #{qs := {[], []}}) ->
     fun(Tab) -> ets:info(Tab, size) end;
-counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) ->
+counting_total_fun(_QueryState = #{match_spec := Ms, fuzzy_fun := undefined}) ->
     %% XXX: Calculating the total number of data that match a certain
     %% condition under a large table is very expensive because the
     %% entire ETS table needs to be scanned.
@@ -390,15 +405,16 @@ counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= und
 %% ResultAcc :: #{count := integer(),
 %%                cursor := integer(),
 %%                rows  := [{node(), Rows :: list()}],
-%%                total := [{node() => integer()}]
+%%                overflow := boolean(),
+%%                hasnext => boolean()
 %%               }
 init_query_result() ->
-    #{cursor => 0, count => 0, rows => [], total => []}.
+    #{cursor => 0, count => 0, rows => [], overflow => false}.
 
 accumulate_query_rows(
     Node,
     Rows,
-    _QueryState = #{page := Page, limit := Limit, total := TotalAcc},
+    _QueryState = #{page := Page, limit := Limit},
     ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc}
 ) ->
     PageStart = (Page - 1) * Limit + 1,
@@ -406,24 +422,35 @@ accumulate_query_rows(
     Len = length(Rows),
     case Cursor + Len of
         NCursor when NCursor < PageStart ->
-            {more, ResultAcc#{cursor => NCursor, total => TotalAcc}};
+            {more, ResultAcc#{cursor => NCursor}};
         NCursor when NCursor < PageEnd ->
+            SubRows = lists:nthtail(max(0, PageStart - Cursor - 1), Rows),
             {more, ResultAcc#{
                 cursor => NCursor,
-                count => Count + length(Rows),
-                total => TotalAcc,
-                rows => [{Node, Rows} | RowsAcc]
+                count => Count + length(SubRows),
+                rows => [{Node, SubRows} | RowsAcc]
             }};
         NCursor when NCursor >= PageEnd ->
             SubRows = lists:sublist(Rows, Limit - Count),
             {enough, ResultAcc#{
                 cursor => NCursor,
                 count => Count + length(SubRows),
-                total => TotalAcc,
-                rows => [{Node, SubRows} | RowsAcc]
+                rows => [{Node, SubRows} | RowsAcc],
+                % there are more rows than can fit in the page
+                overflow => (Limit - Count) < Len
             }}
     end.
 
+finalize_query(Result = #{overflow := Overflow}, QueryState = #{complete := Complete}) ->
+    HasNext = Overflow orelse not Complete,
+    maybe_accumulate_totals(Result#{hasnext => HasNext}, QueryState).
+
+maybe_accumulate_totals(Result, #{total := TotalAcc}) ->
+    QueryTotal = maps:fold(fun(_Node, T, N) -> N + T end, 0, TotalAcc),
+    Result#{total => QueryTotal};
+maybe_accumulate_totals(Result, _QueryState) ->
+    Result.
+
 %%--------------------------------------------------------------------
 %% Internal Functions
 %%--------------------------------------------------------------------
@@ -520,16 +547,22 @@ is_fuzzy_key(<<"match_", _/binary>>) ->
 is_fuzzy_key(_) ->
     false.
 
-format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) ->
+format_query_result(_FmtFun, _MetaIn, Error = {error, _Node, _Reason}) ->
     Error;
 format_query_result(
-    FmtFun, Meta, _ResultAcc = #{total := TotalAcc, rows := RowsAcc}
+    FmtFun, MetaIn, ResultAcc = #{hasnext := HasNext, rows := RowsAcc}
 ) ->
-    Total = lists:foldr(fun({_Node, T}, N) -> N + T end, 0, TotalAcc),
-    #{
+    Meta =
+        case ResultAcc of
+            #{total := QueryTotal} ->
                 %% The `count` is used in HTTP API to indicate the total number of
                 %% queries that can be read
-        meta => Meta#{count => Total},
+                MetaIn#{hasnext => HasNext, count => QueryTotal};
+            #{} ->
+                MetaIn#{hasnext => HasNext}
+        end,
+    #{
+        meta => Meta,
         data => lists:flatten(
             lists:foldl(
                 fun({Node, Rows}, Acc) ->
@@ -552,7 +585,7 @@ parse_pager_params(Params) ->
     Limit = b2i(limit(Params)),
     case Page > 0 andalso Limit > 0 of
         true ->
-            #{page => Page, limit => Limit, count => 0};
+            #{page => Page, limit => Limit};
         false ->
             false
     end.

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -40,6 +40,10 @@
     do_force_create_app/3
 ]).
 
+-ifdef(TEST).
+-export([create/5]).
+-endif.
+
 -define(APP, emqx_app).
 
 -record(?APP, {
@@ -68,8 +72,12 @@ init_bootstrap_file() ->
     init_bootstrap_file(File).
 
 create(Name, Enable, ExpiredAt, Desc) ->
+    ApiSecret = generate_api_secret(),
+    create(Name, ApiSecret, Enable, ExpiredAt, Desc).
+
+create(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
     case mnesia:table_info(?APP, size) < 100 of
-        true -> create_app(Name, Enable, ExpiredAt, Desc);
+        true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc);
         false -> {error, "Maximum ApiKey"}
     end.
 
@@ -157,8 +165,7 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT
 is_expired(undefined) -> false;
 is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second).
 
-create_app(Name, Enable, ExpiredAt, Desc) ->
-    ApiSecret = generate_api_secret(),
+create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
     App =
         #?APP{
             name = Name,
@@ -170,9 +177,10 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
             api_key = list_to_binary(emqx_misc:gen_id(16))
         },
     case create_app(App) of
-        {error, api_key_already_existed} -> create_app(Name, Enable, ExpiredAt, Desc);
-        {ok, Res} -> {ok, Res#{api_secret => ApiSecret}};
-        Error -> Error
+        {ok, Res} ->
+            {ok, Res#{api_secret => ApiSecret}};
+        Error ->
+            Error
     end.
 
 create_app(App = #?APP{api_key = ApiKey, name = Name}) ->

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -88,10 +88,9 @@ t_cluster_query(_Config) ->
 
         %% fuzzy searching can't return total
         {200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
-        ?assertMatch(
-            #{count := 0},
-            maps:get(meta, ClientsNode2)
-        ),
+        MetaNode2 = maps:get(meta, ClientsNode2),
+        ?assertNotMatch(#{count := _}, MetaNode2),
+        ?assertMatch(#{hasnext := false}, MetaNode2),
         ?assertMatch(10, length(maps:get(data, ClientsNode2))),
 
         _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),

apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl

@@ -225,21 +225,23 @@ t_create_unexpired_app(_Config) ->
     ok.
 
 list_app() ->
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
-    case emqx_mgmt_api_test_util:request_api(get, Path) of
+    case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of
         {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])};
         Error -> Error
     end.
 
 read_app(Name) ->
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
-    case emqx_mgmt_api_test_util:request_api(get, Path) of
+    case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of
         {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
         Error -> Error
     end.
 
 create_app(Name) ->
-    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
     ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000),
     App = #{
@@ -254,7 +256,7 @@ create_app(Name) ->
     end.
 
 create_unexpired_app(Name, Params) ->
-    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
     App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params),
     case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of
@@ -263,11 +265,12 @@ create_unexpired_app(Name, Params) ->
     end.
 
 delete_app(Name) ->
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
-    emqx_mgmt_api_test_util:request_api(delete, DeletePath).
+    emqx_mgmt_api_test_util:request_api(delete, DeletePath, AuthHeader).
 
 update_app(Name, Change) ->
-    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
     case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of
         {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])};

apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl

@@ -44,9 +44,8 @@ init_per_suite(Config) ->
 end_per_suite(_) ->
     emqx_mgmt_api_test_util:end_suite().
 
-t_subscription_api(_) ->
-    {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}),
-    {ok, _} = emqtt:connect(Client),
+t_subscription_api(Config) ->
+    Client = proplists:get_value(client, Config),
     {ok, _, _} = emqtt:subscribe(
         Client, [
             {?TOPIC1, [{rh, ?TOPIC1RH}, {rap, ?TOPIC1RAP}, {nl, ?TOPIC1NL}, {qos, ?TOPIC1QOS}]}
@@ -84,40 +83,78 @@ t_subscription_api(_) ->
     ?assertEqual(maps:get(<<"topic">>, Subscriptions2), ?TOPIC2),
     ?assertEqual(maps:get(<<"clientid">>, Subscriptions2), ?CLIENTID),
 
-    QS = uri_string:compose_query([
+    QS = [
         {"clientid", ?CLIENTID},
         {"topic", ?TOPIC2_TOPIC_ONLY},
         {"node", atom_to_list(node())},
         {"qos", "0"},
         {"share_group", "test_group"},
         {"match_topic", "t/#"}
-    ]),
+    ],
     Headers = emqx_mgmt_api_test_util:auth_header_(),
 
-    {ok, ResponseTopic2} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
-    DataTopic2 = emqx_json:decode(ResponseTopic2, [return_maps]),
-    Meta2 = maps:get(<<"meta">>, DataTopic2),
+    DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers),
     ?assertEqual(1, maps:get(<<"page">>, Meta2)),
     ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)),
     ?assertEqual(1, maps:get(<<"count">>, Meta2)),
     SubscriptionsList2 = maps:get(<<"data">>, DataTopic2),
-    ?assertEqual(length(SubscriptionsList2), 1),
+    ?assertEqual(length(SubscriptionsList2), 1).
 
-    MatchQs = uri_string:compose_query([
+t_subscription_fuzzy_search(Config) ->
+    Client = proplists:get_value(client, Config),
+    Topics = [
+        <<"t/foo">>,
+        <<"t/foo/bar">>,
+        <<"t/foo/baz">>,
+        <<"topic/foo/bar">>,
+        <<"topic/foo/baz">>
+    ],
+    _ = [{ok, _, _} = emqtt:subscribe(Client, T) || T <- Topics],
+
+    Headers = emqx_mgmt_api_test_util:auth_header_(),
+    MatchQs = [
         {"clientid", ?CLIENTID},
         {"node", atom_to_list(node())},
-        {"qos", "0"},
         {"match_topic", "t/#"}
-    ]),
+    ],
 
-    {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(get, Path, MatchQs, Headers),
-    MatchData = emqx_json:decode(MatchRes, [return_maps]),
-    MatchMeta = maps:get(<<"meta">>, MatchData),
-    ?assertEqual(1, maps:get(<<"page">>, MatchMeta)),
-    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta)),
-    %% count equals 0 in fuzzy searching
-    ?assertEqual(0, maps:get(<<"count">>, MatchMeta)),
-    MatchSubs = maps:get(<<"data">>, MatchData),
-    ?assertEqual(1, length(MatchSubs)),
+    MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers),
+    ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)),
+    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)),
+    %% count is undefined in fuzzy searching
+    ?assertNot(maps:is_key(<<"count">>, MatchMeta1)),
+    ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))),
+    ?assertEqual(false, maps:get(<<"hasnext">>, MatchMeta1)),
 
+    LimitMatchQuery = [
+        {"clientid", ?CLIENTID},
+        {"match_topic", "+/+/+"},
+        {"limit", "3"}
+    ],
+
+    MatchData2 = #{<<"meta">> := MatchMeta2} = request_json(get, LimitMatchQuery, Headers),
+    ?assertEqual(#{<<"page">> => 1, <<"limit">> => 3, <<"hasnext">> => true}, MatchMeta2),
+    ?assertEqual(3, length(maps:get(<<"data">>, MatchData2))),
+
+    MatchData2P2 =
+        #{<<"meta">> := MatchMeta2P2} =
+        request_json(get, [{"page", "2"} | LimitMatchQuery], Headers),
+    ?assertEqual(#{<<"page">> => 2, <<"limit">> => 3, <<"hasnext">> => false}, MatchMeta2P2),
+    ?assertEqual(1, length(maps:get(<<"data">>, MatchData2P2))).
+
+request_json(Method, Query, Headers) when is_list(Query) ->
+    Qs = uri_string:compose_query(Query),
+    {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers),
+    emqx_json:decode(MatchRes, [return_maps]).
+
+path() ->
+    emqx_mgmt_api_test_util:api_path(["subscriptions"]).
+
+init_per_testcase(_TC, Config) ->
+    {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}),
+    {ok, _} = emqtt:connect(Client),
+    [{client, Client} | Config].
+
+end_per_testcase(_TC, Config) ->
+    Client = proplists:get_value(client, Config),
     emqtt:disconnect(Client).

apps/emqx_management/test/emqx_mgmt_api_test_util.erl

@@ -24,14 +24,19 @@ init_suite() ->
     init_suite([]).
 
 init_suite(Apps) ->
+    init_suite(Apps, fun set_special_configs/1).
+
+init_suite(Apps, SetConfigs) ->
     mria:start(),
     application:load(emqx_management),
-    emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1).
+    emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], SetConfigs),
+    emqx_common_test_http:create_default_app().
 
 end_suite() ->
     end_suite([]).
 
 end_suite(Apps) ->
+    emqx_common_test_http:delete_default_app(),
     application:unload(emqx_management),
     emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]),
     emqx_config:delete_override_conf_files(),
@@ -43,8 +48,23 @@ set_special_configs(emqx_dashboard) ->
 set_special_configs(_App) ->
     ok.
 
+%% there is no difference between the 'request' and 'request_api'
+%% the 'request' is only to be compatible with the 'emqx_dashboard_api_test_helpers:request'
+request(Method, Url) ->
+    request(Method, Url, []).
+
+request(Method, Url, Body) ->
+    request_api_with_body(Method, Url, Body).
+
+uri(Parts) ->
+    emqx_dashboard_api_test_helpers:uri(Parts).
+
+%% compatible_mode will return as same as 'emqx_dashboard_api_test_helpers:request'
+request_api_with_body(Method, Url, Body) ->
+    request_api(Method, Url, [], auth_header_(), Body, #{compatible_mode => true}).
+
 request_api(Method, Url) ->
-    request_api(Method, Url, [], [], [], #{}).
+    request_api(Method, Url, auth_header_()).
 
 request_api(Method, Url, AuthOrHeaders) ->
     request_api(Method, Url, [], AuthOrHeaders, [], #{}).
@@ -90,10 +110,20 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when
 
 do_request_api(Method, Request, Opts) ->
     ReturnAll = maps:get(return_all, Opts, false),
+    CompatibleMode = maps:get(compatible_mode, Opts, false),
+    ReqOpts =
+        case CompatibleMode of
+            true ->
+                [{body_format, binary}];
+            _ ->
+                []
+        end,
     ct:pal("Method: ~p, Request: ~p", [Method, Request]),
-    case httpc:request(Method, Request, [], []) of
+    case httpc:request(Method, Request, [], ReqOpts) of
         {error, socket_closed_remotely} ->
             {error, socket_closed_remotely};
+        {ok, {{_, Code, _}, _Headers, Body}} when CompatibleMode ->
+            {ok, Code, Body};
         {ok, {{"HTTP/1.1", Code, _} = Reason, Headers, Body}} when
             Code >= 200 andalso Code =< 299 andalso ReturnAll
         ->
@@ -109,10 +139,7 @@ do_request_api(Method, Request, Opts) ->
     end.
 
 auth_header_() ->
-    Username = <<"admin">>,
-    Password = <<"public">>,
-    {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
-    {"Authorization", "Bearer " ++ binary_to_list(Token)}.
+    emqx_common_test_http:default_auth_header().
 
 build_http_header(X) when is_list(X) ->
     X;

apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl

@@ -30,6 +30,8 @@
 -define(API_VERSION, "v5").
 -define(BASE_PATH, "api").
 
+-import(emqx_dashboard_SUITE, [auth_header_/0]).
+
 %%--------------------------------------------------------------------
 %% Setups
 %%--------------------------------------------------------------------
@@ -330,13 +332,6 @@ t_stream_log(_Config) ->
 to_rfc3339(Second) ->
     list_to_binary(calendar:system_time_to_rfc3339(Second)).
 
-auth_header_() ->
-    auth_header_("admin", "public").
-
-auth_header_(User, Pass) ->
-    Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
-    {"Authorization", "Basic " ++ Encoded}.
-
 request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}).
 
 request_api(Method, Url, Auth, Body) ->

apps/emqx_modules/test/emqx_delayed_api_SUITE.erl

@@ -26,7 +26,7 @@
     <<"max_delayed_messages">> => <<"0">>
 }).
 
--import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -36,27 +36,21 @@ init_per_suite(Config) ->
         raw_with_default => true
     }),
 
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_modules, emqx_dashboard],
-        fun set_special_configs/1
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_modules]
     ),
     emqx_delayed:load(),
     Config.
 
 end_per_suite(Config) ->
     ok = emqx_delayed:unload(),
-    emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
     Config.
 
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(),
     Config.
 
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
-    ok.
-
 %%------------------------------------------------------------------------------
 %% Test Cases
 %%------------------------------------------------------------------------------

apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -37,20 +37,14 @@ init_per_suite(Config) ->
         raw_with_default => true
     }),
 
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_modules, emqx_dashboard],
-        fun set_special_configs/1
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_modules]
     ),
 
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
     ok.
 
 %%------------------------------------------------------------------------------
@@ -81,7 +75,7 @@ t_mqtt_topic_rewrite(_) ->
 
     ?assertEqual(
         Rules,
-        jsx:decode(Result)
+        emqx_json:decode(Result, [return_maps])
     ).
 
 t_mqtt_topic_rewrite_limit(_) ->

apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -33,8 +33,8 @@ init_per_suite(Config) ->
         raw_with_default => true
     }),
 
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_authn, emqx_authz, emqx_modules, emqx_dashboard],
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_authn, emqx_authz, emqx_modules],
         fun set_special_configs/1
     ),
 
@@ -49,8 +49,8 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    emqx_common_test_helpers:stop_apps([
-        emqx_dashboard, emqx_conf, emqx_authn, emqx_authz, emqx_modules
+    emqx_mgmt_api_test_util:end_suite([
+        emqx_conf, emqx_authn, emqx_authz, emqx_modules
     ]),
     ok.
 

apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -44,9 +44,8 @@ init_per_suite(Config) ->
         raw_with_default => true
     }),
 
-    ok = emqx_common_test_helpers:start_apps(
-        [emqx_conf, emqx_modules, emqx_dashboard],
-        fun set_special_configs/1
+    ok = emqx_mgmt_api_test_util:init_suite(
+        [emqx_conf, emqx_modules]
     ),
 
     %% When many tests run in an obscure order, it may occur that
@@ -59,15 +58,10 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
     application:stop(gen_rpc),
     ok.
 
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
-    ok.
-
 %%------------------------------------------------------------------------------
 %% Tests
 %%------------------------------------------------------------------------------
@@ -315,6 +309,3 @@ t_badrpc(_) ->
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------
-
-request(Method, Url) ->
-    request(Method, Url, []).

apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugin_libs, [
     {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.4"},
+    {vsn, "4.3.5"},
     {modules, []},
     {applications, [kernel, stdlib]},
     {env, []}

apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl

@@ -63,6 +63,8 @@
     can_topic_match_oneof/2
 ]).
 
+-export_type([tmpl_token/0]).
+
 -compile({no_auto_import, [float/1]}).
 
 -define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})").

apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf

@@ -24,6 +24,35 @@ emqx_prometheus_schema {
       zh: """数据推送间隔"""
     }
   }
+
+  headers {
+    desc {
+      en: """A list of HTTP Headers when pushing to Push Gateway.<br/>
+For example, <code> { Authorization = "some-authz-tokens"}</code>"""
+      zh: """推送到 Push Gateway 的 HTTP Headers 列表。<br/>
+例如，<code> { Authorization = "some-authz-tokens"}</code>"""
+    }
+  }
+
+  job_name {
+    desc {
+      en: """Job Name that is pushed to the Push Gateway. Available variables:<br/>
+- ${name}: Name of EMQX node.<br/>
+- ${host}: Host name of EMQX node.<br/>
+For example, when the EMQX node name is <code>emqx@127.0.0.1</code> then the <code>name</code> variable takes value <code>emqx</code> and the <code>host</code> variable takes value <code>127.0.0.1</code>.<br/>
+
+Default value is: <code>${name}/instance/${name}~${host}</code>
+"""
+      zh: """推送到 Push Gateway 的 Job 名称。可用变量为：<br/>
+- ${name}: EMQX 节点的名称。
+- ${host}: EMQX 节点主机名。
+
+例如，当 EMQX 节点名为 <code>emqx@127.0.0.1</code> 则 name 变量的值为 <code>emqx</code>，host 变量的值为 <code>127.0.0.1</code>。<br/>
+
+默认值为: <code>${name}/instance/${name}~${host}</code>"""
+    }
+  }
+
   enable {
     desc {
       en: """Turn Prometheus data pushing on or off"""

apps/emqx_prometheus/src/emqx_prometheus.app.src

@@ -2,7 +2,7 @@
 {application, emqx_prometheus, [
     {description, "Prometheus for EMQX"},
     % strict semver, bump manually!
-    {vsn, "5.0.3"},
+    {vsn, "5.0.4"},
     {modules, []},
     {registered, [emqx_prometheus_sup]},
     {applications, [kernel, stdlib, prometheus, emqx]},

apps/emqx_prometheus/src/emqx_prometheus.erl

@@ -98,8 +98,13 @@ handle_cast(_Msg, State) ->
     {noreply, State}.
 
 handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) ->
-    #{interval := Interval, push_gateway_server := Server} = opts(),
-    PushRes = push_to_push_gateway(Server),
+    #{
+        interval := Interval,
+        headers := Headers,
+        job_name := JobName,
+        push_gateway_server := Server
+    } = opts(),
+    PushRes = push_to_push_gateway(Server, Headers, JobName),
     NewTimer = ensure_timer(Interval),
     NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}),
     %% Data is too big, hibernate for saving memory and stop system monitor warning.
@@ -107,18 +112,27 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) ->
 handle_info(_Msg, State) ->
     {noreply, State}.
 
-push_to_push_gateway(Uri) ->
+push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) ->
     [Name, Ip] = string:tokens(atom_to_list(node()), "@"),
-    Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]),
+    JobName1 = emqx_placeholder:preproc_tmpl(JobName),
+    JobName2 = binary_to_list(
+        emqx_placeholder:proc_tmpl(
+            JobName1,
+            #{<<"name">> => Name, <<"host">> => Ip}
+        )
+    ),
+
+    Url = lists:concat([Uri, "/metrics/job/", JobName2]),
     Data = prometheus_text_format:format(),
-    case httpc:request(post, {Url, [], "text/plain", Data}, ?HTTP_OPTIONS, []) of
-        {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, _Body}} ->
+    case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of
+        {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} ->
             ok;
         Error ->
             ?SLOG(error, #{
                 msg => "post_to_push_gateway_failed",
                 error => Error,
-                url => Url
+                url => Url,
+                headers => Headers
             }),
             failed
     end.

apps/emqx_prometheus/src/emqx_prometheus_api.erl

@@ -121,6 +121,8 @@ prometheus_config_example() ->
         enable => true,
         interval => "15s",
         push_gateway_server => <<"http://127.0.0.1:9091">>,
+        headers => #{'header-name' => 'header-value'},
+        job_name => <<"${name}/instance/${name}~${host}">>,
         vm_dist_collector => enabled,
         mnesia_collector => enabled,
         vm_statistics_collector => enabled,

apps/emqx_prometheus/src/emqx_prometheus_schema.erl

@@ -25,7 +25,8 @@
     roots/0,
     fields/1,
     desc/1,
-    translation/1
+    translation/1,
+    convert_headers/1
 ]).
 
 namespace() -> "prometheus".
@@ -52,6 +53,26 @@ fields("prometheus") ->
                     desc => ?DESC(interval)
                 }
             )},
+        {headers,
+            ?HOCON(
+                list({string(), string()}),
+                #{
+                    default => #{},
+                    required => false,
+                    converter => fun ?MODULE:convert_headers/1,
+                    desc => ?DESC(headers)
+                }
+            )},
+        {job_name,
+            ?HOCON(
+                binary(),
+                #{
+                    default => <<"${name}/instance/${name}~${host}">>,
+                    required => true,
+                    desc => ?DESC(job_name)
+                }
+            )},
+
         {enable,
             ?HOCON(
                 boolean(),
@@ -126,6 +147,17 @@ fields("prometheus") ->
 desc("prometheus") -> ?DESC(prometheus);
 desc(_) -> undefined.
 
+convert_headers(Headers) when is_map(Headers) ->
+    maps:fold(
+        fun(K, V, Acc) ->
+            [{binary_to_list(K), binary_to_list(V)} | Acc]
+        end,
+        [],
+        Headers
+    );
+convert_headers(Headers) when is_list(Headers) ->
+    Headers.
+
 %% for CI test, CI don't load the whole emqx_conf_schema.
 translation(Name) ->
     emqx_conf_schema:translation(Name).

apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl

@@ -27,6 +27,8 @@
     "prometheus {\n"
     "  push_gateway_server = \"http://127.0.0.1:9091\"\n"
     "  interval = \"1s\"\n"
+    "  headers = { Authorization = \"some-authz-tokens\"}\n"
+    "  job_name = \"${name}~${host}\"\n"
     "  enable = true\n"
     "  vm_dist_collector = enabled\n"
     "  mnesia_collector = enabled\n"
@@ -85,6 +87,25 @@ t_collector_no_crash_test(_) ->
     prometheus_text_format:format(),
     ok.
 
+t_assert_push(_) ->
+    meck:new(httpc, [passthrough]),
+    Self = self(),
+    AssertPush = fun(Method, Req = {Url, Headers, ContentType, _Data}, HttpOpts, Opts) ->
+        ?assertEqual(post, Method),
+        ?assertMatch("http://127.0.0.1:9091/metrics/job/test~127.0.0.1", Url),
+        ?assertEqual([{"Authorization", "some-authz-tokens"}], Headers),
+        ?assertEqual("text/plain", ContentType),
+        Self ! pass,
+        meck:passthrough([Method, Req, HttpOpts, Opts])
+    end,
+    meck:expect(httpc, request, AssertPush),
+    ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus)),
+    receive
+        pass -> ok
+    after 2000 ->
+        ct:fail(assert_push_request_failed)
+    end.
+
 t_only_for_coverage(_) ->
     ?assertEqual("5.0.0", emqx_prometheus_proto_v1:introduced_in()),
     ok.

apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf

@@ -36,8 +36,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
 
   health_check_interval {
     desc {
-      en: """Health check interval, in milliseconds."""
-      zh: """健康检查间隔，单位毫秒。"""
+      en: """Health check interval."""
+      zh: """健康检查间隔。"""
     }
     label {
       en: """Health Check Interval"""
@@ -69,8 +69,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
 
   auto_restart_interval {
     desc {
-      en: """The auto restart interval after the resource is disconnected, in milliseconds."""
-      zh: """资源断开以后，自动重连的时间间隔，单位毫秒。"""
+      en: """The auto restart interval after the resource is disconnected."""
+      zh: """资源断开以后，自动重连的时间间隔。"""
     }
     label {
       en: """Auto Restart Interval"""

apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl

@@ -23,6 +23,7 @@
 
 -export([
     namespace/0,
+    tags/0,
     roots/0,
     fields/1,
     desc/1,
@@ -33,6 +34,9 @@
 
 namespace() -> rule_engine.
 
+tags() ->
+    [<<"Rule Engine">>].
+
 roots() -> ["rule_engine"].
 
 fields("rule_engine") ->

apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl

@@ -203,13 +203,7 @@ do_request_api(Method, Request) ->
     end.
 
 auth_header_() ->
-    AppId = <<"admin">>,
-    AppSecret = <<"public">>,
-    auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
-
-auth_header_(User, Pass) ->
-    Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
-    {"Authorization", "Basic " ++ Encoded}.
+    emqx_mgmt_api_test_util:auth_header_().
 
 api_path(Parts) ->
     ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).

bin/nodetool

@@ -24,12 +24,19 @@ main(Args) ->
         ["hocon" | Rest] ->
             %% forward the call to hocon_cli
             hocon_cli:main(Rest);
-        ["check_license_key", Key] ->
-            check_license(#{key => list_to_binary(Key)});
+        ["check_license_key", Key0] ->
+            Key = cleanup_key(Key0),
+            check_license(#{key => Key});
         _ ->
             do(Args)
     end.
 
+%% the key is a string (list) representation of a binary, so we need
+%% to remove the leading and trailing angle brackets.
+cleanup_key(Str0) ->
+    Str1 = iolist_to_binary(string:replace(Str0, "<<", "", leading)),
+    iolist_to_binary(string:replace(Str1, ">>", "", trailing)).
+
 do(Args) ->
     ok = do_with_halt(Args, "mnesia_dir", fun create_mnesia_dir/2),
     ok = do_with_halt(Args, "chkconfig", fun("-config", X) -> chkconfig(X) end),

changes/v5.0.15/feat-9586.en.md

@@ -0,0 +1 @@
+Basic auth is no longer allowed for API calls, must use API key instead.

changes/v5.0.15/feat-9586.zh.md

@@ -0,0 +1 @@
+API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。

changes/v5.0.15/feat-9722.en.md

@@ -0,0 +1,3 @@
+Add the following configuration options for Pushing metrics to Prometheus Push Gateway:
+- `headers`: Allows custom HTTP request headers.
+- `job_name`: allows to customize the name of the Job pushed to Push Gateway.

changes/v5.0.15/feat-9722.zh.md

@@ -0,0 +1,3 @@
+为 Prometheus 推送到 Push Gateway 新增以下配置项：
+- `headers`：允许自定义 HTTP 请求头。
+- `job_name`：允许自定义推送到 Push Gateway 的 Job 名称。

changes/v5.0.15/feat-9725.en.md

@@ -0,0 +1,11 @@
+Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets.
+This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。
+
+The functions of these two config are difficult to distinguish, which will lead to confusion.
+After this change, `auto_reconnect` will not be configurable (always be true), and the underlying
+drivers that support this config will automatically reconnect the abnormally disconnected
+connection every `2s`.
+
+And the config `resource_opts.auto_restart_interval` is still available for user.
+It is the time interval that emqx restarts the resource when the connection cannot be
+established for some reason.

changes/v5.0.15/feat-9725.zh.md

@@ -0,0 +1,8 @@
+从认证、鉴权和数据桥接功能中，删除 `auto_reconnect` 配置项，因为我们还有另一个功能类似的配置项：
+`resource_opts.auto_restart_interval`。
+
+这两个配置项的功能难以区分，会导致困惑。此修改之后，`auto_reconnect` 将不可配置(永远为 true)，
+支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。
+
+而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置，它是资源因为某些原因
+无法建立连接的时候，emqx 重新启动该资源的时间间隔。

changes/v5.0.15/fix-8718-en.md

@@ -0,0 +1 @@
+Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis.

changes/v5.0.15/fix-8718-zh.md

@@ -0,0 +1 @@
+密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。

changes/v5.0.15/fix-9626.en.md

@@ -0,0 +1,2 @@
+Return authorization settings with default values.
+The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard.

changes/v5.0.15/fix-9626.zh.md

@@ -0,0 +1,3 @@
+为授权设置 API 返回默认值。
+授权缓存默认为开启，但是在此修复前，因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失，
+使得在仪表盘配置页面中看起来是关闭了。

changes/v5.0.15/fix-9726-en.md

@@ -0,0 +1 @@
+Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response.

changes/v5.0.15/fix-9726-zh.md

@@ -0,0 +1 @@
+在此修复前，客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息，现在通过在响应中提供 `hasnext` 标志来解决这个问题。

deploy/docker/Dockerfile

@@ -1,6 +1,7 @@
 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
 ARG RUN_FROM=debian:11-slim
-FROM ${BUILD_FROM} AS builder
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder
 
 COPY . /emqx
 

deploy/docker/Dockerfile.alpine

@@ -1,6 +1,7 @@
 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1
 ARG RUN_FROM=alpine:3.15.1
-FROM ${BUILD_FROM} AS builder
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder
 
 RUN apk add --no-cache \
     autoconf \

lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf

@@ -86,4 +86,15 @@ emqx_ee_bridge_mongodb {
       zh: "桥接名称"
     }
   }
+
+  payload_template {
+    desc {
+      en: "The template for formatting the outgoing messages.  If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc."
+      zh: "用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。"
+    }
+    label: {
+      en: "Payload template"
+      zh: "有效载荷模板"
+    }
+  }
 }

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -61,9 +61,9 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, u
 resource_type(kafka) -> emqx_bridge_impl_kafka;
 resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
 resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub;
-resource_type(mongodb_rs) -> emqx_connector_mongo;
-resource_type(mongodb_sharded) -> emqx_connector_mongo;
-resource_type(mongodb_single) -> emqx_connector_mongo;
+resource_type(mongodb_rs) -> emqx_ee_connector_mongodb;
+resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb;
+resource_type(mongodb_single) -> emqx_ee_connector_mongodb;
 resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb;

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl

@@ -196,6 +196,14 @@ to_influx_lines(RawLines) ->
 converter_influx_line(Line, AccIn) ->
     case string:tokens(str(Line), " ") of
         [MeasurementAndTags, Fields, Timestamp] ->
+            append_influx_item(MeasurementAndTags, Fields, Timestamp, AccIn);
+        [MeasurementAndTags, Fields] ->
+            append_influx_item(MeasurementAndTags, Fields, undefined, AccIn);
+        _ ->
+            throw("Bad InfluxDB Line Protocol schema")
+    end.
+
+append_influx_item(MeasurementAndTags, Fields, Timestamp, Acc) ->
     {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
     [
         #{
@@ -204,24 +212,8 @@ converter_influx_line(Line, AccIn) ->
             fields => kv_pairs(string:tokens(Fields, ",")),
             timestamp => Timestamp
         }
-                | AccIn
-            ];
-        [MeasurementAndTags, Fields] ->
-            {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
-            %% TODO: fix here both here and influxdb driver.
-            %% Default value should evaluated by InfluxDB.
-            [
-                #{
-                    measurement => Measurement,
-                    tags => kv_pairs(Tags),
-                    fields => kv_pairs(string:tokens(Fields, ",")),
-                    timestamp => "${timestamp}"
-                }
-                | AccIn
-            ];
-        _ ->
-            throw("Bad InfluxDB Line Protocol schema")
-    end.
+        | Acc
+    ].
 
 split_measurement_and_tags(Subject) ->
     case string:tokens(Subject, ",") of

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl

@@ -37,7 +37,8 @@ roots() ->
 fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})},
-        {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}
+        {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})},
+        {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})}
     ];
 fields(mongodb_rs) ->
     emqx_connector_mongo:fields(rs) ++ fields("config");

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl

@@ -51,7 +51,6 @@ values(post) ->
         pool_size => 8,
         username => <<"root">>,
         password => <<"">>,
-        auto_reconnect => true,
         sql => ?DEFAULT_SQL,
         local_topic => <<"local/topic/#">>,
         resource_opts => #{

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl

@@ -53,7 +53,6 @@ values(post, Type) ->
         pool_size => 8,
         username => <<"root">>,
         password => <<"public">>,
-        auto_reconnect => true,
         sql => ?DEFAULT_SQL,
         local_topic => <<"local/topic/#">>,
         resource_opts => #{