diff --git a/.github/actions/docker-meta/action.yaml b/.github/actions/docker-meta/action.yaml
deleted file mode 100644
index 13ab21da6..000000000
--- a/.github/actions/docker-meta/action.yaml
+++ /dev/null
@@ -1,81 +0,0 @@
-name: 'Docker meta'
-inputs:
- profile:
- required: true
- type: string
- registry:
- required: true
- type: string
- arch:
- required: true
- type: string
- otp:
- required: true
- type: string
- elixir:
- required: false
- type: string
- default: ''
- builder_base:
- required: true
- type: string
- owner:
- required: true
- type: string
- docker_tags:
- required: true
- type: string
-
-outputs:
- emqx_name:
- description: "EMQX name"
- value: ${{ steps.pre-meta.outputs.emqx_name }}
- version:
- description: "docker image version"
- value: ${{ steps.meta.outputs.version }}
- tags:
- description: "docker image tags"
- value: ${{ steps.meta.outputs.tags }}
- labels:
- description: "docker image labels"
- value: ${{ steps.meta.outputs.labels }}
-
-runs:
- using: composite
- steps:
- - name: prepare for docker/metadata-action
- id: pre-meta
- shell: bash
- run: |
- emqx_name=${{ inputs.profile }}
- img_suffix=${{ inputs.arch }}
- img_labels="org.opencontainers.image.otp.version=${{ inputs.otp }}"
- if [ -n "${{ inputs.elixir }}" ]; then
- emqx_name="emqx-elixir"
- img_suffix="elixir-${{ inputs.arch }}"
- img_labels="org.opencontainers.image.elixir.version=${{ inputs.elixir }}\n${img_labels}"
- fi
- if [ "${{ inputs.profile }}" = "emqx" ]; then
- img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}"
- fi
- if [ "${{ inputs.profile }}" = "emqx-enterprise" ]; then
- img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}"
- fi
- if [[ "${{ inputs.builder_base }}" =~ "alpine" ]]; then
- img_suffix="${img_suffix}-alpine"
- fi
- echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT
- echo "img_suffix=${img_suffix}" >> $GITHUB_OUTPUT
- echo "img_labels=${img_labels}" >> $GITHUB_OUTPUT
- echo "img_name=${{ inputs.registry }}/${{ inputs.owner }}/${{ inputs.profile }}" >> $GITHUB_OUTPUT
- - uses: docker/metadata-action@v4
- id: meta
- with:
- images:
- ${{ steps.pre-meta.outputs.img_name }}
- flavor: |
- suffix=-${{ steps.pre-meta.outputs.img_suffix }}
- tags: |
- type=raw,value=${{ inputs.docker_tags }}
- labels:
- ${{ steps.pre-meta.outputs.img_labels }}
diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index 6e6ef1e0c..826a0f2ab 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -9,15 +9,17 @@ on:
tags:
- v*
- e*
- release:
- types:
- - published
+ - docker-latest-*
workflow_dispatch:
inputs:
branch_or_tag:
required: false
profile:
required: false
+ default: 'emqx'
+ is_latest:
+ required: false
+ default: false
jobs:
prepare:
@@ -26,10 +28,11 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
outputs:
- BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
- IS_DOCKER_LATEST: ${{ steps.get_profile.outputs.IS_DOCKER_LATEST }}
+ PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
+ EDITION: ${{ steps.get_profile.outputs.EDITION }}
+ IS_LATEST: ${{ steps.get_profile.outputs.IS_LATEST }}
IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
- DOCKER_TAG_VERSION: ${{ steps.get_profile.outputs.DOCKER_TAG_VERSION }}
+ VERSION: ${{ steps.get_profile.outputs.VERSION }}
steps:
- uses: actions/checkout@v3
@@ -45,14 +48,14 @@ jobs:
tag=${{ github.ref }}
# tag docker-latest-ce or docker-latest-ee
if git describe --tags --exact --match 'docker-latest-*' 2>/dev/null; then
- echo 'docker_latest=true due to docker-latest-* tag'
- docker_latest=true
- elif [ "${{ github.event_name }}" = "release" ]; then
- echo 'docker_latest=true due to release'
- docker_latest=true
+ echo 'is_latest=true due to docker-latest-* tag'
+ is_latest=true
+ elif [ "${{ inputs.is_latest }}" = "true" ]; then
+ echo 'is_latest=true due to manual input from workflow_dispatch'
+ is_latest=true
else
- echo 'docker_latest=false'
- docker_latest=false
+ echo 'is_latest=false'
+ is_latest=false
fi
if git describe --tags --match "[v|e]*" --exact; then
echo "This is an exact git tag, will publish images"
@@ -64,18 +67,20 @@ jobs:
case $tag in
refs/tags/v*)
PROFILE='emqx'
+ EDITION='Opensource'
;;
refs/tags/e*)
PROFILE=emqx-enterprise
+ EDITION='Enterprise'
;;
*)
PROFILE=${{ github.event.inputs.profile }}
case "$PROFILE" in
emqx)
- true
+ EDITION='Opensource'
;;
emqx-enterprise)
- true
+ EDITION='Enterprise'
;;
*)
echo "ERROR: Failed to resolve build profile"
@@ -85,14 +90,18 @@ jobs:
;;
esac
VSN="$(./pkg-vsn.sh "$PROFILE")"
- echo "Building $PROFILE image with tag $VSN (latest=$docker_latest)"
- echo "IS_DOCKER_LATEST=$docker_latest" >> $GITHUB_OUTPUT
+ echo "Building emqx/$PROFILE:$VSN image (latest=$is_latest)"
+ echo "Push = $is_exact"
+ echo "IS_LATEST=$is_latest" >> $GITHUB_OUTPUT
echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT
- echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT
- echo "DOCKER_TAG_VERSION=$VSN" >> $GITHUB_OUTPUT
+ echo "PROFILE=$PROFILE" >> $GITHUB_OUTPUT
+ echo "EDITION=$EDITION" >> $GITHUB_OUTPUT
+ echo "VERSION=$VSN" >> $GITHUB_OUTPUT
- name: get_all_deps
+ env:
+ PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
run: |
- make -C source deps-all
+ PROFILE=$PROFILE make -C source deps-$PROFILE
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
@@ -100,17 +109,17 @@ jobs:
path: source.zip
docker:
- runs-on: ${{ matrix.arch[1] }}
+ runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
- arch:
- - [amd64, ubuntu-20.04]
- - [arm64, aws-arm64]
profile:
- - ${{ needs.prepare.outputs.BUILD_PROFILE }}
+ - "${{ needs.prepare.outputs.PROFILE }}"
+ flavor:
+ - ''
+ - '-elixir'
registry:
- 'docker.io'
- 'public.ecr.aws'
@@ -128,9 +137,10 @@ jobs:
exclude: # TODO: publish enterprise to ecr too?
- registry: 'public.ecr.aws'
profile: emqx-enterprise
+ - flavor: '-elixir'
+ os: [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
+
steps:
- - uses: AutoModality/action-clean@v1
- if: matrix.arch[1] == 'aws-arm64'
- uses: actions/download-artifact@v3
with:
name: source
@@ -138,16 +148,17 @@ jobs:
- name: unzip source code
run: unzip -q source.zip
+ - uses: docker/setup-qemu-action@v2
- uses: docker/setup-buildx-action@v2
- - name: Login for docker.
+ - name: Login to hub.docker.com
uses: docker/login-action@v2
if: matrix.registry == 'docker.io'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- - name: Login for AWS ECR
+ - name: Login to AWS ECR
uses: docker/login-action@v2
if: matrix.registry == 'public.ecr.aws'
with:
@@ -155,230 +166,48 @@ jobs:
username: ${{ secrets.AWS_ACCESS_KEY_ID }}
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ecr: true
+ - name: prepare for docker/metadata-action
+ id: pre-meta
+ shell: bash
+ run: |
+ extra_labels=
+ img_suffix=
+ flavor="${{ matrix.flavor }}"
+ if [ "${{ matrix.flavor }}" = '-elixir' ]; then
+ img_suffix="-elixir"
+ extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}"
+ fi
+ if [[ "${{ matrix.os[0] }}" =~ "alpine" ]]; then
+ img_suffix="${img_suffix}-alpine"
+ fi
- - uses: ./source/.github/actions/docker-meta
+ echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT
+ echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT
+
+ - uses: docker/metadata-action@v4
id: meta
with:
- profile: ${{ matrix.profile }}
- registry: ${{ matrix.registry }}
- arch: ${{ matrix.arch[0] }}
- otp: ${{ matrix.otp }}
- builder_base: ${{ matrix.os[0] }}
- owner: ${{ github.repository_owner }}
- docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
+ images: |
+ ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
+ flavor: |
+ suffix=${{ steps.pre-meta.outputs.img_suffix }}
+ tags: |
+ type=raw,value=${{ needs.prepare.outputs.VERSION }}
+ type=raw,value=latest,enable=${{ needs.prepare.outputs.IS_LATEST }}
+ labels: |
+ org.opencontainers.image.otp.version=${{ matrix.otp }}
+ org.opencontainers.image.edition=${{ needs.prepare.outputs.EDITION }}
+ ${{ steps.pre-meta.outputs.extra_labels }}
- uses: docker/build-push-action@v3
with:
push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
pull: true
no-cache: true
- platforms: linux/${{ matrix.arch[0] }}
+ platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
- BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
- RUN_FROM=${{ matrix.os[1] }}
- EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
+ EMQX_NAME=${{ matrix.profile }}${{ matrix.flavor }}
file: source/${{ matrix.os[2] }}
context: source
-
- - name: Docker Hub Description
- if: matrix.registry == 'docker.io'
- uses: peter-evans/dockerhub-description@v3
- with:
- username: ${{ secrets.DOCKER_HUB_USER }}
- password: ${{ secrets.DOCKER_HUB_TOKEN }}
- repository: "emqx/${{ needs.prepare.outputs.BUILD_PROFILE }}"
- readme-filepath: ./source/deploy/docker/README.md
- short-description: "The most scalable open-source MQTT broker for IoT, IIoT, connected vehicles, and more."
-
- docker-elixir:
- runs-on: ${{ matrix.arch[1] }}
- needs: prepare
- # do not build elixir images for ee for now
- if: needs.prepare.outputs.BUILD_PROFILE == 'emqx'
-
- strategy:
- fail-fast: false
- matrix:
- arch:
- - [amd64, ubuntu-20.04]
- - [arm64, aws-arm64]
- profile:
- - ${{ needs.prepare.outputs.BUILD_PROFILE }}
- registry:
- - 'docker.io'
- os:
- - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
- builder:
- - 5.0-26 # update to latest
- otp:
- - 25.1.2-2 # update to latest
- elixir:
- - 1.13.4 # update to latest
-
- steps:
- - uses: AutoModality/action-clean@v1
- if: matrix.arch[1] == 'aws-arm64'
- - uses: actions/download-artifact@v3
- with:
- name: source
- path: .
- - name: unzip source code
- run: unzip -q source.zip
-
- - uses: docker/setup-buildx-action@v2
-
- - name: Login for docker.
- uses: docker/login-action@v2
- with:
- username: ${{ secrets.DOCKER_HUB_USER }}
- password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
- - uses: ./source/.github/actions/docker-meta
- id: meta
- with:
- profile: ${{ matrix.profile }}
- registry: ${{ matrix.registry }}
- arch: ${{ matrix.arch[0] }}
- otp: ${{ matrix.otp }}
- elixir: ${{ matrix.elixir }}
- builder_base: ${{ matrix.os[0] }}
- owner: ${{ github.repository_owner }}
- docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
- - uses: docker/build-push-action@v3
- with:
- push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
- pull: true
- no-cache: true
- platforms: linux/${{ matrix.arch[0] }}
- tags: ${{ steps.meta.outputs.tags }}
- labels: ${{ steps.meta.outputs.labels }}
- build-args: |
- BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
- RUN_FROM=${{ matrix.os[1] }}
- EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
- file: source/${{ matrix.os[2] }}
- context: source
-
- docker-push-multi-arch-manifest:
- # note, we only run on amd64
- if: needs.prepare.outputs.IS_EXACT_TAG
- needs:
- - prepare
- - docker
- runs-on: ${{ matrix.arch[1] }}
- strategy:
- fail-fast: false
- matrix:
- arch:
- - [amd64, ubuntu-20.04]
- profile:
- - ${{ needs.prepare.outputs.BUILD_PROFILE }}
- os:
- - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
- - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
- # NOTE: only support latest otp version, not a matrix
- otp:
- - 24.3.4.2-1 # switch to 25 once ready to release 5.1
- registry:
- - 'docker.io'
- - 'public.ecr.aws'
- exclude:
- - registry: 'public.ecr.aws'
- profile: emqx-enterprise
-
- steps:
- - uses: actions/download-artifact@v3
- with:
- name: source
- path: .
-
- - name: unzip source code
- run: unzip -q source.zip
-
- - uses: docker/login-action@v2
- if: matrix.registry == 'docker.io'
- with:
- username: ${{ secrets.DOCKER_HUB_USER }}
- password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
- - uses: docker/login-action@v2
- if: matrix.registry == 'public.ecr.aws'
- with:
- registry: public.ecr.aws
- username: ${{ secrets.AWS_ACCESS_KEY_ID }}
- password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- ecr: true
-
- - uses: ./source/.github/actions/docker-meta
- id: meta
- with:
- profile: ${{ matrix.profile }}
- registry: ${{ matrix.registry }}
- arch: ${{ matrix.arch[0] }}
- otp: ${{ matrix.otp }}
- builder_base: ${{ matrix.os[0] }}
- owner: ${{ github.repository_owner }}
- docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
- - name: update manifest for multiarch image
- working-directory: source
- run: |
- is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}"
- scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$is_latest"
-
- docker-elixir-push-multi-arch-manifest:
- # note, we only run on amd64
- # do not build enterprise elixir images for now
- if: needs.prepare.outputs.IS_EXACT_TAG == 'true' && needs.prepare.outputs.BUILD_PROFILE == 'emqx'
- needs:
- - prepare
- - docker-elixir
- runs-on: ${{ matrix.arch[1] }}
- strategy:
- fail-fast: false
- matrix:
- arch:
- - [amd64, ubuntu-20.04]
- profile:
- - ${{ needs.prepare.outputs.BUILD_PROFILE }}
- # NOTE: for docker, only support latest otp version, not a matrix
- otp:
- - 25.1.2-2 # update to latest
- elixir:
- - 1.13.4 # update to latest
- registry:
- - 'docker.io'
-
- steps:
- - uses: actions/download-artifact@v3
- with:
- name: source
- path: .
-
- - name: unzip source code
- run: unzip -q source.zip
-
- - uses: docker/login-action@v2
- with:
- username: ${{ secrets.DOCKER_HUB_USER }}
- password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
- - uses: ./source/.github/actions/docker-meta
- id: meta
- with:
- profile: ${{ matrix.profile }}
- registry: ${{ matrix.registry }}
- arch: ${{ matrix.arch[0] }}
- otp: ${{ matrix.otp }}
- elixir: ${{ matrix.elixir }}
- builder_base: ${{ matrix.os[0] }}
- owner: ${{ github.repository_owner }}
- docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}
-
- - name: update manifest for multiarch image
- working-directory: source
- run: |
- scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" false
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 9e44034fb..a54bb68dd 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -201,12 +201,25 @@ jobs:
echo "waiting emqx started";
sleep 10;
done
+ - name: Get Token
+ timeout-minutes: 1
+ run: |
+ kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &
+
+ while
+ [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ]
+ do
+ echo "waiting emqx"
+ sleep 1
+ done
+
+ echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
+
- name: Check cluster
timeout-minutes: 10
run: |
- kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &
while
- [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
+ [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
do
echo "waiting ${{ matrix.profile }} cluster scale"
sleep 1
diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml
index 6eaf4aa75..4ba246d8c 100644
--- a/.github/workflows/run_jmeter_tests.yaml
+++ b/.github/workflows/run_jmeter_tests.yaml
@@ -92,7 +92,7 @@ jobs:
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
- ref: broker-autotest-v2
+ ref: broker-autotest-v4
path: scripts
- uses: actions/setup-java@v3
with:
@@ -191,7 +191,7 @@ jobs:
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
- ref: broker-autotest-v2
+ ref: broker-autotest-v4
path: scripts
- uses: actions/setup-java@v3
with:
@@ -297,7 +297,7 @@ jobs:
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
- ref: broker-autotest-v2
+ ref: broker-autotest-v4
path: scripts
- uses: actions/setup-java@v3
with:
@@ -396,7 +396,7 @@ jobs:
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
- ref: broker-autotest-v2
+ ref: broker-autotest-v4
path: scripts
- name: run jwks_server
timeout-minutes: 10
@@ -496,7 +496,7 @@ jobs:
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
- ref: broker-autotest-v2
+ ref: broker-autotest-v4
path: scripts
- uses: actions/setup-java@v3
with:
diff --git a/Makefile b/Makefile
index 180028899..faa866753 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.1.5
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.9
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.12
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
ifeq ($(OS),Windows_NT)
diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 750c0c2cd..0665cfb09 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -2050,7 +2050,7 @@ base_listener_enable_authn {
Set true (default) to enable client authentication on this listener, the authentication
process goes through the configured authentication chain.
When set to false to allow any clients with or without authentication information such as username or password to log in.
-When set to quick_deny_anonymous, it behaves like when set to true but clients will be
+When set to quick_deny_anonymous, it behaves like when set to true, but clients will be
denied immediately without going through any authenticators if username is not provided. This is useful to fence off
anonymous clients early.
"""
diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl
index 858ce96ce..08dd08362 100644
--- a/apps/emqx/include/http_api.hrl
+++ b/apps/emqx/include/http_api.hrl
@@ -15,10 +15,8 @@
%%--------------------------------------------------------------------
%% HTTP API Auth
--define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD').
--define(WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET,
- 'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET'
-).
+-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
+-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
%% Bad Request
-define(BAD_REQUEST, 'BAD_REQUEST').
@@ -57,8 +55,8 @@
%% All codes
-define(ERROR_CODES, [
- {'WRONG_USERNAME_OR_PWD', <<"Wrong username or pwd">>},
- {'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET', <<"Wrong username & pwd or key & secret">>},
+ {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>},
+ {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>},
{'BAD_REQUEST', <<"Request parameters are not legal">>},
{'NOT_MATCH', <<"Conditions are not matched">>},
{'ALREADY_EXISTS', <<"Resource already existed">>},
diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl
index 7e9dbca77..4391a9a0b 100644
--- a/apps/emqx/src/emqx_hocon.erl
+++ b/apps/emqx/src/emqx_hocon.erl
@@ -21,7 +21,8 @@
format_path/1,
check/2,
format_error/1,
- format_error/2
+ format_error/2,
+ make_schema/1
]).
%% @doc Format hocon config field path to dot-separated string in iolist format.
@@ -79,6 +80,9 @@ format_error({_Schema, [#{kind := K} = First | Rest] = All}, Opts) when
format_error(_Other, _) ->
false.
+make_schema(Fields) ->
+ #{roots => Fields, fields => #{}}.
+
%% Ensure iolist()
iol(B) when is_binary(B) -> B;
iol(A) when is_atom(A) -> atom_to_binary(A, utf8);
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 4cd78b575..043b57b99 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -111,15 +111,19 @@
comma_separated_atoms/0
]).
--export([namespace/0, roots/0, roots/1, fields/1, desc/1]).
+-export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]).
-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]).
+-export([authz_fields/0]).
-export([sc/2, map/2]).
-elvis([{elvis_style, god_modules, disable}]).
namespace() -> broker.
+tags() ->
+ [<<"EMQX">>].
+
roots() ->
%% TODO change config importance to a field metadata
roots(high) ++ roots(medium) ++ roots(low).
@@ -323,31 +327,7 @@ fields("stats") ->
)}
];
fields("authorization") ->
- [
- {"no_match",
- sc(
- hoconsc:enum([allow, deny]),
- #{
- default => allow,
- required => true,
- desc => ?DESC(fields_authorization_no_match)
- }
- )},
- {"deny_action",
- sc(
- hoconsc:enum([ignore, disconnect]),
- #{
- default => ignore,
- required => true,
- desc => ?DESC(fields_authorization_deny_action)
- }
- )},
- {"cache",
- sc(
- ref(?MODULE, "cache"),
- #{}
- )}
- ];
+ authz_fields();
fields("cache") ->
[
{"enable",
@@ -2088,6 +2068,33 @@ do_default_ciphers(_) ->
%% otherwise resolve default ciphers list at runtime
[].
+authz_fields() ->
+ [
+ {"no_match",
+ sc(
+ hoconsc:enum([allow, deny]),
+ #{
+ default => allow,
+ required => true,
+ desc => ?DESC(fields_authorization_no_match)
+ }
+ )},
+ {"deny_action",
+ sc(
+ hoconsc:enum([ignore, disconnect]),
+ #{
+ default => ignore,
+ required => true,
+ desc => ?DESC(fields_authorization_deny_action)
+ }
+ )},
+ {"cache",
+ sc(
+ ref(?MODULE, "cache"),
+ #{}
+ )}
+ ].
+
%% @private return a list of keys in a parent field
-spec keys(string(), hocon:config()) -> [string()].
keys(Parent, Conf) ->
@@ -2342,7 +2349,7 @@ authentication(Which) ->
undefined -> hoconsc:array(typerefl:map());
Module -> Module:root_type()
end,
- %% It is a lazy type because when handing runtime update requests
+ %% It is a lazy type because when handling runtime update requests
%% the config is not checked by emqx_schema, but by the injected schema
Type = hoconsc:lazy(Type0),
#{
diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl
index 87a35a1e2..575bed5c3 100644
--- a/apps/emqx/test/emqx_common_test_http.erl
+++ b/apps/emqx/test/emqx_common_test_http.erl
@@ -29,6 +29,9 @@
auth_header/2
]).
+-define(DEFAULT_APP_ID, <<"default_appid">>).
+-define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
+
request_api(Method, Url, Auth) ->
request_api(Method, Url, [], Auth, []).
@@ -74,12 +77,18 @@ auth_header(User, Pass) ->
{"Authorization", "Basic " ++ Encoded}.
default_auth_header() ->
- AppId = <<"myappid">>,
- AppSecret = emqx_mgmt_auth:get_appsecret(AppId),
- auth_header(erlang:binary_to_list(AppId), erlang:binary_to_list(AppSecret)).
+ {ok, #{api_key := APIKey}} = emqx_mgmt_auth:read(?DEFAULT_APP_ID),
+ auth_header(
+ erlang:binary_to_list(APIKey), erlang:binary_to_list(?DEFAULT_APP_SECRET)
+ ).
create_default_app() ->
- emqx_mgmt_auth:add_app(<<"myappid">>, <<"test">>).
+ Now = erlang:system_time(second),
+ ExpiredAt = Now + timer:minutes(10),
+ emqx_mgmt_auth:create(
+ ?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">>
+ ),
+ ok.
delete_default_app() ->
- emqx_mgmt_auth:del_app(<<"myappid">>).
+ emqx_mgmt_auth:delete(?DEFAULT_APP_ID).
diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index ea21e5bdc..7f01d94c0 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authn, [
{description, "EMQX Authentication"},
- {vsn, "0.1.11"},
+ {vsn, "0.1.12"},
{modules, []},
{registered, [emqx_authn_sup, emqx_authn_registry]},
{applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl
index 88d8955c5..f40e759f0 100644
--- a/apps/emqx_authn/src/emqx_authn_schema.erl
+++ b/apps/emqx_authn/src/emqx_authn_schema.erl
@@ -22,6 +22,7 @@
-export([
common_fields/0,
roots/0,
+ tags/0,
fields/1,
authenticator_type/0,
authenticator_type_without_scram/0,
@@ -32,6 +33,9 @@
roots() -> [].
+tags() ->
+ [<<"Authentication">>].
+
common_fields() ->
[{enable, fun enable/1}].
diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
index ba13bd069..ac39e2cda 100644
--- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
+++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
@@ -25,6 +25,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -105,6 +106,9 @@ mnesia(boot) ->
namespace() -> "authn-scram-builtin_db".
+tags() ->
+ [<<"Authentication">>].
+
roots() -> [?CONF_NS].
fields(?CONF_NS) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
index 0a9aaa825..faa06b71a 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
@@ -26,6 +26,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1,
@@ -51,6 +52,9 @@
namespace() -> "authn-http".
+tags() ->
+ [<<"Authentication">>].
+
roots() ->
[
{?CONF_NS,
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
index 5709a1fe7..1c44b4d1f 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
@@ -25,6 +25,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -44,6 +45,9 @@
namespace() -> "authn-jwt".
+tags() ->
+ [<<"Authentication">>].
+
roots() ->
[
{?CONF_NS,
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
index e915744e1..7c51644b7 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
@@ -26,6 +26,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -107,6 +108,9 @@ mnesia(boot) ->
namespace() -> "authn-builtin_db".
+tags() ->
+ [<<"Authentication">>].
+
roots() -> [?CONF_NS].
fields(?CONF_NS) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
index 3fac0ed7d..3f140a8eb 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
@@ -25,6 +25,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -44,6 +45,9 @@
namespace() -> "authn-mongodb".
+tags() ->
+ [<<"Authentication">>].
+
roots() ->
[
{?CONF_NS,
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
index 68913443f..ffce42bb3 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
@@ -27,6 +27,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -46,6 +47,9 @@
namespace() -> "authn-mysql".
+tags() ->
+ [<<"Authentication">>].
+
roots() -> [?CONF_NS].
fields(?CONF_NS) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
index 1cadf9c56..2d7974301 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
@@ -26,6 +26,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -50,6 +51,9 @@
namespace() -> "authn-postgresql".
+tags() ->
+ [<<"Authentication">>].
+
roots() -> [?CONF_NS].
fields(?CONF_NS) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
index 0c8fedfb5..12b7422b5 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
@@ -25,6 +25,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1
@@ -44,6 +45,9 @@
namespace() -> "authn-redis".
+tags() ->
+ [<<"Authentication">>].
+
roots() ->
[
{?CONF_NS,
diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
index 1a867b0be..11e2c6773 100644
--- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
@@ -18,7 +18,8 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1, multipart_formdata_request/3]).
+-import(emqx_dashboard_api_test_helpers, [multipart_formdata_request/3]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
-include("emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
@@ -65,9 +66,8 @@ end_per_testcase(_, Config) ->
init_per_suite(Config) ->
emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY),
_ = application:load(emqx_conf),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_authn, emqx_dashboard],
- fun set_special_configs/1
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_authn]
),
?AUTHN:delete_chain(?GLOBAL),
@@ -76,12 +76,7 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
- ok.
-
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
+ emqx_mgmt_api_test_util:end_suite([emqx_authn]),
ok.
%%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
index 8191fe2e9..cd97a15d9 100644
--- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
@@ -197,7 +197,7 @@ t_list_users(_) ->
#{is_superuser := false, user_id := _},
#{is_superuser := false, user_id := _}
],
- meta := #{page := 1, limit := 2, count := 3}
+ meta := #{page := 1, limit := 2, count := 3, hasnext := true}
} = emqx_authn_mnesia:list_users(
#{<<"page">> => 1, <<"limit">> => 2},
State
@@ -205,7 +205,7 @@ t_list_users(_) ->
#{
data := [#{is_superuser := false, user_id := _}],
- meta := #{page := 2, limit := 2, count := 3}
+ meta := #{page := 2, limit := 2, count := 3, hasnext := false}
} = emqx_authn_mnesia:list_users(
#{<<"page">> => 2, <<"limit">> => 2},
State
@@ -213,7 +213,7 @@ t_list_users(_) ->
#{
data := [#{is_superuser := false, user_id := <<"u3">>}],
- meta := #{page := 1, limit := 20, count := 0}
+ meta := #{page := 1, limit := 20, hasnext := false}
} = emqx_authn_mnesia:list_users(
#{
<<"page">> => 1,
diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
index e1a2586cd..b143903b5 100644
--- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
@@ -300,14 +300,14 @@ t_list_users(_) ->
#{
data := [?USER_MAP, ?USER_MAP],
- meta := #{page := 1, limit := 2, count := 3}
+ meta := #{page := 1, limit := 2, count := 3, hasnext := true}
} = emqx_enhanced_authn_scram_mnesia:list_users(
#{<<"page">> => 1, <<"limit">> => 2},
State
),
#{
data := [?USER_MAP],
- meta := #{page := 2, limit := 2, count := 3}
+ meta := #{page := 2, limit := 2, count := 3, hasnext := false}
} = emqx_enhanced_authn_scram_mnesia:list_users(
#{<<"page">> => 2, <<"limit">> => 2},
State
@@ -319,7 +319,7 @@ t_list_users(_) ->
is_superuser := _
}
],
- meta := #{page := 1, limit := 3, count := 0}
+ meta := #{page := 1, limit := 3, hasnext := false}
} = emqx_enhanced_authn_scram_mnesia:list_users(
#{
<<"page">> => 1,
diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md
index 8c05f21be..af543e478 100644
--- a/apps/emqx_authz/README.md
+++ b/apps/emqx_authz/README.md
@@ -15,7 +15,6 @@ authz:{
pool_size: 1
username: root
password: public
- auto_reconnect: true
ssl: {
enable: true
cacertfile: "etc/certs/cacert.pem"
@@ -33,7 +32,6 @@ authz:{
pool_size: 1
username: root
password: public
- auto_reconnect: true
ssl: {enable: false}
}
sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}"
@@ -45,7 +43,6 @@ authz:{
database: 0
pool_size: 1
password: public
- auto_reconnect: true
ssl: {enable: false}
}
cmd: "HGETALL mqtt_authz:${username}"
diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf
index e7fd73498..3bdc180c5 100644
--- a/apps/emqx_authz/etc/emqx_authz.conf
+++ b/apps/emqx_authz/etc/emqx_authz.conf
@@ -1,6 +1,7 @@
authorization {
deny_action = ignore
no_match = allow
+ cache = { enable = true }
sources = [
{
type = file
diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl
index 72a2db35c..db915a795 100644
--- a/apps/emqx_authz/src/emqx_authz_api_settings.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl
@@ -64,7 +64,7 @@ schema("/authorization/settings") ->
}.
ref_authz_schema() ->
- proplists:delete(sources, emqx_conf_schema:fields("authorization")).
+ emqx_schema:authz_fields().
settings(get, _Params) ->
{200, authorization_settings()};
@@ -83,4 +83,6 @@ settings(put, #{
{200, authorization_settings()}.
authorization_settings() ->
- maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})).
+ C = maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})),
+ Schema = emqx_hocon:make_schema(emqx_schema:authz_fields()),
+ hocon_tconf:make_serializable(Schema, C, #{}).
diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl
index f5570f1f1..c692154b1 100644
--- a/apps/emqx_authz/src/emqx_authz_api_sources.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl
@@ -449,7 +449,7 @@ is_ok(ResL) ->
get_raw_sources() ->
RawSources = emqx:get_raw_config([authorization, sources], []),
- Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
+ Schema = emqx_hocon:make_schema(emqx_authz_schema:authz_fields()),
Conf = #{<<"sources">> => RawSources},
#{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}),
merge_default_headers(Sources).
diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl
index a684ae6ba..5527c26d6 100644
--- a/apps/emqx_authz/src/emqx_authz_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_schema.erl
@@ -33,9 +33,11 @@
-export([
namespace/0,
roots/0,
+ tags/0,
fields/1,
validations/0,
- desc/1
+ desc/1,
+ authz_fields/0
]).
-export([
@@ -65,28 +67,15 @@ type_names() ->
namespace() -> authz.
+tags() ->
+ [<<"Authorization">>].
+
%% @doc authorization schema is not exported
%% but directly used by emqx_schema
roots() -> [].
fields("authorization") ->
- Types = [?R_REF(Type) || Type <- type_names()],
- UnionMemberSelector =
- fun
- (all_union_members) -> Types;
- %% must return list
- ({value, Value}) -> [select_union_member(Value)]
- end,
- [
- {sources,
- ?HOCON(
- ?ARRAY(?UNION(UnionMemberSelector)),
- #{
- default => [],
- desc => ?DESC(sources)
- }
- )}
- ];
+ authz_fields();
fields(file) ->
authz_common_fields(file) ++
[{path, ?HOCON(string(), #{required => true, desc => ?DESC(path)})}];
@@ -488,3 +477,22 @@ select_union_member_loop(TypeValue, [Type | Types]) ->
false ->
select_union_member_loop(TypeValue, Types)
end.
+
+authz_fields() ->
+ Types = [?R_REF(Type) || Type <- type_names()],
+ UnionMemberSelector =
+ fun
+ (all_union_members) -> Types;
+ %% must return list
+ ({value, Value}) -> [select_union_member(Value)]
+ end,
+ [
+ {sources,
+ ?HOCON(
+ ?ARRAY(?UNION(UnionMemberSelector)),
+ #{
+ default => [],
+ desc => ?DESC(sources)
+ }
+ )}
+ ].
diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
index 24b8fe25e..45e6d7287 100644
--- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/2, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -32,8 +32,8 @@ groups() ->
[].
init_per_suite(Config) ->
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_authz, emqx_dashboard, emqx_management],
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_authz],
fun set_special_configs/1
),
Config.
@@ -47,7 +47,7 @@ end_per_suite(_Config) ->
<<"sources">> => []
}
),
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]),
+ emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
ok.
set_special_configs(emqx_dashboard) ->
@@ -67,12 +67,12 @@ t_clean_cahce(_) ->
ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0),
{ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
- ?assertEqual(2, length(jsx:decode(Result3))),
+ ?assertEqual(2, length(emqx_json:decode(Result3))),
request(delete, uri(["authorization", "cache"])),
{ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
- ?assertEqual(0, length(jsx:decode(Result4))),
+ ?assertEqual(0, length(emqx_json:decode(Result4))),
ok.
diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
index 62c747433..62bce770e 100644
--- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
@@ -22,7 +22,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
all() ->
emqx_common_test_helpers:all(?MODULE).
@@ -31,8 +31,8 @@ groups() ->
[].
init_per_suite(Config) ->
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_authz, emqx_dashboard],
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_authz],
fun set_special_configs/1
),
Config.
@@ -46,7 +46,7 @@ end_per_suite(_Config) ->
<<"sources">> => []
}
),
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+ emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
ok.
set_special_configs(emqx_dashboard) ->
@@ -92,7 +92,8 @@ t_api(_) ->
<<"meta">> := #{
<<"count">> := 1,
<<"limit">> := 100,
- <<"page">> := 1
+ <<"page">> := 1,
+ <<"hasnext">> := false
}
} = jsx:decode(Request1),
?assertEqual(3, length(Rules1)),
@@ -111,9 +112,9 @@ t_api(_) ->
#{
<<"data">> := [],
<<"meta">> := #{
- <<"count">> := 0,
<<"limit">> := 20,
- <<"page">> := 1
+ <<"page">> := 1,
+ <<"hasnext">> := false
}
} = jsx:decode(Request1_1),
diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
index 275b04e40..41eba109e 100644
--- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -30,7 +30,7 @@ groups() ->
[].
init_per_suite(Config) ->
- ok = emqx_common_test_helpers:start_apps(
+ ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_authz, emqx_dashboard],
fun set_special_configs/1
),
@@ -46,7 +46,7 @@ end_per_suite(_Config) ->
}
),
ok = stop_apps([emqx_resource]),
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+ emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
ok.
set_special_configs(emqx_dashboard) ->
diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
index 34638d0aa..76b025716 100644
--- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -115,8 +115,8 @@ init_per_suite(Config) ->
end
),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_authz, emqx_dashboard],
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_authz],
fun set_special_configs/1
),
ok = start_apps([emqx_resource]),
@@ -134,7 +134,7 @@ end_per_suite(_Config) ->
%% resource and connector should be stop first,
%% or authz_[mysql|pgsql|redis..]_SUITE would be failed
ok = stop_apps([emqx_resource]),
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
+ emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]),
meck:unload(emqx_resource),
ok.
diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
index 959f8ec1b..900f39ebb 100644
--- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
+++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
@@ -93,9 +93,8 @@ init_per_suite(Config) ->
" }"
>>
),
- emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_dashboard, ?APP],
- fun set_special_configs/1
+ emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, ?APP]
),
Config.
@@ -111,12 +110,6 @@ end_per_testcase(t_get_basic_usage_info, _Config) ->
end_per_testcase(_TestCase, _Config) ->
ok.
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config(),
- ok;
-set_special_configs(_) ->
- ok.
-
topic_config(T) ->
#{
topic => T,
@@ -132,7 +125,7 @@ end_per_suite(_) ->
application:unload(?APP),
meck:unload(emqx_resource),
meck:unload(emqx_schema),
- emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]).
+ emqx_mgmt_api_test_util:end_suite([emqx_conf, ?APP]).
t_auto_subscribe(_) ->
emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 1ad5d7aba..845c1ef90 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -20,7 +20,7 @@
-import(hoconsc, [mk/2, ref/2]).
--export([roots/0, fields/1, desc/1, namespace/0]).
+-export([roots/0, fields/1, desc/1, namespace/0, tags/0]).
-export([
get_response/0,
@@ -104,6 +104,9 @@ metrics_status_fields() ->
namespace() -> "bridge".
+tags() ->
+ [<<"Bridge">>].
+
roots() -> [bridges].
fields(bridges) ->
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index 557eced13..4d16f1692 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -60,9 +60,8 @@ init_per_suite(Config) ->
%% some testcases (may from other app) already get emqx_connector started
_ = application:stop(emqx_resource),
_ = application:stop(emqx_connector),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_rule_engine, emqx_bridge, emqx_dashboard],
- fun set_special_configs/1
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_rule_engine, emqx_bridge]
),
ok = emqx_common_test_helpers:load_config(
emqx_rule_engine_schema,
@@ -72,12 +71,7 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
- emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_bridge, emqx_dashboard]),
- ok.
-
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config(<<"bridge_admin">>);
-set_special_configs(_) ->
+ emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge]),
ok.
init_per_testcase(_, Config) ->
@@ -605,9 +599,6 @@ t_with_redact_update(_Config) ->
?assertEqual(Password, Value),
ok.
-request(Method, Url, Body) ->
- request(<<"bridge_admin">>, Method, Url, Body).
-
operation_path(node, Oper, BridgeID) ->
uri(["nodes", node(), "bridges", BridgeID, "operation", Oper]);
operation_path(cluster, Oper, BridgeID) ->
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index a7b388964..90af47aca 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -38,7 +38,9 @@
cipher/0
]).
--export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]).
+-export([
+ namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0
+]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]).
%% Static apps which merge their configs into the merged emqx.conf
@@ -67,6 +69,9 @@
%% root config should not have a namespace
namespace() -> undefined.
+tags() ->
+ [<<"EMQX">>].
+
roots() ->
PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY,
case persistent_term:get(PtKey, undefined) of
@@ -942,8 +947,8 @@ fields("log_burst_limit") ->
)}
];
fields("authorization") ->
- emqx_schema:fields("authorization") ++
- emqx_authz_schema:fields("authorization").
+ emqx_schema:authz_fields() ++
+ emqx_authz_schema:authz_fields().
desc("cluster") ->
?DESC("desc_cluster");
diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md
index 7ef3a8c4a..6baba29de 100644
--- a/apps/emqx_connector/README.md
+++ b/apps/emqx_connector/README.md
@@ -14,7 +14,7 @@ An MySQL connector can be used as following:
```
(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose().
[#{config =>
- #{auto_reconnect => true,cacertfile => [],certfile => [],
+ #{cacertfile => [],certfile => [],
database => "mqtt",keyfile => [],password => "public",
pool_size => 1,
server => {{127,0,0,1},3306},
diff --git a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf b/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf
index f5caf29c4..8f25e0352 100644
--- a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf
+++ b/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf
@@ -68,13 +68,13 @@ emqx_connector_schema_lib {
auto_reconnect {
desc {
- en: "Enable automatic reconnect to the database."
- zh: "自动重连数据库。"
+ en: "Deprecated. Enable automatic reconnect to the database."
+ zh: "已弃用。自动重连数据库。"
}
label: {
- en: "Auto Reconnect Database"
- zh: "自动重连数据库"
- }
+ en: "Deprecated. Auto Reconnect Database"
+ zh: "已弃用。自动重连数据库"
+ }
}
}
diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl
index 96b6ba4d6..82c946cfc 100644
--- a/apps/emqx_connector/include/emqx_connector.hrl
+++ b/apps/emqx_connector/include/emqx_connector.hrl
@@ -24,6 +24,8 @@
-define(REDIS_DEFAULT_PORT, 6379).
-define(PGSQL_DEFAULT_PORT, 5432).
+-define(AUTO_RECONNECT_INTERVAL, 2).
+
-define(SERVERS_DESC,
"A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`
"
"For each Node should be: "
diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config
index b84d87e38..ed0bc827d 100644
--- a/apps/emqx_connector/rebar.config
+++ b/apps/emqx_connector/rebar.config
@@ -12,7 +12,7 @@
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}},
%% NOTE: mind poolboy version when updating mongodb-erlang version
- {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.18"}}},
+ {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}},
%% NOTE: mind poolboy version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.5"}}},
%% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index a04850746..817cd44f6 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -209,7 +209,7 @@ on_start(
?SLOG(info, #{
msg => "starting_http_connector",
connector => InstId,
- config => Config
+ config => emqx_misc:redact(Config)
}),
{Transport, TransportOpts} =
case Scheme of
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 1cb65034d..82d622e09 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -59,14 +59,13 @@ on_start(
bind_password := BindPassword,
timeout := Timeout,
pool_size := PoolSize,
- auto_reconnect := AutoReconn,
ssl := SSL
} = Config
) ->
?SLOG(info, #{
msg => "starting_ldap_connector",
connector => InstId,
- config => Config
+ config => emqx_misc:redact(Config)
}),
Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS),
SslOpts =
@@ -86,11 +85,11 @@ on_start(
{bind_password, BindPassword},
{timeout, Timeout},
{pool_size, PoolSize},
- {auto_reconnect, reconn_interval(AutoReconn)}
+ {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of
- ok -> {ok, #{poolname => PoolName, auto_reconnect => AutoReconn}};
+ ok -> {ok, #{poolname => PoolName}};
{error, Reason} -> {error, Reason}
end.
@@ -129,9 +128,6 @@ on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = S
on_get_status(_InstId, _State) -> connected.
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
-
search(Conn, Base, Filter, Attributes) ->
eldap2:search(Conn, [
{base, Base},
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 0bcc39208..1b0bcf94d 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -155,7 +155,7 @@ on_start(
rs -> "starting_mongodb_replica_set_connector";
sharded -> "starting_mongodb_sharded_connector"
end,
- ?SLOG(info, #{msg => Msg, connector => InstId, config => Config}),
+ ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}),
NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config),
SslOpts =
case maps:get(enable, SSL) of
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 522f15ccf..585122539 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -15,6 +15,8 @@
%%--------------------------------------------------------------------
-module(emqx_connector_mqtt).
+-include("emqx_connector.hrl").
+
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
@@ -147,7 +149,7 @@ on_start(InstId, Conf) ->
?SLOG(info, #{
msg => "starting_mqtt_connector",
connector => InstanceId,
- config => Conf
+ config => emqx_misc:redact(Conf)
}),
BasicConf = basic_config(Conf),
BridgeConf = BasicConf#{
@@ -198,12 +200,10 @@ on_query_async(
?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplyFun, Args}).
-on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) ->
- AutoReconn = maps:get(auto_reconnect, Conf, true),
+on_get_status(_InstId, #{name := InstanceId}) ->
case emqx_connector_mqtt_worker:status(InstanceId) of
connected -> connected;
- _ when AutoReconn == true -> connecting;
- _ when AutoReconn == false -> disconnected
+ _ -> connecting
end.
ensure_mqtt_worker_started(InstanceId, BridgeConf) ->
@@ -236,7 +236,6 @@ make_forward_confs(FrowardConf) ->
basic_config(
#{
server := Server,
- reconnect_interval := ReconnIntv,
proto_ver := ProtoVer,
bridge_mode := BridgeMode,
clean_start := CleanStart,
@@ -252,7 +251,7 @@ basic_config(
%% 30s
connect_timeout => 30,
auto_reconnect => true,
- reconnect_interval => ReconnIntv,
+ reconnect_interval => ?AUTO_RECONNECT_INTERVAL,
proto_ver => ProtoVer,
%% Opening bridge_mode will form a non-standard mqtt connection message.
%% A load balancing server (such as haproxy) is often set up before the emqx broker server.
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 6c0ff7210..ea6a84240 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -52,7 +52,6 @@
-type state() ::
#{
poolname := atom(),
- auto_reconnect := boolean(),
prepare_statement := prepares(),
params_tokens := params_tokens(),
batch_inserts := sqls(),
@@ -84,8 +83,6 @@ on_start(
server := Server,
database := DB,
username := User,
- password := Password,
- auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL
} = Config
@@ -94,7 +91,7 @@ on_start(
?SLOG(info, #{
msg => "starting_mysql_connector",
connector => InstId,
- config => Config
+ config => emqx_misc:redact(Config)
}),
SslOpts =
case maps:get(enable, SSL) of
@@ -107,14 +104,14 @@ on_start(
{host, Host},
{port, Port},
{user, User},
- {password, Password},
+ {password, maps:get(password, Config, <<>>)},
{database, DB},
- {auto_reconnect, reconn_interval(AutoReconn)},
+ {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
{pool_size, PoolSize}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
Prepares = parse_prepare_sql(Config),
- State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
+ State = maps:merge(#{poolname => PoolName}, Prepares),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
ok ->
{ok, init_prepare(State)};
@@ -194,7 +191,7 @@ mysql_function(prepared_query) ->
mysql_function(_) ->
mysql_function(prepared_query).
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
+on_get_status(_InstId, #{poolname := Pool} = State) ->
case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
true ->
case do_check_prepares(State) of
@@ -205,10 +202,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State
{connected, NState};
{error, _Reason} ->
%% do not log error, it is logged in prepare_sql_to_conn
- conn_status(AutoReconn)
+ connecting
end;
false ->
- conn_status(AutoReconn)
+ connecting
end.
do_get_status(Conn) ->
@@ -227,11 +224,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, P
end.
%% ===================================================================
-conn_status(_AutoReconn = true) -> connecting;
-conn_status(_AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
connect(Options) ->
mysql:start_link(Options).
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 4b565a614..9965ff3b4 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -56,7 +56,6 @@
-type state() ::
#{
poolname := atom(),
- auto_reconnect := boolean(),
prepare_sql := prepares(),
params_tokens := params_tokens(),
prepare_statement := epgsql:statement()
@@ -87,8 +86,6 @@ on_start(
server := Server,
database := DB,
username := User,
- password := Password,
- auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL
} = Config
@@ -97,7 +94,7 @@ on_start(
?SLOG(info, #{
msg => "starting_postgresql_connector",
connector => InstId,
- config => Config
+ config => emqx_misc:redact(Config)
}),
SslOpts =
case maps:get(enable, SSL) of
@@ -113,14 +110,14 @@ on_start(
{host, Host},
{port, Port},
{username, User},
- {password, emqx_secret:wrap(Password)},
+ {password, emqx_secret:wrap(maps:get(password, Config, ""))},
{database, DB},
- {auto_reconnect, reconn_interval(AutoReconn)},
+ {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
{pool_size, PoolSize}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
Prepares = parse_prepare_sql(Config),
- InitState = #{poolname => PoolName, auto_reconnect => AutoReconn, prepare_statement => #{}},
+ InitState = #{poolname => PoolName, prepare_statement => #{}},
State = maps:merge(InitState, Prepares),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
ok ->
@@ -247,7 +244,7 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) ->
end,
Result.
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
+on_get_status(_InstId, #{poolname := Pool} = State) ->
case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
true ->
case do_check_prepares(State) of
@@ -258,10 +255,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State
{connected, NState};
false ->
%% do not log error, it is logged in prepare_sql_to_conn
- conn_status(AutoReconn)
+ connecting
end;
false ->
- conn_status(AutoReconn)
+ connecting
end.
do_get_status(Conn) ->
@@ -280,11 +277,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepare
end.
%% ===================================================================
-conn_status(_AutoReconn = true) -> connecting;
-conn_status(_AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
connect(Opts) ->
Host = proplists:get_value(host, Opts),
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 726af2d9b..7fdf9d28d 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -117,14 +117,13 @@ on_start(
#{
redis_type := Type,
pool_size := PoolSize,
- auto_reconnect := AutoReconn,
ssl := SSL
} = Config
) ->
?SLOG(info, #{
msg => "starting_redis_connector",
connector => InstId,
- config => Config
+ config => emqx_misc:redact(Config)
}),
ConfKey =
case Type of
@@ -142,7 +141,7 @@ on_start(
[
{pool_size, PoolSize},
{password, maps:get(password, Config, "")},
- {auto_reconnect, reconn_interval(AutoReconn)}
+ {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
] ++ Database ++ Servers,
Options =
case maps:get(enable, SSL) of
@@ -155,7 +154,7 @@ on_start(
[{ssl, false}]
end ++ [{sentinel, maps:get(sentinel, Config, undefined)}],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
- State = #{poolname => PoolName, type => Type, auto_reconnect => AutoReconn},
+ State = #{poolname => PoolName, type => Type},
case Type of
cluster ->
case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of
@@ -229,18 +228,18 @@ eredis_cluster_workers_exist_and_are_connected(Workers) ->
Workers
).
-on_get_status(_InstId, #{type := cluster, poolname := PoolName, auto_reconnect := AutoReconn}) ->
+on_get_status(_InstId, #{type := cluster, poolname := PoolName}) ->
case eredis_cluster:pool_exists(PoolName) of
true ->
Workers = extract_eredis_cluster_workers(PoolName),
Health = eredis_cluster_workers_exist_and_are_connected(Workers),
- status_result(Health, AutoReconn);
+ status_result(Health);
false ->
disconnected
end;
-on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn}) ->
+on_get_status(_InstId, #{poolname := Pool}) ->
Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1),
- status_result(Health, AutoReconn).
+ status_result(Health).
do_get_status(Conn) ->
case eredis:q(Conn, ["PING"]) of
@@ -248,12 +247,8 @@ do_get_status(Conn) ->
_ -> false
end.
-status_result(_Status = true, _AutoReconn) -> connected;
-status_result(_Status = false, _AutoReconn = true) -> connecting;
-status_result(_Status = false, _AutoReconn = false) -> disconnected.
-
-reconn_interval(true) -> 15;
-reconn_interval(false) -> false.
+status_result(_Status = true) -> connected;
+status_result(_Status = false) -> connecting.
do_cmd(PoolName, cluster, {cmd, Command}) ->
eredis_cluster:q(PoolName, Command);
diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index 2364aeeaa..5d8f6941c 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -106,4 +106,5 @@ password(_) -> undefined.
auto_reconnect(type) -> boolean();
auto_reconnect(desc) -> ?DESC("auto_reconnect");
auto_reconnect(default) -> true;
+auto_reconnect(deprecated) -> {since, "v5.0.15"};
auto_reconnect(_) -> undefined.
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl
index f15467658..36c7660cc 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard.erl
@@ -65,8 +65,12 @@ start_listeners(Listeners) ->
components => #{
schemas => #{},
'securitySchemes' => #{
- 'basicAuth' => #{type => http, scheme => basic},
- 'bearerAuth' => #{type => http, scheme => bearer}
+ 'basicAuth' => #{
+ type => http,
+ scheme => basic,
+ description =>
+ <<"Authorize with [API Keys](https://www.emqx.io/docs/en/v5.0/admin/api.html#api-keys)">>
+ }
}
}
},
@@ -215,28 +219,7 @@ listener_name(Protocol) ->
authorize(Req) ->
case cowboy_req:parse_header(<<"authorization">>, Req) of
{basic, Username, Password} ->
- case emqx_dashboard_admin:check(Username, Password) of
- ok ->
- ok;
- {error, <<"username_not_found">>} ->
- Path = cowboy_req:path(Req),
- case emqx_mgmt_auth:authorize(Path, Username, Password) of
- ok ->
- ok;
- {error, <<"not_allowed">>} ->
- return_unauthorized(
- ?WRONG_USERNAME_OR_PWD,
- <<"Check username/password">>
- );
- {error, _} ->
- return_unauthorized(
- ?WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET,
- <<"Check username/password or api_key/api_secret">>
- )
- end;
- {error, _} ->
- return_unauthorized(?WRONG_USERNAME_OR_PWD, <<"Check username/password">>)
- end;
+ api_key_authorize(Req, Username, Password);
{bearer, Token} ->
case emqx_dashboard_admin:verify_token(Token) of
ok ->
@@ -269,3 +252,20 @@ i18n_file() ->
listeners() ->
emqx_conf:get([dashboard, listeners], []).
+
+api_key_authorize(Req, Key, Secret) ->
+ Path = cowboy_req:path(Req),
+ case emqx_mgmt_auth:authorize(Path, Key, Secret) of
+ ok ->
+ ok;
+ {error, <<"not_allowed">>} ->
+ return_unauthorized(
+ ?BAD_API_KEY_OR_SECRET,
+ <<"Not allowed, Check api_key/api_secret">>
+ );
+ {error, _} ->
+ return_unauthorized(
+ ?BAD_API_KEY_OR_SECRET,
+ <<"Check api_key/api_secret">>
+ )
+ end.
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
index 9facac59c..a4322c696 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
@@ -47,7 +47,7 @@
-define(EMPTY(V), (V == undefined orelse V == <<>>)).
--define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD').
+-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
-define(WRONG_TOKEN_OR_USERNAME, 'WRONG_TOKEN_OR_USERNAME').
-define(USER_NOT_FOUND, 'USER_NOT_FOUND').
-define(ERROR_PWD_NOT_MATCH, 'ERROR_PWD_NOT_MATCH').
@@ -164,7 +164,7 @@ schema("/users/:username/change_pwd") ->
}.
response_schema(401) ->
- emqx_dashboard_swagger:error_codes([?WRONG_USERNAME_OR_PWD], ?DESC(login_failed401));
+ emqx_dashboard_swagger:error_codes([?BAD_USERNAME_OR_PWD], ?DESC(login_failed401));
response_schema(404) ->
emqx_dashboard_swagger:error_codes([?USER_NOT_FOUND], ?DESC(users_api404)).
@@ -223,7 +223,7 @@ login(post, #{body := Params}) ->
}};
{error, R} ->
?SLOG(info, #{msg => "Dashboard login failed", username => Username, reason => R}),
- {401, ?WRONG_USERNAME_OR_PWD, <<"Auth failed">>}
+ {401, ?BAD_USERNAME_OR_PWD, <<"Auth failed">>}
end.
logout(_, #{
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 102b95f4e..0afbf5f1e 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -139,14 +139,20 @@ fields(limit) ->
[{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}];
fields(count) ->
Desc = <<
- "Total number of records counted.
"
- "Note: this field is 0 when the queryed table is empty, "
- "or if the query can not be optimized and requires a full table scan."
+ "Total number of records matching the query.
"
+ "Note: this field is present only if the query can be optimized and does "
+ "not require a full table scan."
+ >>,
+ Meta = #{desc => Desc, required => false},
+ [{count, hoconsc:mk(non_neg_integer(), Meta)}];
+fields(hasnext) ->
+ Desc = <<
+ "Flag indicating whether there are more results available on next pages."
>>,
Meta = #{desc => Desc, required => true},
- [{count, hoconsc:mk(non_neg_integer(), Meta)}];
+ [{hasnext, hoconsc:mk(boolean(), Meta)}];
fields(meta) ->
- fields(page) ++ fields(limit) ++ fields(count).
+ fields(page) ++ fields(limit) ++ fields(count) ++ fields(hasnext).
-spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map().
schema_with_example(Type, Example) ->
@@ -708,6 +714,8 @@ typename_to_spec("qos()", _Mod) ->
#{type => integer, minimum => 0, maximum => 2, example => 0};
typename_to_spec("{binary(), binary()}", _Mod) ->
#{type => object, example => #{}};
+typename_to_spec("{string(), string()}", _Mod) ->
+ #{type => object, example => #{}};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_binary()", _Mod) ->
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
index 934d6055d..23d1b40c1 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
@@ -114,9 +114,9 @@ t_admin_delete_self_failed(_) ->
?assertEqual(1, length(Admins)),
Header = auth_header_(<<"username1">>, <<"password">>),
{error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header),
- Token = erlang:iolist_to_binary(["Basic ", base64:encode("username1:password")]),
+ Token = ["Basic ", base64:encode("username1:password")],
Header2 = {"Authorization", Token},
- {error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2),
+ {error, {_, 401, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2),
mnesia:clear_table(?ADMIN).
t_rest_api(_Config) ->
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl
index b7fbf889e..a9b448662 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl
@@ -25,43 +25,24 @@
-define(SERVER, "http://127.0.0.1:18083/api/v5").
+-import(emqx_mgmt_api_test_util, [request/2]).
+
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
mria:start(),
- application:load(emqx_dashboard),
- emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
+ emqx_mgmt_api_test_util:init_suite([emqx_conf]),
Config.
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config(),
- ok;
-set_special_configs(_) ->
- ok.
-
end_per_suite(Config) ->
end_suite(),
Config.
end_suite() ->
- application:unload(emqx_management),
- emqx_common_test_helpers:stop_apps([emqx_dashboard]).
+ emqx_mgmt_api_test_util:end_suite([emqx_conf]).
t_bad_api_path(_) ->
Url = ?SERVER ++ "/for/test/some/path/not/exist",
- {error, {"HTTP/1.1", 404, "Not Found"}} = request(Url),
+ {ok, 404, _} = request(get, Url),
ok.
-
-request(Url) ->
- Request = {Url, []},
- case httpc:request(get, Request, [], []) of
- {error, Reason} ->
- {error, Reason};
- {ok, {{"HTTP/1.1", Code, _}, _, Return}} when
- Code >= 200 andalso Code =< 299
- ->
- {ok, emqx_json:decode(Return, [return_maps])};
- {ok, {Reason, _, _}} ->
- {error, Reason}
- end.
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
index 6f4a0e0fd..7d4980320 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
@@ -19,6 +19,8 @@
-compile(nowarn_export_all).
-compile(export_all).
+-import(emqx_dashboard_SUITE, [auth_header_/0]).
+
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx.hrl").
@@ -153,10 +155,6 @@ do_request_api(Method, Request) ->
{error, Reason}
end.
-auth_header_() ->
- Basic = binary_to_list(base64:encode(<<"admin:public">>)),
- {"Authorization", "Basic " ++ Basic}.
-
restart_monitor() ->
OldMonitor = erlang:whereis(emqx_dashboard_monitor),
erlang:exit(OldMonitor, kill),
diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl
index 7be940a53..8a4fb7a44 100644
--- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl
@@ -347,13 +347,7 @@ do_request_api(Method, Request) ->
end.
auth_header_() ->
- AppId = <<"admin">>,
- AppSecret = <<"public">>,
- auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
-
-auth_header_(User, Pass) ->
- Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
- {"Authorization", "Basic " ++ Encoded}.
+ emqx_mgmt_api_test_util:auth_header_().
api_path(Parts) ->
?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index e89280f14..804e1f862 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -49,12 +49,15 @@
]).
-elvis([{elvis_style, dont_repeat_yourself, disable}]).
--export([namespace/0, roots/0, fields/1, desc/1]).
+-export([namespace/0, roots/0, fields/1, desc/1, tags/0]).
-export([proxy_protocol_opts/0]).
namespace() -> gateway.
+tags() ->
+ [<<"Gateway">>].
+
roots() -> [gateway].
fields(gateway) ->
diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
index a6791a36b..deb602bc7 100644
--- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
@@ -106,8 +106,6 @@ assert_fields_exist(Ks, Map) ->
%% http
-define(http_api_host, "http://127.0.0.1:18083/api/v5").
--define(default_user, "admin").
--define(default_pass, "public").
request(delete = Mth, Path) ->
do_request(Mth, req(Path, []));
@@ -176,5 +174,4 @@ url(Path, Qs) ->
lists:concat([?http_api_host, Path, "?", binary_to_list(cow_qs:qs(Qs))]).
auth(Headers) ->
- Token = base64:encode(?default_user ++ ":" ++ ?default_pass),
- [{"Authorization", "Basic " ++ binary_to_list(Token)}] ++ Headers.
+ [emqx_mgmt_api_test_util:auth_header_() | Headers].
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 893007ebf..ba232daf3 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -20,7 +20,6 @@
-elvis([{elvis_style, dont_repeat_yourself, #{min_complexity => 100}}]).
--define(FRESH_SELECT, fresh_select).
-define(LONG_QUERY_TIMEOUT, 50000).
-export([
@@ -174,13 +173,12 @@ do_node_query(
case do_query(Node, QueryState) of
{error, {badrpc, R}} ->
{error, Node, {badrpc, R}};
- {Rows, NQueryState = #{continuation := ?FRESH_SELECT}} ->
- {_, NResultAcc} = accumulate_query_rows(Node, Rows, NQueryState, ResultAcc),
- NResultAcc;
- {Rows, NQueryState} ->
+ {Rows, NQueryState = #{complete := Complete}} ->
case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
{enough, NResultAcc} ->
- NResultAcc;
+ finalize_query(NResultAcc, NQueryState);
+ {_, NResultAcc} when Complete ->
+ finalize_query(NResultAcc, NQueryState);
{more, NResultAcc} ->
do_node_query(Node, NQueryState, NResultAcc)
end
@@ -212,8 +210,8 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
end.
%% @private
-do_cluster_query([], _QueryState, ResultAcc) ->
- ResultAcc;
+do_cluster_query([], QueryState, ResultAcc) ->
+ finalize_query(ResultAcc, mark_complete(QueryState));
do_cluster_query(
[Node | Tail] = Nodes,
QueryState,
@@ -222,31 +220,29 @@ do_cluster_query(
case do_query(Node, QueryState) of
{error, {badrpc, R}} ->
{error, Node, {badrpc, R}};
- {Rows, NQueryState} ->
+ {Rows, NQueryState = #{complete := Complete}} ->
case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
{enough, NResultAcc} ->
- maybe_collect_total_from_tail_nodes(Tail, NQueryState, NResultAcc);
+ FQueryState = maybe_collect_total_from_tail_nodes(Tail, NQueryState),
+ FComplete = Complete andalso Tail =:= [],
+ finalize_query(NResultAcc, mark_complete(FQueryState, FComplete));
+ {more, NResultAcc} when not Complete ->
+ do_cluster_query(Nodes, NQueryState, NResultAcc);
+ {more, NResultAcc} when Tail =/= [] ->
+ do_cluster_query(Tail, reset_query_state(NQueryState), NResultAcc);
{more, NResultAcc} ->
- NextNodes =
- case NQueryState of
- #{continuation := ?FRESH_SELECT} -> Tail;
- _ -> Nodes
- end,
- do_cluster_query(NextNodes, NQueryState, NResultAcc)
+ finalize_query(NResultAcc, NQueryState)
end
end.
-maybe_collect_total_from_tail_nodes([], _QueryState, ResultAcc) ->
- ResultAcc;
-maybe_collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) ->
- case counting_total_fun(QueryState) of
- false ->
- ResultAcc;
- _Fun ->
- collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc)
- end.
+maybe_collect_total_from_tail_nodes([], QueryState) ->
+ QueryState;
+maybe_collect_total_from_tail_nodes(Nodes, QueryState = #{total := _}) ->
+ collect_total_from_tail_nodes(Nodes, QueryState);
+maybe_collect_total_from_tail_nodes(_Nodes, QueryState) ->
+ QueryState.
-collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc}) ->
+collect_total_from_tail_nodes(Nodes, QueryState = #{total := TotalAcc}) ->
%% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
case rpc:multicall(Nodes, ?MODULE, apply_total_query, [QueryState], ?LONG_QUERY_TIMEOUT) of
{_, [Node | _]} ->
@@ -257,7 +253,8 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
[{Node, {badrpc, Reason}} | _] ->
{error, Node, {badrpc, Reason}};
[] ->
- ResultAcc#{total => ResL ++ TotalAcc}
+ NTotalAcc = maps:merge(TotalAcc, maps:from_list(ResL)),
+ QueryState#{total := NTotalAcc}
end
end.
@@ -266,13 +263,14 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
%%--------------------------------------------------------------------
%% QueryState ::
-%% #{continuation := ets:continuation(),
+%% #{continuation => ets:continuation(),
%% page := pos_integer(),
%% limit := pos_integer(),
-%% total := [{node(), non_neg_integer()}],
+%% total => #{node() => non_neg_integer()},
%% table := atom(),
-%% qs := {Qs, Fuzzy} %% parsed query params
-%% msfun := query_to_match_spec_fun()
+%% qs := {Qs, Fuzzy}, %% parsed query params
+%% msfun := query_to_match_spec_fun(),
+%% complete := boolean()
%% }
init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
#{match_spec := Ms, fuzzy_fun := FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
@@ -285,17 +283,31 @@ init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) -
true = is_list(Args),
{type, external} = erlang:fun_info(NamedFun, type)
end,
- #{
+ QueryState = #{
page => Page,
limit => Limit,
table => Tab,
qs => QString,
msfun => MsFun,
- mactch_spec => Ms,
+ match_spec => Ms,
fuzzy_fun => FuzzyFun,
- total => [],
- continuation => ?FRESH_SELECT
- }.
+ complete => false
+ },
+ case counting_total_fun(QueryState) of
+ false ->
+ QueryState;
+ Fun when is_function(Fun) ->
+ QueryState#{total => #{}}
+ end.
+
+reset_query_state(QueryState) ->
+ maps:remove(continuation, mark_complete(QueryState, false)).
+
+mark_complete(QueryState) ->
+ mark_complete(QueryState, true).
+
+mark_complete(QueryState, Complete) ->
+ QueryState#{complete => Complete}.
%% @private This function is exempt from BPAPI
do_query(Node, QueryState) when Node =:= node() ->
@@ -318,47 +330,50 @@ do_select(
Node,
QueryState0 = #{
table := Tab,
- mactch_spec := Ms,
- fuzzy_fun := FuzzyFun,
- continuation := Continuation,
- limit := Limit
+ match_spec := Ms,
+ limit := Limit,
+ complete := false
}
) ->
QueryState = maybe_apply_total_query(Node, QueryState0),
Result =
- case Continuation of
- ?FRESH_SELECT ->
+ case maps:get(continuation, QueryState, undefined) of
+ undefined ->
ets:select(Tab, Ms, Limit);
- _ ->
+ Continuation ->
%% XXX: Repair is necessary because we pass Continuation back
%% and forth through the nodes in the `do_cluster_query`
ets:select(ets:repair_continuation(Continuation, Ms))
end,
case Result of
- '$end_of_table' ->
- {[], QueryState#{continuation => ?FRESH_SELECT}};
+ {Rows, '$end_of_table'} ->
+ NRows = maybe_apply_fuzzy_filter(Rows, QueryState),
+ {NRows, mark_complete(QueryState)};
{Rows, NContinuation} ->
- NRows =
- case FuzzyFun of
- undefined ->
- Rows;
- {FilterFun, Args0} when is_function(FilterFun), is_list(Args0) ->
- lists:filter(
- fun(E) -> erlang:apply(FilterFun, [E | Args0]) end,
- Rows
- )
- end,
- {NRows, QueryState#{continuation => NContinuation}}
+ NRows = maybe_apply_fuzzy_filter(Rows, QueryState),
+ {NRows, QueryState#{continuation => NContinuation}};
+ '$end_of_table' ->
+ {[], mark_complete(QueryState)}
end.
-maybe_apply_total_query(Node, QueryState = #{total := TotalAcc}) ->
- case proplists:get_value(Node, TotalAcc, undefined) of
- undefined ->
- Total = apply_total_query(QueryState),
- QueryState#{total := [{Node, Total} | TotalAcc]};
- _ ->
- QueryState
- end.
+maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := undefined}) ->
+ Rows;
+maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := {FilterFun, Args}}) ->
+ lists:filter(
+ fun(E) -> erlang:apply(FilterFun, [E | Args]) end,
+ Rows
+ ).
+
+maybe_apply_total_query(Node, QueryState = #{total := Acc}) ->
+ case Acc of
+ #{Node := _} ->
+ QueryState;
+ #{} ->
+ NodeTotal = apply_total_query(QueryState),
+ QueryState#{total := Acc#{Node => NodeTotal}}
+ end;
+maybe_apply_total_query(_Node, QueryState = #{}) ->
+ QueryState.
apply_total_query(QueryState = #{table := Tab}) ->
case counting_total_fun(QueryState) of
@@ -371,7 +386,7 @@ apply_total_query(QueryState = #{table := Tab}) ->
counting_total_fun(_QueryState = #{qs := {[], []}}) ->
fun(Tab) -> ets:info(Tab, size) end;
-counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) ->
+counting_total_fun(_QueryState = #{match_spec := Ms, fuzzy_fun := undefined}) ->
%% XXX: Calculating the total number of data that match a certain
%% condition under a large table is very expensive because the
%% entire ETS table needs to be scanned.
@@ -390,15 +405,16 @@ counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= und
%% ResultAcc :: #{count := integer(),
%% cursor := integer(),
%% rows := [{node(), Rows :: list()}],
-%% total := [{node() => integer()}]
+%% overflow := boolean(),
+%% hasnext => boolean()
%% }
init_query_result() ->
- #{cursor => 0, count => 0, rows => [], total => []}.
+ #{cursor => 0, count => 0, rows => [], overflow => false}.
accumulate_query_rows(
Node,
Rows,
- _QueryState = #{page := Page, limit := Limit, total := TotalAcc},
+ _QueryState = #{page := Page, limit := Limit},
ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc}
) ->
PageStart = (Page - 1) * Limit + 1,
@@ -406,24 +422,35 @@ accumulate_query_rows(
Len = length(Rows),
case Cursor + Len of
NCursor when NCursor < PageStart ->
- {more, ResultAcc#{cursor => NCursor, total => TotalAcc}};
+ {more, ResultAcc#{cursor => NCursor}};
NCursor when NCursor < PageEnd ->
+ SubRows = lists:nthtail(max(0, PageStart - Cursor - 1), Rows),
{more, ResultAcc#{
cursor => NCursor,
- count => Count + length(Rows),
- total => TotalAcc,
- rows => [{Node, Rows} | RowsAcc]
+ count => Count + length(SubRows),
+ rows => [{Node, SubRows} | RowsAcc]
}};
NCursor when NCursor >= PageEnd ->
SubRows = lists:sublist(Rows, Limit - Count),
{enough, ResultAcc#{
cursor => NCursor,
count => Count + length(SubRows),
- total => TotalAcc,
- rows => [{Node, SubRows} | RowsAcc]
+ rows => [{Node, SubRows} | RowsAcc],
+ % there are more rows than can fit in the page
+ overflow => (Limit - Count) < Len
}}
end.
+finalize_query(Result = #{overflow := Overflow}, QueryState = #{complete := Complete}) ->
+ HasNext = Overflow orelse not Complete,
+ maybe_accumulate_totals(Result#{hasnext => HasNext}, QueryState).
+
+maybe_accumulate_totals(Result, #{total := TotalAcc}) ->
+ QueryTotal = maps:fold(fun(_Node, T, N) -> N + T end, 0, TotalAcc),
+ Result#{total => QueryTotal};
+maybe_accumulate_totals(Result, _QueryState) ->
+ Result.
+
%%--------------------------------------------------------------------
%% Internal Functions
%%--------------------------------------------------------------------
@@ -520,16 +547,22 @@ is_fuzzy_key(<<"match_", _/binary>>) ->
is_fuzzy_key(_) ->
false.
-format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) ->
+format_query_result(_FmtFun, _MetaIn, Error = {error, _Node, _Reason}) ->
Error;
format_query_result(
- FmtFun, Meta, _ResultAcc = #{total := TotalAcc, rows := RowsAcc}
+ FmtFun, MetaIn, ResultAcc = #{hasnext := HasNext, rows := RowsAcc}
) ->
- Total = lists:foldr(fun({_Node, T}, N) -> N + T end, 0, TotalAcc),
+ Meta =
+ case ResultAcc of
+ #{total := QueryTotal} ->
+ %% The `count` is used in HTTP API to indicate the total number of
+ %% queries that can be read
+ MetaIn#{hasnext => HasNext, count => QueryTotal};
+ #{} ->
+ MetaIn#{hasnext => HasNext}
+ end,
#{
- %% The `count` is used in HTTP API to indicate the total number of
- %% queries that can be read
- meta => Meta#{count => Total},
+ meta => Meta,
data => lists:flatten(
lists:foldl(
fun({Node, Rows}, Acc) ->
@@ -552,7 +585,7 @@ parse_pager_params(Params) ->
Limit = b2i(limit(Params)),
case Page > 0 andalso Limit > 0 of
true ->
- #{page => Page, limit => Limit, count => 0};
+ #{page => Page, limit => Limit};
false ->
false
end.
diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl
index 0bf849d3c..6f2a27414 100644
--- a/apps/emqx_management/src/emqx_mgmt_auth.erl
+++ b/apps/emqx_management/src/emqx_mgmt_auth.erl
@@ -40,6 +40,10 @@
do_force_create_app/3
]).
+-ifdef(TEST).
+-export([create/5]).
+-endif.
+
-define(APP, emqx_app).
-record(?APP, {
@@ -68,8 +72,12 @@ init_bootstrap_file() ->
init_bootstrap_file(File).
create(Name, Enable, ExpiredAt, Desc) ->
+ ApiSecret = generate_api_secret(),
+ create(Name, ApiSecret, Enable, ExpiredAt, Desc).
+
+create(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
case mnesia:table_info(?APP, size) < 100 of
- true -> create_app(Name, Enable, ExpiredAt, Desc);
+ true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc);
false -> {error, "Maximum ApiKey"}
end.
@@ -157,8 +165,7 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT
is_expired(undefined) -> false;
is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second).
-create_app(Name, Enable, ExpiredAt, Desc) ->
- ApiSecret = generate_api_secret(),
+create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
App =
#?APP{
name = Name,
@@ -170,9 +177,10 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
api_key = list_to_binary(emqx_misc:gen_id(16))
},
case create_app(App) of
- {error, api_key_already_existed} -> create_app(Name, Enable, ExpiredAt, Desc);
- {ok, Res} -> {ok, Res#{api_secret => ApiSecret}};
- Error -> Error
+ {ok, Res} ->
+ {ok, Res#{api_secret => ApiSecret}};
+ Error ->
+ Error
end.
create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index a14305d8b..a8bbfa6d9 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -88,10 +88,9 @@ t_cluster_query(_Config) ->
%% fuzzy searching can't return total
{200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
- ?assertMatch(
- #{count := 0},
- maps:get(meta, ClientsNode2)
- ),
+ MetaNode2 = maps:get(meta, ClientsNode2),
+ ?assertNotMatch(#{count := _}, MetaNode2),
+ ?assertMatch(#{hasnext := false}, MetaNode2),
?assertMatch(10, length(maps:get(data, ClientsNode2))),
_ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),
diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl
index 079351538..241a73dc4 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl
@@ -225,21 +225,23 @@ t_create_unexpired_app(_Config) ->
ok.
list_app() ->
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
- case emqx_mgmt_api_test_util:request_api(get, Path) of
+ case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of
{ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])};
Error -> Error
end.
read_app(Name) ->
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
- case emqx_mgmt_api_test_util:request_api(get, Path) of
+ case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of
{ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
end.
create_app(Name) ->
- AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000),
App = #{
@@ -254,7 +256,7 @@ create_app(Name) ->
end.
create_unexpired_app(Name, Params) ->
- AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of
@@ -263,11 +265,12 @@ create_unexpired_app(Name, Params) ->
end.
delete_app(Name) ->
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
- emqx_mgmt_api_test_util:request_api(delete, DeletePath).
+ emqx_mgmt_api_test_util:request_api(delete, DeletePath, AuthHeader).
update_app(Name, Change) ->
- AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+ AuthHeader = emqx_dashboard_SUITE:auth_header_(),
UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of
{ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])};
diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
index 965ed0997..2ab213e30 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
@@ -44,9 +44,8 @@ init_per_suite(Config) ->
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite().
-t_subscription_api(_) ->
- {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}),
- {ok, _} = emqtt:connect(Client),
+t_subscription_api(Config) ->
+ Client = proplists:get_value(client, Config),
{ok, _, _} = emqtt:subscribe(
Client, [
{?TOPIC1, [{rh, ?TOPIC1RH}, {rap, ?TOPIC1RAP}, {nl, ?TOPIC1NL}, {qos, ?TOPIC1QOS}]}
@@ -84,40 +83,78 @@ t_subscription_api(_) ->
?assertEqual(maps:get(<<"topic">>, Subscriptions2), ?TOPIC2),
?assertEqual(maps:get(<<"clientid">>, Subscriptions2), ?CLIENTID),
- QS = uri_string:compose_query([
+ QS = [
{"clientid", ?CLIENTID},
{"topic", ?TOPIC2_TOPIC_ONLY},
{"node", atom_to_list(node())},
{"qos", "0"},
{"share_group", "test_group"},
{"match_topic", "t/#"}
- ]),
+ ],
Headers = emqx_mgmt_api_test_util:auth_header_(),
- {ok, ResponseTopic2} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
- DataTopic2 = emqx_json:decode(ResponseTopic2, [return_maps]),
- Meta2 = maps:get(<<"meta">>, DataTopic2),
+ DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers),
?assertEqual(1, maps:get(<<"page">>, Meta2)),
?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)),
?assertEqual(1, maps:get(<<"count">>, Meta2)),
SubscriptionsList2 = maps:get(<<"data">>, DataTopic2),
- ?assertEqual(length(SubscriptionsList2), 1),
+ ?assertEqual(length(SubscriptionsList2), 1).
- MatchQs = uri_string:compose_query([
+t_subscription_fuzzy_search(Config) ->
+ Client = proplists:get_value(client, Config),
+ Topics = [
+ <<"t/foo">>,
+ <<"t/foo/bar">>,
+ <<"t/foo/baz">>,
+ <<"topic/foo/bar">>,
+ <<"topic/foo/baz">>
+ ],
+ _ = [{ok, _, _} = emqtt:subscribe(Client, T) || T <- Topics],
+
+ Headers = emqx_mgmt_api_test_util:auth_header_(),
+ MatchQs = [
{"clientid", ?CLIENTID},
{"node", atom_to_list(node())},
- {"qos", "0"},
{"match_topic", "t/#"}
- ]),
+ ],
- {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(get, Path, MatchQs, Headers),
- MatchData = emqx_json:decode(MatchRes, [return_maps]),
- MatchMeta = maps:get(<<"meta">>, MatchData),
- ?assertEqual(1, maps:get(<<"page">>, MatchMeta)),
- ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta)),
- %% count equals 0 in fuzzy searching
- ?assertEqual(0, maps:get(<<"count">>, MatchMeta)),
- MatchSubs = maps:get(<<"data">>, MatchData),
- ?assertEqual(1, length(MatchSubs)),
+ MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers),
+ ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)),
+ ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)),
+ %% count is undefined in fuzzy searching
+ ?assertNot(maps:is_key(<<"count">>, MatchMeta1)),
+ ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))),
+ ?assertEqual(false, maps:get(<<"hasnext">>, MatchMeta1)),
+ LimitMatchQuery = [
+ {"clientid", ?CLIENTID},
+ {"match_topic", "+/+/+"},
+ {"limit", "3"}
+ ],
+
+ MatchData2 = #{<<"meta">> := MatchMeta2} = request_json(get, LimitMatchQuery, Headers),
+ ?assertEqual(#{<<"page">> => 1, <<"limit">> => 3, <<"hasnext">> => true}, MatchMeta2),
+ ?assertEqual(3, length(maps:get(<<"data">>, MatchData2))),
+
+ MatchData2P2 =
+ #{<<"meta">> := MatchMeta2P2} =
+ request_json(get, [{"page", "2"} | LimitMatchQuery], Headers),
+ ?assertEqual(#{<<"page">> => 2, <<"limit">> => 3, <<"hasnext">> => false}, MatchMeta2P2),
+ ?assertEqual(1, length(maps:get(<<"data">>, MatchData2P2))).
+
+request_json(Method, Query, Headers) when is_list(Query) ->
+ Qs = uri_string:compose_query(Query),
+ {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers),
+ emqx_json:decode(MatchRes, [return_maps]).
+
+path() ->
+ emqx_mgmt_api_test_util:api_path(["subscriptions"]).
+
+init_per_testcase(_TC, Config) ->
+ {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}),
+ {ok, _} = emqtt:connect(Client),
+ [{client, Client} | Config].
+
+end_per_testcase(_TC, Config) ->
+ Client = proplists:get_value(client, Config),
emqtt:disconnect(Client).
diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl
index 5bb0ba818..82d55bb6a 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl
@@ -24,14 +24,19 @@ init_suite() ->
init_suite([]).
init_suite(Apps) ->
+ init_suite(Apps, fun set_special_configs/1).
+
+init_suite(Apps, SetConfigs) ->
mria:start(),
application:load(emqx_management),
- emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1).
+ emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], SetConfigs),
+ emqx_common_test_http:create_default_app().
end_suite() ->
end_suite([]).
end_suite(Apps) ->
+ emqx_common_test_http:delete_default_app(),
application:unload(emqx_management),
emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]),
emqx_config:delete_override_conf_files(),
@@ -43,8 +48,23 @@ set_special_configs(emqx_dashboard) ->
set_special_configs(_App) ->
ok.
+%% there is no difference between the 'request' and 'request_api'
+%% the 'request' is only to be compatible with the 'emqx_dashboard_api_test_helpers:request'
+request(Method, Url) ->
+ request(Method, Url, []).
+
+request(Method, Url, Body) ->
+ request_api_with_body(Method, Url, Body).
+
+uri(Parts) ->
+ emqx_dashboard_api_test_helpers:uri(Parts).
+
+%% compatible_mode will return as same as 'emqx_dashboard_api_test_helpers:request'
+request_api_with_body(Method, Url, Body) ->
+ request_api(Method, Url, [], auth_header_(), Body, #{compatible_mode => true}).
+
request_api(Method, Url) ->
- request_api(Method, Url, [], [], [], #{}).
+ request_api(Method, Url, auth_header_()).
request_api(Method, Url, AuthOrHeaders) ->
request_api(Method, Url, [], AuthOrHeaders, [], #{}).
@@ -90,10 +110,20 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when
do_request_api(Method, Request, Opts) ->
ReturnAll = maps:get(return_all, Opts, false),
+ CompatibleMode = maps:get(compatible_mode, Opts, false),
+ ReqOpts =
+ case CompatibleMode of
+ true ->
+ [{body_format, binary}];
+ _ ->
+ []
+ end,
ct:pal("Method: ~p, Request: ~p", [Method, Request]),
- case httpc:request(Method, Request, [], []) of
+ case httpc:request(Method, Request, [], ReqOpts) of
{error, socket_closed_remotely} ->
{error, socket_closed_remotely};
+ {ok, {{_, Code, _}, _Headers, Body}} when CompatibleMode ->
+ {ok, Code, Body};
{ok, {{"HTTP/1.1", Code, _} = Reason, Headers, Body}} when
Code >= 200 andalso Code =< 299 andalso ReturnAll
->
@@ -109,10 +139,7 @@ do_request_api(Method, Request, Opts) ->
end.
auth_header_() ->
- Username = <<"admin">>,
- Password = <<"public">>,
- {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
- {"Authorization", "Bearer " ++ binary_to_list(Token)}.
+ emqx_common_test_http:default_auth_header().
build_http_header(X) when is_list(X) ->
X;
diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
index 8e8c5b06f..0ba05b280 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
@@ -30,6 +30,8 @@
-define(API_VERSION, "v5").
-define(BASE_PATH, "api").
+-import(emqx_dashboard_SUITE, [auth_header_/0]).
+
%%--------------------------------------------------------------------
%% Setups
%%--------------------------------------------------------------------
@@ -330,13 +332,6 @@ t_stream_log(_Config) ->
to_rfc3339(Second) ->
list_to_binary(calendar:system_time_to_rfc3339(Second)).
-auth_header_() ->
- auth_header_("admin", "public").
-
-auth_header_(User, Pass) ->
- Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
- {"Authorization", "Basic " ++ Encoded}.
-
request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}).
request_api(Method, Url, Auth, Body) ->
diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl
index 96cdf7840..ed3cd9292 100644
--- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl
@@ -26,7 +26,7 @@
<<"max_delayed_messages">> => <<"0">>
}).
--import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
all() ->
emqx_common_test_helpers:all(?MODULE).
@@ -36,27 +36,21 @@ init_per_suite(Config) ->
raw_with_default => true
}),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_modules, emqx_dashboard],
- fun set_special_configs/1
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_modules]
),
emqx_delayed:load(),
Config.
end_per_suite(Config) ->
ok = emqx_delayed:unload(),
- emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
+ emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
Config.
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(),
Config.
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
- ok.
-
%%------------------------------------------------------------------------------
%% Test Cases
%%------------------------------------------------------------------------------
diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl
index 90e90d788..ddb136f1e 100644
--- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -37,20 +37,14 @@ init_per_suite(Config) ->
raw_with_default => true
}),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_modules, emqx_dashboard],
- fun set_special_configs/1
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_modules]
),
Config.
end_per_suite(_Config) ->
- emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
- ok.
-
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
+ emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
ok.
%%------------------------------------------------------------------------------
@@ -81,7 +75,7 @@ t_mqtt_topic_rewrite(_) ->
?assertEqual(
Rules,
- jsx:decode(Result)
+ emqx_json:decode(Result, [return_maps])
).
t_mqtt_topic_rewrite_limit(_) ->
diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl
index 288a155d9..16f942bc0 100644
--- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -33,8 +33,8 @@ init_per_suite(Config) ->
raw_with_default => true
}),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_authn, emqx_authz, emqx_modules, emqx_dashboard],
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_authn, emqx_authz, emqx_modules],
fun set_special_configs/1
),
@@ -49,8 +49,8 @@ end_per_suite(_Config) ->
<<"sources">> => []
}
),
- emqx_common_test_helpers:stop_apps([
- emqx_dashboard, emqx_conf, emqx_authn, emqx_authz, emqx_modules
+ emqx_mgmt_api_test_util:end_suite([
+ emqx_conf, emqx_authn, emqx_authz, emqx_modules
]),
ok.
diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl
index 8c23d042c..ea85d1fe9 100644
--- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl
@@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
--import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -44,9 +44,8 @@ init_per_suite(Config) ->
raw_with_default => true
}),
- ok = emqx_common_test_helpers:start_apps(
- [emqx_conf, emqx_modules, emqx_dashboard],
- fun set_special_configs/1
+ ok = emqx_mgmt_api_test_util:init_suite(
+ [emqx_conf, emqx_modules]
),
%% When many tests run in an obscure order, it may occur that
@@ -59,15 +58,10 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
- emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]),
+ emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]),
application:stop(gen_rpc),
ok.
-set_special_configs(emqx_dashboard) ->
- emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(_App) ->
- ok.
-
%%------------------------------------------------------------------------------
%% Tests
%%------------------------------------------------------------------------------
@@ -315,6 +309,3 @@ t_badrpc(_) ->
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
-
-request(Method, Url) ->
- request(Method, Url, []).
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
index bcdcfe420..3120b8503 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_plugin_libs, [
{description, "EMQX Plugin utility libs"},
- {vsn, "4.3.4"},
+ {vsn, "4.3.5"},
{modules, []},
{applications, [kernel, stdlib]},
{env, []}
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index 57bdd16e5..969374309 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -63,6 +63,8 @@
can_topic_match_oneof/2
]).
+-export_type([tmpl_token/0]).
+
-compile({no_auto_import, [float/1]}).
-define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})").
diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf
index 7f251ff4b..f25e35219 100644
--- a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf
+++ b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf
@@ -24,6 +24,35 @@ emqx_prometheus_schema {
zh: """数据推送间隔"""
}
}
+
+ headers {
+ desc {
+ en: """A list of HTTP Headers when pushing to Push Gateway.
+For example, { Authorization = "some-authz-tokens"}"""
+ zh: """推送到 Push Gateway 的 HTTP Headers 列表。
+例如， { Authorization = "some-authz-tokens"}"""
+ }
+ }
+
+ job_name {
+ desc {
+ en: """Job Name that is pushed to the Push Gateway. Available variables:
+- ${name}: Name of EMQX node.
+- ${host}: Host name of EMQX node.
+For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
+
+Default value is: ${name}/instance/${name}~${host}
+"""
+ zh: """推送到 Push Gateway 的 Job 名称。可用变量为：
+- ${name}: EMQX 节点的名称。
+- ${host}: EMQX 节点主机名。
+
+例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
+
+默认值为: ${name}/instance/${name}~${host}"""
+ }
+ }
+
enable {
desc {
en: """Turn Prometheus data pushing on or off"""
diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src
index d95c89c3b..31f8cbfaf 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus.app.src
+++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src
@@ -2,7 +2,7 @@
{application, emqx_prometheus, [
{description, "Prometheus for EMQX"},
% strict semver, bump manually!
- {vsn, "5.0.3"},
+ {vsn, "5.0.4"},
{modules, []},
{registered, [emqx_prometheus_sup]},
{applications, [kernel, stdlib, prometheus, emqx]},
diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl
index 5424c4e24..a66f275f8 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus.erl
@@ -98,8 +98,13 @@ handle_cast(_Msg, State) ->
{noreply, State}.
handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) ->
- #{interval := Interval, push_gateway_server := Server} = opts(),
- PushRes = push_to_push_gateway(Server),
+ #{
+ interval := Interval,
+ headers := Headers,
+ job_name := JobName,
+ push_gateway_server := Server
+ } = opts(),
+ PushRes = push_to_push_gateway(Server, Headers, JobName),
NewTimer = ensure_timer(Interval),
NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}),
%% Data is too big, hibernate for saving memory and stop system monitor warning.
@@ -107,18 +112,27 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) ->
handle_info(_Msg, State) ->
{noreply, State}.
-push_to_push_gateway(Uri) ->
+push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) ->
[Name, Ip] = string:tokens(atom_to_list(node()), "@"),
- Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]),
+ JobName1 = emqx_placeholder:preproc_tmpl(JobName),
+ JobName2 = binary_to_list(
+ emqx_placeholder:proc_tmpl(
+ JobName1,
+ #{<<"name">> => Name, <<"host">> => Ip}
+ )
+ ),
+
+ Url = lists:concat([Uri, "/metrics/job/", JobName2]),
Data = prometheus_text_format:format(),
- case httpc:request(post, {Url, [], "text/plain", Data}, ?HTTP_OPTIONS, []) of
- {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, _Body}} ->
+ case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of
+ {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} ->
ok;
Error ->
?SLOG(error, #{
msg => "post_to_push_gateway_failed",
error => Error,
- url => Url
+ url => Url,
+ headers => Headers
}),
failed
end.
diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl
index 7466a1fd1..945c6eba9 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl
@@ -121,6 +121,8 @@ prometheus_config_example() ->
enable => true,
interval => "15s",
push_gateway_server => <<"http://127.0.0.1:9091">>,
+ headers => #{'header-name' => 'header-value'},
+ job_name => <<"${name}/instance/${name}~${host}">>,
vm_dist_collector => enabled,
mnesia_collector => enabled,
vm_statistics_collector => enabled,
diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
index 688c9be58..c13d198a2 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
@@ -25,7 +25,8 @@
roots/0,
fields/1,
desc/1,
- translation/1
+ translation/1,
+ convert_headers/1
]).
namespace() -> "prometheus".
@@ -52,6 +53,26 @@ fields("prometheus") ->
desc => ?DESC(interval)
}
)},
+ {headers,
+ ?HOCON(
+ list({string(), string()}),
+ #{
+ default => #{},
+ required => false,
+ converter => fun ?MODULE:convert_headers/1,
+ desc => ?DESC(headers)
+ }
+ )},
+ {job_name,
+ ?HOCON(
+ binary(),
+ #{
+ default => <<"${name}/instance/${name}~${host}">>,
+ required => true,
+ desc => ?DESC(job_name)
+ }
+ )},
+
{enable,
?HOCON(
boolean(),
@@ -126,6 +147,17 @@ fields("prometheus") ->
desc("prometheus") -> ?DESC(prometheus);
desc(_) -> undefined.
+convert_headers(Headers) when is_map(Headers) ->
+ maps:fold(
+ fun(K, V, Acc) ->
+ [{binary_to_list(K), binary_to_list(V)} | Acc]
+ end,
+ [],
+ Headers
+ );
+convert_headers(Headers) when is_list(Headers) ->
+ Headers.
+
%% for CI test, CI don't load the whole emqx_conf_schema.
translation(Name) ->
emqx_conf_schema:translation(Name).
diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl
index b9df1103b..77d9902a2 100644
--- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl
+++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl
@@ -27,6 +27,8 @@
"prometheus {\n"
" push_gateway_server = \"http://127.0.0.1:9091\"\n"
" interval = \"1s\"\n"
+ " headers = { Authorization = \"some-authz-tokens\"}\n"
+ " job_name = \"${name}~${host}\"\n"
" enable = true\n"
" vm_dist_collector = enabled\n"
" mnesia_collector = enabled\n"
@@ -85,6 +87,25 @@ t_collector_no_crash_test(_) ->
prometheus_text_format:format(),
ok.
+t_assert_push(_) ->
+ meck:new(httpc, [passthrough]),
+ Self = self(),
+ AssertPush = fun(Method, Req = {Url, Headers, ContentType, _Data}, HttpOpts, Opts) ->
+ ?assertEqual(post, Method),
+ ?assertMatch("http://127.0.0.1:9091/metrics/job/test~127.0.0.1", Url),
+ ?assertEqual([{"Authorization", "some-authz-tokens"}], Headers),
+ ?assertEqual("text/plain", ContentType),
+ Self ! pass,
+ meck:passthrough([Method, Req, HttpOpts, Opts])
+ end,
+ meck:expect(httpc, request, AssertPush),
+ ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus)),
+ receive
+ pass -> ok
+ after 2000 ->
+ ct:fail(assert_push_request_failed)
+ end.
+
t_only_for_coverage(_) ->
?assertEqual("5.0.0", emqx_prometheus_proto_v1:introduced_in()),
ok.
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 52756f70d..0b6cbd0a2 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -36,8 +36,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
health_check_interval {
desc {
- en: """Health check interval, in milliseconds."""
- zh: """健康检查间隔，单位毫秒。"""
+ en: """Health check interval."""
+ zh: """健康检查间隔。"""
}
label {
en: """Health Check Interval"""
@@ -69,8 +69,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
auto_restart_interval {
desc {
- en: """The auto restart interval after the resource is disconnected, in milliseconds."""
- zh: """资源断开以后，自动重连的时间间隔，单位毫秒。"""
+ en: """The auto restart interval after the resource is disconnected."""
+ zh: """资源断开以后，自动重连的时间间隔。"""
}
label {
en: """Auto Restart Interval"""
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
index cbe7dae82..d6913cbc6 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
@@ -23,6 +23,7 @@
-export([
namespace/0,
+ tags/0,
roots/0,
fields/1,
desc/1,
@@ -33,6 +34,9 @@
namespace() -> rule_engine.
+tags() ->
+ [<<"Rule Engine">>].
+
roots() -> ["rule_engine"].
fields("rule_engine") ->
diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl
index 5b5ed063f..6b0721e3d 100644
--- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl
+++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl
@@ -203,13 +203,7 @@ do_request_api(Method, Request) ->
end.
auth_header_() ->
- AppId = <<"admin">>,
- AppSecret = <<"public">>,
- auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
-
-auth_header_(User, Pass) ->
- Encoded = base64:encode_to_string(lists:append([User, ":", Pass])),
- {"Authorization", "Basic " ++ Encoded}.
+ emqx_mgmt_api_test_util:auth_header_().
api_path(Parts) ->
?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).
diff --git a/bin/nodetool b/bin/nodetool
index b4f0a0183..9a5d5e069 100755
--- a/bin/nodetool
+++ b/bin/nodetool
@@ -24,12 +24,19 @@ main(Args) ->
["hocon" | Rest] ->
%% forward the call to hocon_cli
hocon_cli:main(Rest);
- ["check_license_key", Key] ->
- check_license(#{key => list_to_binary(Key)});
+ ["check_license_key", Key0] ->
+ Key = cleanup_key(Key0),
+ check_license(#{key => Key});
_ ->
do(Args)
end.
+%% the key is a string (list) representation of a binary, so we need
+%% to remove the leading and trailing angle brackets.
+cleanup_key(Str0) ->
+ Str1 = iolist_to_binary(string:replace(Str0, "<<", "", leading)),
+ iolist_to_binary(string:replace(Str1, ">>", "", trailing)).
+
do(Args) ->
ok = do_with_halt(Args, "mnesia_dir", fun create_mnesia_dir/2),
ok = do_with_halt(Args, "chkconfig", fun("-config", X) -> chkconfig(X) end),
diff --git a/changes/v5.0.15/feat-9586.en.md b/changes/v5.0.15/feat-9586.en.md
new file mode 100644
index 000000000..777fb81df
--- /dev/null
+++ b/changes/v5.0.15/feat-9586.en.md
@@ -0,0 +1 @@
+Basic auth is no longer allowed for API calls, must use API key instead.
diff --git a/changes/v5.0.15/feat-9586.zh.md b/changes/v5.0.15/feat-9586.zh.md
new file mode 100644
index 000000000..102266a46
--- /dev/null
+++ b/changes/v5.0.15/feat-9586.zh.md
@@ -0,0 +1 @@
+API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。
diff --git a/changes/v5.0.15/feat-9722.en.md b/changes/v5.0.15/feat-9722.en.md
new file mode 100644
index 000000000..b86f37b83
--- /dev/null
+++ b/changes/v5.0.15/feat-9722.en.md
@@ -0,0 +1,3 @@
+Add the following configuration options for Pushing metrics to Prometheus Push Gateway:
+- `headers`: Allows custom HTTP request headers.
+- `job_name`: allows to customize the name of the Job pushed to Push Gateway.
diff --git a/changes/v5.0.15/feat-9722.zh.md b/changes/v5.0.15/feat-9722.zh.md
new file mode 100644
index 000000000..a806cb1de
--- /dev/null
+++ b/changes/v5.0.15/feat-9722.zh.md
@@ -0,0 +1,3 @@
+为 Prometheus 推送到 Push Gateway 新增以下配置项：
+- `headers`：允许自定义 HTTP 请求头。
+- `job_name`：允许自定义推送到 Push Gateway 的 Job 名称。
diff --git a/changes/v5.0.15/feat-9725.en.md b/changes/v5.0.15/feat-9725.en.md
new file mode 100644
index 000000000..832aa6bf9
--- /dev/null
+++ b/changes/v5.0.15/feat-9725.en.md
@@ -0,0 +1,11 @@
+Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets.
+This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。
+
+The functions of these two config are difficult to distinguish, which will lead to confusion.
+After this change, `auto_reconnect` will not be configurable (always be true), and the underlying
+drivers that support this config will automatically reconnect the abnormally disconnected
+connection every `2s`.
+
+And the config `resource_opts.auto_restart_interval` is still available for user.
+It is the time interval that emqx restarts the resource when the connection cannot be
+established for some reason.
diff --git a/changes/v5.0.15/feat-9725.zh.md b/changes/v5.0.15/feat-9725.zh.md
new file mode 100644
index 000000000..e7a2412d4
--- /dev/null
+++ b/changes/v5.0.15/feat-9725.zh.md
@@ -0,0 +1,8 @@
+从认证、鉴权和数据桥接功能中，删除 `auto_reconnect` 配置项，因为我们还有另一个功能类似的配置项：
+`resource_opts.auto_restart_interval`。
+
+这两个配置项的功能难以区分，会导致困惑。此修改之后，`auto_reconnect` 将不可配置(永远为 true)，
+支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。
+
+而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置，它是资源因为某些原因
+无法建立连接的时候，emqx 重新启动该资源的时间间隔。
diff --git a/changes/v5.0.15/fix-8718-en.md b/changes/v5.0.15/fix-8718-en.md
new file mode 100644
index 000000000..6085adecd
--- /dev/null
+++ b/changes/v5.0.15/fix-8718-en.md
@@ -0,0 +1 @@
+Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis.
diff --git a/changes/v5.0.15/fix-8718-zh.md b/changes/v5.0.15/fix-8718-zh.md
new file mode 100644
index 000000000..d8aa81fd1
--- /dev/null
+++ b/changes/v5.0.15/fix-8718-zh.md
@@ -0,0 +1 @@
+密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。
diff --git a/changes/v5.0.15/fix-9626.en.md b/changes/v5.0.15/fix-9626.en.md
new file mode 100644
index 000000000..cc1c86d3e
--- /dev/null
+++ b/changes/v5.0.15/fix-9626.en.md
@@ -0,0 +1,2 @@
+Return authorization settings with default values.
+The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard.
diff --git a/changes/v5.0.15/fix-9626.zh.md b/changes/v5.0.15/fix-9626.zh.md
new file mode 100644
index 000000000..bc2391f48
--- /dev/null
+++ b/changes/v5.0.15/fix-9626.zh.md
@@ -0,0 +1,3 @@
+为授权设置 API 返回默认值。
+授权缓存默认为开启，但是在此修复前，因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失，
+使得在仪表盘配置页面中看起来是关闭了。
diff --git a/changes/v5.0.15/fix-9726-en.md b/changes/v5.0.15/fix-9726-en.md
new file mode 100644
index 000000000..9aa522690
--- /dev/null
+++ b/changes/v5.0.15/fix-9726-en.md
@@ -0,0 +1 @@
+Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response.
diff --git a/changes/v5.0.15/fix-9726-zh.md b/changes/v5.0.15/fix-9726-zh.md
new file mode 100644
index 000000000..3554d2db7
--- /dev/null
+++ b/changes/v5.0.15/fix-9726-zh.md
@@ -0,0 +1 @@
+在此修复前，客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息，现在通过在响应中提供 `hasnext` 标志来解决这个问题。
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 4a00c68fb..03533eec4 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,6 +1,7 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
ARG RUN_FROM=debian:11-slim
-FROM ${BUILD_FROM} AS builder
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder
COPY . /emqx
diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine
index 0f72be9ab..ebce2f539 100644
--- a/deploy/docker/Dockerfile.alpine
+++ b/deploy/docker/Dockerfile.alpine
@@ -1,6 +1,7 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1
ARG RUN_FROM=alpine:3.15.1
-FROM ${BUILD_FROM} AS builder
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder
RUN apk add --no-cache \
autoconf \
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
index f8009f0a4..81ebc1e31 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
@@ -86,4 +86,15 @@ emqx_ee_bridge_mongodb {
zh: "桥接名称"
}
}
+
+ payload_template {
+ desc {
+ en: "The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc."
+ zh: "用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。"
+ }
+ label: {
+ en: "Payload template"
+ zh: "有效载荷模板"
+ }
+ }
}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index a709601bb..43a26111a 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -61,9 +61,9 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, u
resource_type(kafka) -> emqx_bridge_impl_kafka;
resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub;
-resource_type(mongodb_rs) -> emqx_connector_mongo;
-resource_type(mongodb_sharded) -> emqx_connector_mongo;
-resource_type(mongodb_single) -> emqx_connector_mongo;
+resource_type(mongodb_rs) -> emqx_ee_connector_mongodb;
+resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb;
+resource_type(mongodb_single) -> emqx_ee_connector_mongodb;
resource_type(mysql) -> emqx_connector_mysql;
resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb;
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 62c8b6ab7..b42c27832 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -196,33 +196,25 @@ to_influx_lines(RawLines) ->
converter_influx_line(Line, AccIn) ->
case string:tokens(str(Line), " ") of
[MeasurementAndTags, Fields, Timestamp] ->
- {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
- [
- #{
- measurement => Measurement,
- tags => kv_pairs(Tags),
- fields => kv_pairs(string:tokens(Fields, ",")),
- timestamp => Timestamp
- }
- | AccIn
- ];
+ append_influx_item(MeasurementAndTags, Fields, Timestamp, AccIn);
[MeasurementAndTags, Fields] ->
- {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
- %% TODO: fix here both here and influxdb driver.
- %% Default value should evaluated by InfluxDB.
- [
- #{
- measurement => Measurement,
- tags => kv_pairs(Tags),
- fields => kv_pairs(string:tokens(Fields, ",")),
- timestamp => "${timestamp}"
- }
- | AccIn
- ];
+ append_influx_item(MeasurementAndTags, Fields, undefined, AccIn);
_ ->
throw("Bad InfluxDB Line Protocol schema")
end.
+append_influx_item(MeasurementAndTags, Fields, Timestamp, Acc) ->
+ {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
+ [
+ #{
+ measurement => Measurement,
+ tags => kv_pairs(Tags),
+ fields => kv_pairs(string:tokens(Fields, ",")),
+ timestamp => Timestamp
+ }
+ | Acc
+ ].
+
split_measurement_and_tags(Subject) ->
case string:tokens(Subject, ",") of
[] ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
index 516c75f65..bb4082681 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
@@ -37,7 +37,8 @@ roots() ->
fields("config") ->
[
{enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})},
- {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}
+ {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})},
+ {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})}
];
fields(mongodb_rs) ->
emqx_connector_mongo:fields(rs) ++ fields("config");
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index bf5d2e140..114459149 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -51,7 +51,6 @@ values(post) ->
pool_size => 8,
username => <<"root">>,
password => <<"">>,
- auto_reconnect => true,
sql => ?DEFAULT_SQL,
local_topic => <<"local/topic/#">>,
resource_opts => #{
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl
index dc8697e37..be9fc9dc8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl
@@ -53,7 +53,6 @@ values(post, Type) ->
pool_size => 8,
username => <<"root">>,
password => <<"public">>,
- auto_reconnect => true,
sql => ?DEFAULT_SQL,
local_topic => <<"local/topic/#">>,
resource_opts => #{
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl
index a0c6ba834..5c273e050 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl
@@ -79,7 +79,6 @@ values(common, RedisType, SpecificOpts) ->
local_topic => <<"local/topic/#">>,
pool_size => 8,
password => <<"secret">>,
- auto_reconnect => true,
command_template => [<<"LPUSH">>, <<"MSGS">>, <<"${payload}">>],
resource_opts => #{
batch_size => 1,
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
index 6331611d0..bb87a9f37 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
@@ -525,7 +525,6 @@ t_start_ok(Config) ->
SentData = #{
<<"clientid">> => ClientId,
<<"topic">> => atom_to_binary(?FUNCTION_NAME),
- <<"timestamp">> => erlang:system_time(nanosecond),
<<"payload">> => Payload
},
?check_trace(
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
index fb8f1fcc3..7e44347f3 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
@@ -25,7 +25,8 @@ all() ->
group_tests() ->
[
t_setup_via_config_and_publish,
- t_setup_via_http_api_and_publish
+ t_setup_via_http_api_and_publish,
+ t_payload_template
].
groups() ->
@@ -196,9 +197,14 @@ parse_and_check(ConfigString, Type, Name) ->
Config.
create_bridge(Config) ->
+ create_bridge(Config, _Overrides = #{}).
+
+create_bridge(Config, Overrides) ->
Type = mongo_type_bin(?config(mongo_type, Config)),
Name = ?config(mongo_name, Config),
- MongoConfig = ?config(mongo_config, Config),
+ MongoConfig0 = ?config(mongo_config, Config),
+ MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides),
+ ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]),
emqx_bridge:create(Type, Name, MongoConfig).
delete_bridge(Config) ->
@@ -219,7 +225,8 @@ clear_db(Config) ->
Name = ?config(mongo_name, Config),
#{<<"collection">> := Collection} = ?config(mongo_config, Config),
ResourceID = emqx_bridge_resource:resource_id(Type, Name),
- {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID),
+ {ok, _, #{state := #{connector_state := #{poolname := PoolName}}}} =
+ emqx_resource:get_instance(ResourceID),
Selector = #{},
{true, _} = ecpool:pick_and_do(
PoolName, {mongo_api, delete, [Collection, Selector]}, no_handover
@@ -275,3 +282,14 @@ t_setup_via_http_api_and_publish(Config) ->
find_all(Config)
),
ok.
+
+t_payload_template(Config) ->
+ {ok, _} = create_bridge(Config, #{<<"payload_template">> => <<"{\"foo\": \"${clientid}\"}">>}),
+ Val = erlang:unique_integer(),
+ ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
+ ok = send_message(Config, #{key => Val, clientid => ClientId}),
+ ?assertMatch(
+ {ok, [#{<<"foo">> := ClientId}]},
+ find_all(Config)
+ ),
+ ok.
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index ab4c88396..262641d44 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,7 +1,7 @@
{erl_opts, [debug_info]}.
{deps, [
{hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
- {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.5"}}},
+ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.6"}}},
{emqx, {path, "../../apps/emqx"}}
]}.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 7974bf028..db99c4475 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -142,7 +142,11 @@ fields(common) ->
[
{server, server()},
{precision,
- mk(enum([ns, us, ms, s, m, h]), #{
+ %% The influxdb only supports these 4 precision:
+ %% See "https://github.com/influxdata/influxdb/blob/
+ %% 6b607288439a991261307518913eb6d4e280e0a7/models/points.go#L487" for
+ %% more information.
+ mk(enum([ns, us, ms, s]), #{
required => false, default => ms, desc => ?DESC("precision")
})}
];
@@ -210,9 +214,7 @@ start_client(InstId, Config) ->
do_start_client(
InstId,
ClientConfig,
- Config = #{
- write_syntax := Lines
- }
+ Config = #{write_syntax := Lines}
) ->
case influxdb:start_client(ClientConfig) of
{ok, Client} ->
@@ -220,7 +222,9 @@ do_start_client(
true ->
State = #{
client => Client,
- write_syntax => to_config(Lines)
+ write_syntax => to_config(
+ Lines, proplists:get_value(precision, ClientConfig)
+ )
},
?SLOG(info, #{
msg => "starting influxdb connector success",
@@ -348,30 +352,33 @@ do_async_query(InstId, Client, Points, ReplyFunAndArgs) ->
%% -------------------------------------------------------------------------------------------------
%% Tags & Fields Config Trans
-to_config(Lines) ->
- to_config(Lines, []).
+to_config(Lines, Precision) ->
+ to_config(Lines, [], Precision).
-to_config([], Acc) ->
+to_config([], Acc, _Precision) ->
lists:reverse(Acc);
-to_config(
- [
- #{
- measurement := Measurement,
- timestamp := Timestamp,
- tags := Tags,
- fields := Fields
- }
- | Rest
- ],
- Acc
-) ->
- Res = #{
- measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement),
- timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp),
- tags => to_kv_config(Tags),
- fields => to_kv_config(Fields)
+to_config([Item0 | Rest], Acc, Precision) ->
+ Ts = maps:get(timestamp, Item0, undefined),
+ Item = #{
+ measurement => emqx_plugin_libs_rule:preproc_tmpl(maps:get(measurement, Item0)),
+ timestamp => preproc_tmpl_timestamp(Ts, Precision),
+ tags => to_kv_config(maps:get(tags, Item0)),
+ fields => to_kv_config(maps:get(fields, Item0))
},
- to_config(Rest, [Res | Acc]).
+ to_config(Rest, [Item | Acc], Precision).
+
+preproc_tmpl_timestamp(undefined, <<"ns">>) ->
+ erlang:system_time(nanosecond);
+preproc_tmpl_timestamp(undefined, <<"us">>) ->
+ erlang:system_time(microsecond);
+preproc_tmpl_timestamp(undefined, <<"ms">>) ->
+ erlang:system_time(millisecond);
+preproc_tmpl_timestamp(undefined, <<"s">>) ->
+ erlang:system_time(second);
+preproc_tmpl_timestamp(Ts, _) when is_integer(Ts) ->
+ Ts;
+preproc_tmpl_timestamp(Ts, _) when is_binary(Ts); is_list(Ts) ->
+ emqx_plugin_libs_rule:preproc_tmpl(Ts).
to_kv_config(KVfields) ->
maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)).
@@ -414,7 +421,7 @@ parse_batch_data(InstId, BatchData, SyntaxLines) ->
fields := [{binary(), binary()}],
measurement := binary(),
tags := [{binary(), binary()}],
- timestamp := binary()
+ timestamp := emqx_plugin_libs_rule:tmpl_token() | integer()
}
]) -> {ok, [map()]} | {error, term()}.
data_to_points(Data, SyntaxLines) ->
@@ -430,46 +437,50 @@ lines_to_points(_, [], Points, ErrorPoints) ->
%% ignore trans succeeded points
{error, ErrorPoints}
end;
-lines_to_points(
- Data,
- [
- #{
- measurement := Measurement,
- timestamp := Timestamp,
- tags := Tags,
- fields := Fields
- }
- | Rest
- ],
- ResultPointsAcc,
- ErrorPointsAcc
-) ->
+lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when
+ is_list(Ts)
+->
TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
- case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of
- [TimestampInt] when is_integer(TimestampInt) ->
- {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
- {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields),
- Point = #{
- measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data),
- timestamp => TimestampInt,
- tags => EncodedTags,
- fields => EncodedFields
- },
- case map_size(EncodedFields) =:= 0 of
- true ->
- %% influxdb client doesn't like empty field maps...
- lines_to_points(Data, Rest, ResultPointsAcc, [
- {error, no_fields} | ErrorPointsAcc
- ]);
- false ->
- lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc)
- end;
- BadTimestamp ->
+ case emqx_plugin_libs_rule:proc_tmpl(Ts, Data, TransOptions) of
+ [TsInt] when is_integer(TsInt) ->
+ Item1 = Item#{timestamp => TsInt},
+ continue_lines_to_points(Data, Item1, Rest, ResultPointsAcc, ErrorPointsAcc);
+ BadTs ->
lines_to_points(Data, Rest, ResultPointsAcc, [
- {error, {bad_timestamp, BadTimestamp}} | ErrorPointsAcc
+ {error, {bad_timestamp, BadTs}} | ErrorPointsAcc
])
+ end;
+lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when
+ is_integer(Ts)
+->
+ continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc).
+
+continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc) ->
+ case line_to_point(Data, Item) of
+ #{fields := Fields} when map_size(Fields) =:= 0 ->
+ %% influxdb client doesn't like empty field maps...
+ ErrorPointsAcc1 = [{error, no_fields} | ErrorPointsAcc],
+ lines_to_points(Data, Rest, ResultPointsAcc, ErrorPointsAcc1);
+ Point ->
+ lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc)
end.
+line_to_point(
+ Data,
+ #{
+ measurement := Measurement,
+ tags := Tags,
+ fields := Fields
+ } = Item
+) ->
+ {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
+ {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields),
+ Item#{
+ measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data),
+ tags => EncodedTags,
+ fields => EncodedFields
+ }.
+
maps_config_to_data(K, V, {Data, Res}) ->
KTransOptions = #{return => rawlist, var_trans => fun key_filter/1},
VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
new file mode 100644
index 000000000..b1327fef6
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
@@ -0,0 +1,78 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_mongodb).
+
+-behaviour(emqx_resource).
+
+-include_lib("emqx_connector/include/emqx_connector_tables.hrl").
+-include_lib("emqx_resource/include/emqx_resource.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+%% `emqx_resource' API
+-export([
+ callback_mode/0,
+ is_buffer_supported/0,
+ on_start/2,
+ on_stop/2,
+ on_query/3,
+ on_get_status/2
+]).
+
+%%========================================================================================
+%% `emqx_resource' API
+%%========================================================================================
+
+callback_mode() -> emqx_connector_mongo:callback_mode().
+
+is_buffer_supported() -> false.
+
+on_start(InstanceId, Config) ->
+ case emqx_connector_mongo:on_start(InstanceId, Config) of
+ {ok, ConnectorState} ->
+ PayloadTemplate0 = maps:get(payload_template, Config, undefined),
+ PayloadTemplate = preprocess_template(PayloadTemplate0),
+ State = #{
+ payload_template => PayloadTemplate,
+ connector_state => ConnectorState
+ },
+ {ok, State};
+ Error ->
+ Error
+ end.
+
+on_stop(InstanceId, _State = #{connector_state := ConnectorState}) ->
+ emqx_connector_mongo:on_stop(InstanceId, ConnectorState).
+
+on_query(InstanceId, {send_message, Message0}, State) ->
+ #{
+ payload_template := PayloadTemplate,
+ connector_state := ConnectorState
+ } = State,
+ Message = render_message(PayloadTemplate, Message0),
+ emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, ConnectorState);
+on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) ->
+ emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState).
+
+on_get_status(InstanceId, _State = #{connector_state := ConnectorState}) ->
+ emqx_connector_mongo:on_get_status(InstanceId, ConnectorState).
+
+%%========================================================================================
+%% Helper fns
+%%========================================================================================
+
+preprocess_template(undefined = _PayloadTemplate) ->
+ undefined;
+preprocess_template(PayloadTemplate) ->
+ emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate).
+
+render_message(undefined = _PayloadTemplate, Message) ->
+ Message;
+render_message(PayloadTemplate, Message) ->
+ %% Note: mongo expects a map as a document, so the rendered result
+ %% must be JSON-serializable
+ Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message),
+ emqx_json:decode(Rendered, [return_maps]).
diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl
index bde4ed076..9d16f697c 100644
--- a/lib-ee/emqx_license/src/emqx_license_schema.erl
+++ b/lib-ee/emqx_license/src/emqx_license_schema.erl
@@ -13,7 +13,7 @@
-behaviour(hocon_schema).
--export([roots/0, fields/1, validations/0, desc/1]).
+-export([roots/0, fields/1, validations/0, desc/1, tags/0]).
-export([
default_license/0,
@@ -31,10 +31,13 @@ roots() ->
)}
].
+tags() ->
+ [<<"License">>].
+
fields(key_license) ->
[
{key, #{
- type => string(),
+ type => binary(),
default => default_license(),
%% so it's not logged
sensitive => true,
@@ -82,7 +85,9 @@ check_license_watermark(Conf) ->
%% NOTE: when updating a new key, the schema doc in emqx_license_schema_i18n.conf
%% should be updated accordingly
default_license() ->
- "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ"
- "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f"
- "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn"
- "8mgRDtiT3R8RZooqy6dEsOmDI=".
+ <<
+ "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ"
+ "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f"
+ "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn"
+ "8mgRDtiT3R8RZooqy6dEsOmDI="
+ >>.
diff --git a/mix.exs b/mix.exs
index 45bc820e0..27d9b43ce 100644
--- a/mix.exs
+++ b/mix.exs
@@ -69,7 +69,7 @@ defmodule EMQXUmbrella.MixProject do
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.35.0", override: true},
- {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
+ {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
# in conflict by ehttpc and emqtt
@@ -131,7 +131,7 @@ defmodule EMQXUmbrella.MixProject do
defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
[
{:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
- {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true},
+ {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.6", override: true},
{:wolff, github: "kafka4beam/wolff", tag: "1.7.4"},
{:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true},
{:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},
diff --git a/rebar.config b/rebar.config
index a61bfe16a..f05624477 100644
--- a/rebar.config
+++ b/rebar.config
@@ -69,7 +69,7 @@
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.0"}}}
- , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
+ , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}
, {telemetry, "1.1.0"}
diff --git a/scripts/docker-create-push-manifests.sh b/scripts/docker-create-push-manifests.sh
deleted file mode 100755
index db9c01bfb..000000000
--- a/scripts/docker-create-push-manifests.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-##!/usr/bin/env bash
-set -exuo pipefail
-
-img_amd64=$1
-push_latest=${2:-false}
-
-img_arm64=$(echo ${img_amd64} | sed 's/-amd64$/-arm64/g')
-img_name=${img_amd64%-amd64}
-docker pull "$img_amd64"
-docker pull --platform linux/arm64 "$img_arm64"
-img_amd64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_amd64")
-img_arm64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_arm64")
-echo "sha256 of amd64 is $img_amd64_digest"
-echo "sha256 of arm64 is $img_arm64_digest"
-docker manifest create "${img_name}" \
- --amend "$img_amd64_digest" \
- --amend "$img_arm64_digest"
-docker manifest push "${img_name}"
-
-# PUSH latest if it is a release build
-if [ "$push_latest" = "true" ]; then
- img_latest=$(echo "$img_arm64" | cut -d: -f 1):latest
- docker manifest create "${img_latest}" \
- --amend "$img_amd64_digest" \
- --amend "$img_arm64_digest"
- docker manifest push "${img_latest}"
-fi