Merge remote-tracking branch 'origin/release-v43' into release-v44

.github/workflows/run_test_cases.yaml

@@ -14,31 +14,37 @@ jobs:
     runs-on: ubuntu-20.04
     container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
     outputs:
-      ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }}
+      fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
+      docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
     steps:
       - uses: actions/checkout@v3
         with:
           path: source
           fetch-depth: 0
       - name: git credentials
+        if: endsWith(github.repository, 'emqx-enterprise')
         run: |
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          fi
       - name: find_ct_apps
         working-directory: source
         id: run_find_apps
         # emqx_plugin_libs doesn't have a test suite -> excluded from app list
-        # emqx ct is run independently -> exclude it from the app list
+        # emqx ct is run independently -> exclude it from the app list (regular ct)
+        # lib-ee/emqx_node_rebalance -> exclude it from app list (regular ct)
         run: |
-          ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . ==  "emqx"))')"
+          fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
-          echo "ct-apps: $ct_apps"
+          docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
-          echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT
+          echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT
+          echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT
       - name: get_all_deps
         working-directory: source
         run: |
-          make deps-all
+          # build the default profile for two purposes
+          # 1. download all dependencies (so the individual app runs do not depend on github credentials)
+          # 2. some of the files such as segmented config files are not created when compiling only the test profile
+          make
+          # compile test profile to speed up the individual runs
           ./rebar3 as test compile
           cd ..
           zip -ryq source.zip source/* source/.[^.]*
@@ -75,12 +81,24 @@ jobs:
           path: source/_build/test/cover
           if-no-files-found: warn
 
-  emqx_ct:
+  fast_ct:
     needs: prepare
     runs-on: ubuntu-20.04
     container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
     strategy:
       fail-fast: false
+      matrix:
+        app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
@@ -89,24 +107,26 @@ jobs:
           path: .
       - name: unzip source code
         run: unzip -o -q source.zip
-      # produces emqx-emqx.coverdata
+      # produces emqx-<appname>.coverdata
-      - name: emqx-ct-pipeline
+      - name: make-app-ct-pipeline
         working-directory: source
-        run: make emqx-ct-pipeline
+        run: |
+          export CT_READABLE=true
+          make ${{ matrix.app_name }}-ct-pipeline
       - uses: actions/upload-artifact@v3
         with:
           name: cover
           path: source/_build/test/cover
           if-no-files-found: warn
 
-  ct:
+  docker_ct:
     needs: prepare
     runs-on: ${{ matrix.runs-on }}
     strategy:
       max-parallel: 12
       fail-fast: false
       matrix:
-        app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }}
+        app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
         runs-on:
           - aws-amd64
           - ubuntu-20.04
@@ -132,28 +152,6 @@ jobs:
           docker network rm $(docker network ls -q) || true
       - name: docker compose up
         working-directory: source
-        if: endsWith(github.repository, 'emqx')
-        env:
-          MYSQL_TAG: 8
-          REDIS_TAG: 6
-          MONGO_TAG: 4
-          PGSQL_TAG: 13
-          LDAP_TAG: 2.4.50
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          docker-compose \
-              -f .ci/docker-compose-file/docker-compose.yaml \
-              -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-              -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-              -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-              -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-              -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-              up -d --build
-          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-      - name: docker compose up
-        working-directory: source
-        if: endsWith(github.repository, 'emqx-enterprise')
         env:
           MYSQL_TAG: 8
           REDIS_TAG: 6
@@ -169,17 +167,20 @@ jobs:
           KAFKA_TAG: 2.5.0
           PULSAR_TAG: 2.3.2
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        timeout-minutes: 20
+        timeout-minutes: 40
         run: |
-          docker-compose \
+          ulimit -n
+          docker_compose_files="\
               -f .ci/docker-compose-file/docker-compose.yaml \
               -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-              -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml"
+          ee_docker_compose_files="\
               -f .ci/docker-compose-file/docker-compose-enterprise.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
@@ -189,16 +190,30 @@ jobs:
               -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
-              -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml"
-              up -d --build
+          if [ -f EMQX_ENTERPRISE ]; then
+            docker_compose_files="${docker_compose_files} ${ee_docker_compose_files}"
+          fi
+          # only build ldap docker image when necessary, it takes 9 minutes
+          if [[ ${{ matrix.app_name }} == *ldap ]]; then
+            docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml"
+          fi
+          docker-compose $docker_compose_files up -d --build
+          if [ -f EMQX_ENTERPRISE ]; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
+          fi
           docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-          while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+          clients="$(docker ps -q --filter name=client)"
-               != $(docker ps -a --filter name=client | wc -l) ]; do
+          for client in ${clients}; do
-            sleep 5
+            docker ps -a --filter name=client
+            echo "waiting for docker ${client} to exit"
+            if ! timeout 60 docker wait "${client}"; then
+              docker-compose $docker_compose_files logs | tee docker-compose.log
+              exit 1
+            fi
           done
       - name: run common test
-        run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline"
+        run: docker exec -i erlang bash -c "env CT_READABLE=true make ${{ matrix.app_name }}-ct-pipeline"
       - name: cat rebar.crashdump
         if: failure()
         working-directory: source
@@ -210,7 +225,9 @@ jobs:
         if: failure()
         with:
           name: ${{ env.LOGFILENAME }}
-          path: source/_build/test/logs
+          path: |
+            docker-compose.log
+            source/_build/test/logs
           if-no-files-found: warn
       - uses: actions/upload-artifact@v3
         with:
@@ -221,8 +238,8 @@ jobs:
   make_cover:
     needs:
       - eunit_and_proper
-      - emqx_ct
+      - fast_ct
-      - ct
+      - docker_ct
     runs-on: ubuntu-20.04
     container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
     steps:

CHANGES-4.3.md

@@ -30,7 +30,7 @@ File format:
 - TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
   Eliminate uncertainty that the buffer size is set by OS default.
 
-- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
+- Disable authorization for `api/v4/emqx_prometheus` endpoint. [#8955](https://github.com/emqx/emqx/pull/8955)
 
 - Added a test to prevent a last will testament message to be
   published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)

Makefile

@@ -21,7 +21,9 @@ REL_PROFILES := emqx emqx-edge
 PKG_PROFILES := emqx-pkg emqx-edge-pkg
 PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
 
-export REBAR_GIT_CLONE_OPTIONS += --depth=1 --quiet
+CT_READABLE ?= false
+
+export REBAR_GIT_CLONE_OPTIONS += --depth=1
 
 .PHONY: default
 default: $(REBAR) $(PROFILE)
@@ -58,7 +60,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
 $1-ct: $(REBAR)
-	$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+	$(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 
@@ -66,7 +68,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 .PHONY: $(APPS:%=%-ct-pipeline)
 define gen-app-ct-target-pipeline
 $1-ct-pipeline: $(REBAR)
-	$(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+	$(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
 
@@ -87,6 +89,7 @@ coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
 .PHONY: $(REL_PROFILES)
+
 $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 	@$(REBAR) as $(@) do compile,release
 

apps/emqx_management/src/emqx_mgmt.erl

@@ -508,7 +508,7 @@ delete_all_deactivated_alarms() ->
 delete_all_deactivated_alarms(Node) when Node =:= node() ->
     emqx_alarm:delete_all_deactivated_alarms();
 delete_all_deactivated_alarms(Node) ->
-    rpc_call(Node, delete_deactivated_alarms, [Node]).
+    rpc_call(Node, emqx_alarm, delete_all_deactivated_alarms, []).
 
 add_duration_field(Alarms) ->
     Now = erlang:system_time(microsecond),
@@ -581,7 +581,10 @@ item(route, {Topic, Node}) ->
 %%--------------------------------------------------------------------
 
 rpc_call(Node, Fun, Args) ->
-    case rpc:call(Node, ?MODULE, Fun, Args) of
+    rpc_call(Node, ?MODULE, Fun, Args).
+
+rpc_call(Node, Mod, Fun, Args) ->
+    case rpc:call(Node, Mod, Fun, Args) of
         {badrpc, Reason} -> {error, Reason};
         Res -> Res
     end.

apps/emqx_management/src/emqx_mgmt_http.erl

@@ -125,6 +125,11 @@ handle_request(_Method, _Path, Req) ->
     cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req).
 
 authorize_appid(Req) ->
+    authorize_appid(cowboy_req:method(Req), cowboy_req:path(Req), Req).
+
+authorize_appid(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) ->
+    true;
+authorize_appid(_Method, _Path, Req) ->
     try
         {basic, AppId, AppSecret} = cowboy_req:parse_header(<<"authorization">>, Req),
         emqx_mgmt_auth:is_authorized(AppId, AppSecret)

apps/emqx_management/test/emqx_mgmt_SUITE.erl

@@ -34,7 +34,7 @@ all() ->
      {group, check_cli}] ++ OtherTCs.
 
 manage_apps_tests() ->
-    [t_app].
+    [t_app, t_alarms].
 
 check_cli_tests() ->
     [t_cli,
@@ -99,6 +99,13 @@ init_per_testcase(t_backup_file, Config) ->
               file:delete(filename:join(BackupDir, F))
       end, Files),
     Config;
+init_per_testcase(Case, Config) when Case =:= t_alarms ->
+    try
+        ?MODULE:Case({'init', Config})
+    catch
+        error : function_clause ->
+            Config
+    end;
 init_per_testcase(_Case, Config) ->
     mock_print(),
     Config.
@@ -124,6 +131,13 @@ end_per_testcase(t_backup_file, _Config) ->
               file:delete(filename:join(BackupDir, F))
       end, Files),
     ok;
+end_per_testcase(Case, Config) when Case =:= t_alarms ->
+    try
+        ?MODULE:Case({'end', Config})
+    catch
+        error : function_clause ->
+            ok
+    end;
 end_per_testcase(_Case, _Config) ->
     unmock_print().
 
@@ -158,6 +172,16 @@ t_app(_Config) ->
     emqx_mgmt_auth:del_app(<<"app_id">>),
     ok.
 
+t_alarms({'init', Config}) ->
+    meck:new(rpc, [unstick]),
+    meck:expect(rpc, call, 4, ok),
+    Config;
+t_alarms({'end', _}) ->
+    meck:unload(rpc);
+t_alarms(_) ->
+    ok = emqx_mgmt:delete_all_deactivated_alarms(remote_node),
+    ?assert(meck:called(rpc, call, [remote_node, emqx_alarm, delete_all_deactivated_alarms, []])).
+
 t_log_cmd(_) ->
     lists:foreach(fun(Level) ->
                       emqx_mgmt_cli:log(["primary-level", Level]),

changes/v4.3.22-en.md

@@ -70,4 +70,8 @@
   Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`.
   See docs [Create Rule](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources).
 
+- Calling 'DELETE /alarms/deactivated' now deletes deactived alarms on all nodes, including remote nodes, not just the local node [#9280](https://github.com/emqx/emqx/pull/9280).
+
 - When republishing messages or bridge messages to other brokers, check the validity of the topic and make sure it does not have topic wildcards [#9291](https://github.com/emqx/emqx/pull/9291).
+
+- Disable authorization for `api/v4/emqx_prometheus` endpoint on management api listener (default 8081) [#9294](https://github.com/emqx/emqx/pull/9294).

changes/v4.3.22-zh.md

@@ -64,4 +64,8 @@
   注意在创建规则或资源时，HTTP body 中的 `id` 字段仍为字面值，而不是编码之后的值。
   详情请参考 [创建规则](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources)。
 
+- 修复调用 'DELETE /alarms/deactivated' 只在单个节点上生效的问题，现在将会删除所有节点上的非活跃警告 [#9280](https://github.com/emqx/emqx/pull/9280)。
+
 - 在进行消息重发布或桥接消息到其他 mqtt broker 时，检查 topic 合法性，确定其不带有主题通配符 [#9291](https://github.com/emqx/emqx/pull/9291)。
+
+- 关闭管理端口（默认为8081）上对 HTTP API `api/v4/emqx_prometheus` 的认证，Prometheus 对时序数据抓取不在需要配置认证 [#9294](https://github.com/emqx/emqx/pull/9294)。

lib-ce/emqx_dashboard/src/emqx_dashboard.erl

@@ -107,13 +107,13 @@ http_handlers() ->
 %%--------------------------------------------------------------------
 
 is_authorized(Req) ->
-    is_authorized(binary_to_list(cowboy_req:path(Req)), Req).
+    is_authorized(cowboy_req:method(Req), cowboy_req:path(Req), Req).
 
-is_authorized("/api/v4/emqx_prometheus", _Req) ->
+is_authorized(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) ->
     true;
-is_authorized("/api/v4/auth", _Req) ->
+is_authorized(<<"POST">>, <<"/api/v4/auth">>, _Req) ->
     true;
-is_authorized(_Path, Req) ->
+is_authorized(_Method, _Path, Req) ->
     try
         {basic, Username, Password} = cowboy_req:parse_header(<<"authorization">>, Req),
         case emqx_dashboard_admin:check(iolist_to_binary(Username), iolist_to_binary(Password)) of

scripts/find-apps.sh

@@ -9,9 +9,11 @@ help() {
     echo
     echo "-h|--help:       To display this usage info"
     echo "--json:          Print apps in json"
+    echo "-ct fast|docker: Only find apps for CT"
 }
 
 WANT_JSON='no'
+CT='novalue'
 while [ "$#" -gt 0 ]; do
     case $1 in
         -h|--help)
@@ -22,12 +24,17 @@ while [ "$#" -gt 0 ]; do
             WANT_JSON='yes'
             shift 1
             ;;
+        --ct)
+            CT="$2"
+            shift 2
+            ;;
         *)
             echo "unknown option $1"
             exit 1
             ;;
     esac
 done
+
 if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
     # Otherwise windows may resolve to find.exe
     FIND="/usr/bin/find"
@@ -41,25 +48,44 @@ find_app() {
 }
 
 EM="emqx"
-CE="$(find_app 'apps')"
+SHARED="$(find_app 'apps')"
-
 if [ -f 'EMQX_ENTERPRISE' ]; then
     LIB="$(find_app 'lib-ee')"
 else
     LIB="$(find_app 'lib-ce')"
 fi
-
 ## find directories in lib-extra
 LIBE="$(find_app 'lib-extra')"
-
 ## find symlinks in lib-extra
 LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)"
 
-APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")"
+APPS_ALL="$(echo -e "${EM}\n${SHARED}\n${LIB}\n${LIBE}\n${LIBES}")"
 
-if [ "$WANT_JSON" = 'yes' ]; then
+if [ "$CT" = 'novalue' ]; then
-    echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
+    RESULT="${APPS_ALL}"
 else
-    echo "${APPS_ALL}"
+    APPS_NORMAL_CT=( )
+    APPS_DOCKER_CT=( )
+    for app in ${APPS_ALL}; do
+        if [ "$app" = 'apps/emqx_plugin_libs' ]; then
+            # This app has no test SUITE
+            continue
+        fi
+        if [ -f "${app}/docker-ct" ]; then
+            APPS_DOCKER_CT+=("$app")
+        else
+            APPS_NORMAL_CT+=("$app")
+        fi
+    done
+    if [ "$CT" = 'docker' ]; then
+        RESULT="${APPS_DOCKER_CT[*]}"
+    else
+        RESULT="${APPS_NORMAL_CT[*]}"
+    fi
 fi
 
+if [ "$WANT_JSON" = 'yes' ]; then
+    echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
+else
+    echo "${RESULT}" | xargs
+fi

scripts/get-dashboard.sh

@@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
 case "${PKG_VSN}" in
     4.3*)
-        EMQX_CE_DASHBOARD_VERSION='v4.3.11'
+        EMQX_CE_DASHBOARD_VERSION='v4.3.12'
-        EMQX_EE_DASHBOARD_VERSION='v4.3.26'
+        EMQX_EE_DASHBOARD_VERSION='v4.3.27'
         ;;
     4.4*)
         # keep the above 4.3 untouched, otherwise conflicts!