Merge remote-tracking branch 'origin/release-v44' into main-v4.4

.ci/build_packages/tests.sh

@@ -116,7 +116,20 @@ emqx_test(){
             fi
         ;;
         "rpm")
-            yum install -y "${PACKAGE_PATH}/${packagename}"
+            # yum wants python2
+            if [[ "${SYSTEM:-}" == "amzn2" ]]; then
+                alternatives --list | grep python && alternatives --set python /usr/bin/python2
+            fi
+            YUM_RES="$(yum install -y "${PACKAGE_PATH}/${packagename}"| tee /dev/null)"
+            if [[ $YUM_RES =~ "Failed" ]]; then
+               echo "yum install failed"
+               exit 1
+            fi
+            # restore python3
+            if [[ "${SYSTEM:-}" == "amzn2" ]]; then
+                alternatives --list | grep python && alternatives --set python /usr/bin/python3
+            fi
+
             if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then
                 echo "package install error"
                 exit 1

.github/actions/package-macos/action.yaml

@@ -23,27 +23,38 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: prepare
+    - id: prepare
       shell: bash
       run: |
         brew update
-        brew install curl zip unzip gnu-sed coreutils unixodbc freetds openssl@1.1
+        brew install curl zip unzip gnu-sed coreutils autoconf automake cmake unixodbc freetds openssl@1.1
         echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
+        OTP_SOURCE_PATH="$HOME/src/otp-${{ inputs.otp }}"
+        OTP_INSTALL_PATH="$HOME/otp/${{ inputs.otp }}"
+        echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT
+        echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT
+        mkdir -p "$OTP_INSTALL_PATH"
     - uses: actions/cache@v3
       id: cache
       with:
-        path: /opt/erlang/${{ inputs.otp }}
+        path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }}
         key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git $HOME/otp-${{ inputs.otp }}
+        OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}"
-        cd $HOME/otp-${{ inputs.otp }}
+        OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}"
-        ./configure --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit --prefix=/opt/erlang/${{ inputs.otp }}
+        if [ -d "$OTP_SOURCE_PATH" ]; then
+          rm -rf "$OTP_SOURCE_PATH"
+        fi
+        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH"
+        cd "$OTP_SOURCE_PATH"
+        ./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH"
         make -j$(nproc)
-        sudo make install
+        rm -rf "$OTP_INSTALL_PATH"
+        make install
     - name: build
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
@@ -56,13 +67,16 @@ runs:
         APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
       shell: bash
       run: |
-        export PATH="/opt/erlang/${{ inputs.otp }}/bin:$PATH"
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
+        mkdir -p $HOME/bin
+        cp rebar3 $HOME/bin/rebar3
+        export PATH="$HOME/bin:$PATH"
         make ${EMQX_NAME}-zip
     - name: test
       shell: bash
       run: |
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
         # test with a spaces in path

.github/workflows/apps_version_check.yaml

@@ -13,7 +13,7 @@ jobs:
         os:
         - ubuntu20.04
 
-    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/build_packages.yaml

@@ -110,9 +110,12 @@ jobs:
           - 24.3.4.2-1
         os:
           - macos-11
+          - macos-12-arm64
     runs-on: ${{ matrix.os }}
 
     steps:
+    - uses: emqx/self-hosted-cleanup-action@v1.0.3
+      if: matrix.os == macos-12-arm64
     - uses: actions/download-artifact@v3
       with:
         name: source
@@ -162,31 +165,17 @@ jobs:
           - debian9
           - el8
           - el7
-          # - raspbian10 #armv6l is too slow to emulate
+          - amzn2
-        exclude:
-        - os: raspbian9
-          arch: amd64
-        - os: raspbian10
-          arch: amd64
-        - os: raspbian9
-          profile: emqx
-        - os: raspbian10
-          profile: emqx
-        - os: raspbian9
-          profile: emqx-ee
-        - os: raspbian10
-          profile: emqx-ee
-
     defaults:
       run:
         shell: bash
 
     steps:
-    - uses: docker/setup-buildx-action@v2
     - uses: docker/setup-qemu-action@v2
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
+    - uses: docker/setup-buildx-action@v2
     - uses: actions/download-artifact@v3
       with:
         name: source
@@ -206,7 +195,7 @@ jobs:
           --profile "${PROFILE}" \
           --pkgtype "${PACKAGE}" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}"
+          --builder "ghcr.io/emqx/emqx-builder/4.4-23:${OTP}-${SYSTEM}"
     - uses: actions/upload-artifact@v3
       with:
         name: ${{ matrix.profile }}
@@ -239,11 +228,11 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - uses: docker/setup-buildx-action@v2
     - uses: docker/setup-qemu-action@v2
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
+    - uses: docker/setup-buildx-action@v2
     - uses: aws-actions/configure-aws-credentials@v1-node16
       if: matrix.registry == 'public.ecr.aws'
       with:
@@ -271,7 +260,7 @@ jobs:
           type=match,pattern=[v|e](.*),group=1
         labels:
           org.opencontainers.image.otp.version=${{ matrix.otp }}
-    - uses: docker/build-push-action@v2
+    - uses: docker/build-push-action@v3
       if: matrix.profile != 'emqx-ee'
       with:
         ## only push when stable tag and rc tag
@@ -282,7 +271,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
@@ -298,7 +287,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise

.github/workflows/build_slim_packages.yaml

@@ -20,7 +20,7 @@ jobs:
           - 24.3.4.2-1
         os:
           - ubuntu20.04
-          - el8
+          - el7
         runs-on:
           - aws-amd64
           - ubuntu-20.04
@@ -32,7 +32,7 @@ jobs:
           - runs-on: aws-amd64
             use-self-hosted: false
 
-    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
     - uses: AutoModality/action-clean@v1
@@ -120,8 +120,11 @@ jobs:
           - 24.3.4.2-1
         os:
           - macos-11
+          - macos-12-arm64
     runs-on: ${{ matrix.os }}
     steps:
+    - uses: emqx/self-hosted-cleanup-action@v1.0.3
+      if: ${{ matrix.os == 'macos-12-arm64' }}
     - uses: actions/checkout@v3
       with:
           fetch-depth: 0 # clone full git history
@@ -146,5 +149,5 @@ jobs:
         path: ./rebar3.crashdump
     - uses: actions/upload-artifact@v3
       with:
-        name: macos
+        name: ${{ matrix.os }}
         path: _packages/**/*.zip

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/release.yaml

@@ -7,7 +7,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.detect-profiles.outputs.profiles}}

.github/workflows/run_acl_migration_tests.yaml

@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         strategy:
             fail-fast: true
         env:

.github/workflows/run_fvt_tests.yaml

@@ -200,7 +200,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -251,7 +251,7 @@ jobs:
              otp:
              - 24.3.4.2-1
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         defaults:
           run:
             shell: bash
@@ -288,7 +288,7 @@ jobs:
           - relup_test_plan
           - relup_test_build
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         strategy:
           fail-fast: false
           matrix:

.github/workflows/run_test_cases.yaml

@@ -12,7 +12,7 @@ on:
 jobs:
   prepare:
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     outputs:
       fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
       docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
@@ -58,7 +58,7 @@ jobs:
   eunit_and_proper:
     needs: prepare
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     strategy:
       fail-fast: false
       matrix:
@@ -86,7 +86,7 @@ jobs:
   fast_ct:
     needs: prepare
     runs-on: ${{ matrix.runs-on }}
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     strategy:
       fail-fast: false
       matrix:
@@ -249,7 +249,7 @@ jobs:
       - fast_ct
       - docker_ct
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -3,6 +3,9 @@
 {VSN,
   [{"4.4.11",[
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+    {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
    ]},
    {"4.4.10",
@@ -11,6 +14,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -19,7 +24,6 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.4.9",
     [{add_module,emqx_rule_engine_jwt},
@@ -27,6 +31,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -35,7 +41,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.4.8",
@@ -44,6 +49,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -51,7 +58,6 @@
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
@@ -61,6 +67,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -68,7 +76,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -79,6 +86,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -91,13 +100,14 @@
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.4.4",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -105,7 +115,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
@@ -117,6 +126,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -131,13 +142,14 @@
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.4.2",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -153,13 +165,14 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.4.1",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -171,7 +184,6 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -182,6 +194,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -197,15 +211,18 @@
      {update,emqx_rule_metrics,{advanced,["4.4.0"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.4.11",[
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
+    {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
    ]},
    {"4.4.10",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -223,6 +240,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.9",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -241,6 +259,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.8",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -260,6 +279,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {<<"4\\.4\\.[6-7]">>,
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -279,6 +299,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.5",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -299,6 +320,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.4",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -319,6 +341,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.3",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
@@ -341,6 +364,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.2",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -364,6 +388,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.1",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -387,6 +412,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.0",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_rule_engine_api).
 
+-behaviour(gen_server).
+
 -include("rule_engine.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -156,6 +158,17 @@
             descr  => "List all events with detailed info"
            }).
 
+-export([start_link/0]).
+
+% gen_server Callbacks
+-export([ init/1
+        , handle_call/3
+        , handle_cast/2
+        , handle_info/2
+        , terminate/2
+        , code_change/3
+        ]).
+
 -export([ create_rule/2
         , update_rule/2
         , list_rules/2
@@ -207,10 +220,90 @@
             <<"Bad Arguments: ", R0/binary>>
         end).
 
+-define(T_CALL, 30000).
+
+start_link() ->
+    %% The caller process (the cowboy process serves the HTTP request) may times out and dies
+    %% before some time-consuming operations complete, e.g. creating rules/resources or testing
+    %% the connectivity on unreachable resources.
+    %% To avoid this problem, we delegate the operations to a gen_server.
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+create_rule(_Bindings, Params) ->
+    delegate_call({create_rule, _Bindings, Params}).
+
+update_rule(_Bindings, Params) ->
+    delegate_call({update_rule, _Bindings, Params}).
+
+delete_rule(_Bindings, Params) ->
+    delegate_call({delete_rule, _Bindings, Params}).
+
+create_resource(_Bindings, Params) ->
+    delegate_call({create_resource, _Bindings, Params}).
+
+update_resource(_Bindings, Params) ->
+    delegate_call({update_resource, _Bindings, Params}).
+
+start_resource(_Bindings, Params) ->
+    delegate_call({start_resource, _Bindings, Params}).
+
+delete_resource(_Bindings, Params) ->
+    delegate_call({delete_resource, _Bindings, Params}).
+
+%% delegate API calls to a single process.
+delegate_call(Req) ->
+    gen_server:call(?MODULE, Req, ?T_CALL).
+
+%%------------------------------------------------------------------------------
+%% gen_server callbacks
+%%------------------------------------------------------------------------------
+
+init([]) ->
+    {ok, #{}}.
+
+handle_call({create_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_create_rule(_Bindings, Params), State};
+
+handle_call({update_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_update_rule(_Bindings, Params), State};
+
+handle_call({delete_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_delete_rule(_Bindings, Params), State};
+
+handle_call({create_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_create_resource(_Bindings, Params), State};
+
+handle_call({start_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_start_resource(_Bindings, Params), State};
+
+handle_call({update_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_update_resource(_Bindings, Params), State};
+
+handle_call({delete_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_delete_resource(_Bindings, Params), State};
+
+handle_call(Req, _From, State) ->
+    ?LOG(error, "unexpected call: ~p", [Req]),
+    {reply, ignored, State}.
+
+handle_cast(Msg, State) ->
+    ?LOG(error, "unexpected cast: ~p", [Msg]),
+    {noreply, State}.
+
+handle_info(Info, State) ->
+    ?LOG(error, "unexpected info: ~p", [Info]),
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
 %%------------------------------------------------------------------------------
 %% Rules API
 %%------------------------------------------------------------------------------
-create_rule(_Bindings, Params) ->
+delegate_create_rule(_Bindings, Params) ->
     if_test(fun() -> test_rule_sql(Params) end,
             fun() -> do_create_rule(Params) end,
             Params).
@@ -250,7 +343,7 @@ do_create_rule2(ParsedParams) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-update_rule(#{id := Id0}, Params) ->
+delegate_update_rule(#{id := Id0}, Params) ->
     Id = urldecode(Id0),
     case parse_rule_params(Params, #{id => Id}) of
         {ok, ParsedParams} ->
@@ -280,7 +373,7 @@ show_rule(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     reply_with(fun emqx_rule_registry:get_rule/1, Id).
 
-delete_rule(#{id := Id0}, _Params) ->
+delegate_delete_rule(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     ok = emqx_rule_engine:delete_rule(Id),
     return(ok).
@@ -309,7 +402,7 @@ show_action(#{name := Name}, _Params) ->
 %%------------------------------------------------------------------------------
 %% Resources API
 %%------------------------------------------------------------------------------
-create_resource(#{}, Params) ->
+delegate_create_resource(#{}, Params) ->
     case parse_resource_params(Params) of
         {ok, ParsedParams} ->
             if_test(fun() -> do_create_resource(test_resource, maps:without([id], ParsedParams)) end,
@@ -382,7 +475,7 @@ get_resource_status(#{id := Id0}, _Params) ->
             return({error, 400, ?ERR_NO_RESOURCE(Id)})
     end.
 
-start_resource(#{id := Id0}, _Params) ->
+delegate_start_resource(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     case emqx_rule_engine:start_resource(Id) of
         ok ->
@@ -394,7 +487,7 @@ start_resource(#{id := Id0}, _Params) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-update_resource(#{id := Id0}, NewParams) ->
+delegate_update_resource(#{id := Id0}, NewParams) ->
     Id = urldecode(Id0),
     P1 = case proplists:get_value(<<"description">>, NewParams) of
         undefined -> #{};
@@ -419,7 +512,7 @@ update_resource(#{id := Id0}, NewParams) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-delete_resource(#{id := Id0}, _Params) ->
+delegate_delete_resource(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     case emqx_rule_engine:delete_resource(Id) of
         ok -> return(ok);

apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl

@@ -24,6 +24,8 @@
 
 -export([ start_locker/0
         , start_jwt_sup/0
+        , ensure_api_delegator_started/0
+        , ensure_api_delegator_stopped/0
         ]).
 
 -export([init/1]).
@@ -58,7 +60,8 @@ init([]) ->
                 type => worker,
                 modules => [emqx_rule_monitor]},
     JWTSup = jwt_sup_child_spec(),
-    {ok, {SupFlags, [Registry, Metrics, Monitor, JWTSup]}}.
+    API = api_delegator_sup_spec(),
+    {ok, {SupFlags, [Registry, Metrics, Monitor, JWTSup, API]}}.
 
 start_locker() ->
     Locker = #{id => emqx_rule_locker,
@@ -97,3 +100,31 @@ ensure_table(Name, Opts) ->
         error:badarg ->
             ok
     end.
+
+%% This is called by the emqx_rule_engine.appup.src when release upgrade
+ensure_api_delegator_started() ->
+    case supervisor:start_child(?MODULE, api_delegator_sup_spec()) of
+        {ok, _} -> ok;
+        {error, already_present} -> ok;
+        {error, {already_started, _Pid}} -> ok;
+        {error, _} = Err -> throw({failed_to_start_ensure_api, Err})
+    end.
+
+%% This is called by the emqx_rule_engine.appup.src when release downgrade
+ensure_api_delegator_stopped() ->
+    case supervisor:terminate_child(?MODULE, emqx_rule_engine_api) of
+        ok ->
+            %% don't crash if delete failed
+            supervisor:delete_child(?MODULE, emqx_rule_engine_api);
+        {error, not_found} -> ok
+    end.
+
+api_delegator_sup_spec() ->
+    #{
+        id => emqx_rule_engine_api,
+        start => {emqx_rule_engine_api, start_link, []},
+        restart => permanent,
+        shutdown => 5000,
+        type => worker,
+        modules => [emqx_rule_engine_api]
+    }.

build

@@ -195,6 +195,24 @@ make_zip() {
                     ;;
             esac
             ;;
+        amzn2)
+            case "$PKG_VSN" in
+                4.4.12*)
+                    # this is the first version for amzn2, no relup
+                    has_relup='no'
+                    ;;
+            esac
+            ;;
+        macos*)
+            case "$PKG_VSN" in
+                4.4.12*)
+                    # this is the first version for macos arm (M1/M2), no relup
+                    if [ "$ARCH" = arm64 ]; then
+                        has_relup='no'
+                    fi
+                    ;;
+            esac
+            ;;
     esac
     # shellcheck disable=SC2207
     bases=($(relup_db base-vsns "$PKG_VSN"))
@@ -228,10 +246,18 @@ make_zip() {
                     #   --apple-id <apple id> \
                     #   --password <apple id password>
                     #   --team-id <apple team id> <submission-id>
-                xcrun notarytool submit \
+                echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
-                      --apple-id "${APPLE_ID}" \
+                notarytool_output="$(xcrun notarytool submit \
-                      --password "${APPLE_ID_PASSWORD}" \
+                                           --apple-id "${APPLE_ID}" \
-                      --team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait
+                                           --password "${APPLE_ID_PASSWORD}" \
+                                           --team-id "${APPLE_TEAM_ID}" "${target_zip}" \
+                                           --no-progress \
+                                           --wait)"
+                echo "$notarytool_output"
+                echo "$notarytool_output" | grep -q 'status: Accepted' || {
+                    echo 'Notarization failed';
+                    exit 1;
+                }
             fi
             # sha256sum may not be available on macos
             openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2  > "${target_zip}.sha256"

changes/v4.4.11-en.md

@@ -18,9 +18,9 @@
 - Added support for specifying custom modules for custom authentication [#9297](https://github.com/emqx/emqx/pull/9297).
   To support simple authentication rules, it is no longer necessary to implement a full-blown plugin.
 
-- Added a JWT management for Rule-Engin, for creating and refreshing JWT tokens in rule engine actions [#9241](https://github.com/emqx/emqx/pull/9241).
+- Added a JWT management for Rule-Engine, for creating and refreshing JWT tokens in rule engine actions [#9241](https://github.com/emqx/emqx/pull/9241).
   This feature is so far only used in EMQX Enterprise Google PubSub integration.
-  Can be used as webhook integration's JWT authenticationa against the webhook service endpoint.
+  Can be used as webhook integration's JWT authentication against the webhook service endpoint.
 
 - Update `gen_rpc` dependency to 3.0.0.
 

changes/v4.4.12-en.md

@@ -1,13 +1,27 @@
-### Enhancements
+# v4.4.12
+
+## Enhancements
+
+- Added Amazon Linxu 2 (amzn2) package release [#9498](https://github.com/emqx/emqx/pull/9498).
 
 - Upgrade http client library `ehttpc` from `0.2.1` to `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456).
 
 - Add more PSK ciphers support [#9493](https://github.com/emqx/emqx/pull/9493).
 
-### Bug Fixes
+- Users can define the `externalTrafficPolicy` of service in EMQX Helm Chart [#9527](https://github.com/emqx/emqx/pull/9527).
+
+- Start building MacOS packages for Apple Silicon M1/M2 [#9558](https://github.com/emqx/emqx/pull/9558).
+
+## Bug Fixes
 
 - Fixed load bootstrap file when no bootstrap user in `mqtt_app` [#9474](https://github.com/emqx/emqx/pull/9474).
 
 - Trigger `message.dropped` hook when QoS2 message is resend by client with a same packet id, or 'awaiting_rel' queue is full [#9486](https://github.com/emqx/emqx/pull/9486).
 
 - When dashboard creates a new user, the password length must match 3-32 and the format is `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475).
+
+- Fixed EMQX Helm Chart using incorrect secret values when custom credentials are provided [#9508](https://github.com/emqx/emqx/pull/9508).
+
+- Fixed EMQX Helm Chart can not set JSON type value for EMQX Broker configuration items [#9504](https://github.com/emqx/emqx/pull/9504).
+
+- When resource creation is too slow, there may be some temporary probing connections left [#9539](https://github.com/emqx/emqx/pull/9539).

changes/v4.4.12-zh.md

@@ -1,13 +1,28 @@
-### 增强
+# v4.4.12
+
+
+## 增强
+
+- 发布 Amazon Linux 2 安装包 [#9498](https://github.com/emqx/emqx/pull/9498)。
 
 - HTTP 客户端库 `ehttpc` 从 `0.2.1` 升级到 `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456)。
 
 - 支持更多的 PSK 密码套件[#9493](https://github.com/emqx/emqx/pull/9493)。
 
-### 修复
+- 用户可以在 EMQX Helm Chart 中自定义 service 资源的 `externalTrafficPolicy` [#9527](https://github.com/emqx/emqx/pull/9527)。
+
+- 新增 Apple Silicon M1/M2 安装包发布 [#9558](https://github.com/emqx/emqx/pull/9558)。
+
+## 修复
 
 - 修复 mqtt_app 表内没有 boostrap user 里未导入用户的问题 [#9474](https://github.com/emqx/emqx/pull/9474).
 
 - 当 QoS2 消息被重发(使用相同 Packet ID)，或当 'awaiting_rel' 队列已满时，触发消息丢弃钩子(`message.dropped`)及计数器 [#9486](https://github.com/emqx/emqx-enterprise/pull/9486)。
 
 - 当控制台创建新用户时，密码长度必须在 3-32 之间，且格式为 `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475)。
+
+- 修复了 EMQX Helm Chart 中当用户使用自定义的用户名和密码时，创建的 Secret 资源不正确问题 [#9508](https://github.com/emqx/emqx/pull/9508)。
+
+- 修复 EMQX Helm Chart 无法配置 value 为 JSON 类型的 EMQX Broker 配置项 [#9504](https://github.com/emqx/emqx/pull/9504)。
+
+- 当创建资源过慢的情况下，有可能会残留一些用来探活的临时的连接 [#9539](https://github.com/emqx/emqx/pull/9539)。

deploy/charts/emqx/README.md

@@ -68,6 +68,7 @@ Parameter  | Description | Default Value
 `service.loadBalancerIP`  | loadBalancerIP for Service |	`nil`
 `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
 `service.externalIPs` |	ExternalIPs for the service |	`[]`
+`service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
 `service.annotations` |	Service annotations (evaluated as a template) |	`{}`
 `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
 `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard

deploy/charts/emqx/templates/configmap.env.yaml

@@ -19,15 +19,18 @@ data:
   EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname"
   EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
   {{- else if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY)  "dns"  }}
-  EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
   EMQX_CLUSTER__DNS__APP: {{ .Release.Name }}
+  EMQX_CLUSTER__DNS__NAME: "{{ include "emqx.fullname" . }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
   EMQX_CLUSTER__DNS__TYPE: "srv"
   {{- end -}}
   {{- range $index, $value := .Values.emqxConfig }}
   {{- if $value }}
   {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+  {{- if or (kindIs "map" $value) (kindIs "slice" $value) }}
+  {{ print "EMQX_" $key }}: {{ tpl (printf "%q" (toJson $value)) $ }}
+  {{- else }}
   {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
   {{- end }}
-  {{- end}}
+  {{- end }}
-
+  {{- end }}
 {{- end }}

deploy/charts/emqx/templates/secret.yaml

@@ -6,14 +6,6 @@ metadata:
   namespace: {{ .Release.Namespace }}
 type: kubernetes.io/basic-auth
 stringData:
-  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }}
+  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID | default "admin" }}
-  username: admin
+  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET | default "public" }}
-  {{- else }}
-  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }}
-  {{- end }}
-  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }}
-  password: public
-  {{- else }}
-  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}}
-  {{- end }}
 {{- end }}

deploy/charts/emqx/templates/service.yaml

@@ -14,6 +14,9 @@ metadata:
   {{- end }}
 spec:
   type: {{ .Values.service.type }}
+  {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }}
+  externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | default "Cluster" }}
+  {{- end }}
   {{- if eq .Values.service.type "LoadBalancer" }}
   {{- if .Values.service.loadBalancerIP }}
   loadBalancerIP: {{ .Values.service.loadBalancerIP }}

deploy/charts/emqx/values.yaml

@@ -72,7 +72,7 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
-  ## Cluster discovery by k8s
+  ## Cluster discovery by 'k8s' or 'dns'
   EMQX_CLUSTER__DISCOVERY: "k8s"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"
@@ -174,6 +174,12 @@ service:
   ## Set the ExternalIPs
   ##
   externalIPs: []
+  ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints.
+  ## There are two available options: Cluster (default) and Local.
+  ## Cluster obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.
+  ## Local preserves the client source IP and avoids a second hop for LoadBalancer and NodePort type Services, but risks potentially imbalanced traffic spreading.
+  ##
+  externalTrafficPolicy: "Cluster"
   ## Provide any additional annotations which may be required. Evaluated as a template
   ##
   annotations: {}

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.12-alpha.1"}).
+-define(EMQX_RELEASE, {opensource, "4.4.12-alpha.2"}).
 
 -else.
 

rebar.config

@@ -29,7 +29,7 @@
 
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}.
+{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb, emqx_cover]}.
 
 {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}.
 
@@ -40,6 +40,7 @@
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {redbug, "2.0.7"}
+    , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.2"}}}
     , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.8"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.4"}}}

rebar.config.erl

@@ -253,6 +253,7 @@ relx_apps(ReleaseType) ->
     , {ekka, load}
     , {emqx_plugin_libs, load}
     , observer_cli
+    , {covertool, load}
     ]
     ++ [emqx_modules || not is_enterprise()]
     ++ [emqx_license || is_enterprise()]

scripts/get-distro.sh

@@ -14,7 +14,11 @@ case "$UNAME" in
         SYSTEM="${DIST}${VERSION_ID}"
         ;;
     Linux)
-        if grep -q -i 'rhel' /etc/*-release; then
+        # /etc/os-release on amazon linux 2 contains both rhel and centos strings
+        if grep -q -i 'amzn' /etc/*-release; then
+            DIST='amzn'
+            VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+        elif grep -q -i 'rhel' /etc/*-release; then
             DIST='el'
             VERSION_ID="$(rpm --eval '%{rhel}')"
         elif grep -q -i 'centos' /etc/*-release; then

scripts/macos-sign-binaries.sh

@@ -43,10 +43,24 @@ done
 security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
 
 # known runtime executables and binaries
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+         "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
 # other files from runtime and dependencies
-for f in asn1rt_nif.so bcrypt_nif.so crypto.so otp_test_engine.so crypto_callback.so jiffy.so crc32cer_nif.so sasl_auth.so snappyer.so odbcserver; do
+for f in \
+        asn1rt_nif.so \
+        bcrypt_nif.so \
+        crypto.so \
+        otp_test_engine.so \
+        crypto_callback.so \
+        jiffy.so \
+        crc32cer_nif.so \
+        sasl_auth.so \
+        snappyer.so \
+        odbcserver \
+    ; do
     find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
 done

scripts/rel_otp_apps.eterm

@@ -14,4 +14,5 @@
 , {mnesia, load}
 , xmerl
 , tools
+, covertool % this is not really a otp app, but we don't need to worry about relup of it
 ].

scripts/relup-base-vsns.escript

@@ -162,16 +162,26 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
     Froms1 =
         case get_system() of
             %% we do not support relup for windows
-            "windows" ->
+            {"windows", _} ->
                 [];
             %% debian11 is introduced since v4.4.2 and e4.4.2
             %% exclude tags before them
-            "debian11" ->
+            {"debian11", _} ->
                 lists:filter(
                   fun(Vsn) ->
                           not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>])
                   end, Froms0);
-            _ ->
+            %% amzn2 is introduced since v4.4.12 and e4.4.12
+            %% exclude tags before them
+            {"amzn2", _} ->
+                Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
+                lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
+            %% macos arm64 (M1/M2) packages are introduced since v4.4.12 and e4.4.12
+            %% exclude tags before them
+            {"macos" ++ _, "aarch64" ++ _} ->
+                Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
+                lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
+            {_, _} ->
                 Froms0
         end,
     lists:filter(
@@ -179,11 +189,12 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
       Froms1).
 
 get_system() ->
+    Arch = erlang:system_info(system_architecture),
     case os:getenv("SYSTEM") of
         false ->
-            string:trim(os:cmd("./scripts/get-distro.sh"));
+            {string:trim(os:cmd("./scripts/get-distro.sh")), Arch};
         System ->
-            System
+            {System, Arch}
     end.
 
 %% assumes that's X.Y.Z, without pre-releases

scripts/xref_check.eterm

@@ -5,6 +5,7 @@
    , excl_apps =>
          [ observer
          , redbug
+         , covertool
          ]
    , excl_mods =>
          [ hipe_unified_loader
@@ -14,6 +15,7 @@
          , basho_bench_driver_erldis
          , release_handler
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}},
@@ -239,6 +241,7 @@
    , analysis => undefined_functions
    , excl_apps =>
          [ observer
+         , covertool
          ]
    , excl_mods =>
          [ systools
@@ -246,6 +249,7 @@
          , release_handler
          , systools_relup
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{'Elixir.Atom',to_string,1},

src/emqx_cover.erl

@@ -18,16 +18,68 @@
 %% It is used to collect coverage data when running blackbox test
 -module(emqx_cover).
 
+-include_lib("covertool/include/covertool.hrl").
+
+-ifdef(EMQX_ENTERPRISE).
+-define(OUTPUT_APPNAME, 'EMQX Enterprise').
+-else.
+-define(OUTPUT_APPNAME, 'EMQX').
+-endif.
+
 -export([start/0,
          start/1,
-         export_and_stop/1
+         abort/0,
+         export_and_stop/1,
+         lookup_source/1
         ]).
 
+%% This is a ETS table to keep a mapping of module name (atom) to
+%% .erl file path (relative path from project root)
+%% We needed this ETS table because the source file information
+%% is missing from the .beam metadata sicne we are using 'deterministic'
+%% compile flag.
+-define(SRC, emqx_cover_module_src).
+
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
 start() ->
     start(#{}).
 
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
+%% Supported options:
+%% - project_root: the directory to search for .erl source code
+%% - debug_secret_file: only applicable to EMQX Enterprise
 start(Opts) ->
-    ok = maybe_set_secret(),
+    ok = abort(),
+    %% spawn a ets table owner
+    %% this implementation is kept dead-simple
+    %% because there is no concurrency requirement
+    Parent = self(),
+    {Pid, Ref} =
+        erlang:spawn_monitor(
+          fun() ->
+                  true = register(?SRC, self()),
+                  _ = ets:new(?SRC, [named_table, public]),
+                  _ = Parent ! {started, self()},
+                  receive
+                      stop ->
+                          ok
+                  end
+          end),
+    receive
+        {started, Pid} ->
+            ok;
+        {'DOWN', Ref, process, Pid, Reason} ->
+            throw({failed_to_start, Reason})
+    after
+        1000 ->
+            throw({failed_to_start, timeout})
+    end,
+    Modules = modules(Opts),
+    ok = maybe_set_secret(Opts),
     case cover:start() of
         {ok, _Pid} ->
             ok;
@@ -36,28 +88,97 @@ start(Opts) ->
         Other ->
             throw(Other)
     end,
-    ok = cover_compile(Opts).
+    ok = cover_compile(Modules),
-
+    io:format("cover-compiled ~p modules~n", [length(Modules)]),
-export_and_stop(Path) ->
+    ok = build_source_mapping(Opts, sets:from_list(Modules, [{version, 2}])),
-    ok = cover:export(Path),
+    CachedModulesCount = ets:info(?SRC, size),
-    _ = cover:stop(),
+    io:format("source-cached ~p modules~n", [CachedModulesCount]),
     ok.
 
-maybe_set_secret() ->
+%% @doc Abort cover data collection without exporting.
-    case os:getenv("EMQX_DEBUG_SECRET_FILE") of
+abort() ->
-        false ->
+    _ = cover:stop(),
-            ok;
+    case whereis(?SRC) of
+        undefined -> ok;
+        Pid ->
+            Ref = monitor(process, Pid),
+            exit(Pid, kill),
+            receive
+                {'DOWN', Ref, process, Pid, _} ->
+                    ok
+            end
+    end,
+    ok.
+
+%% @doc Export coverage report (xml) format.
+%% e.g. `emqx_cover:export_and_stop("/tmp/cover.xml").'
+export_and_stop(Path) when is_list(Path) ->
+    ProjectRoot = get_project_root(),
+    Config = #config{appname = ?OUTPUT_APPNAME,
+                     sources = [ProjectRoot],
+                     output = Path,
+                     lookup_source = fun ?MODULE:lookup_source/1
+                    },
+    covertool:generate_report(Config, cover:modules()).
+
+build_source_mapping(Opts, Modules) ->
+    Default = os_env("EMQX_PROJECT_ROOT"),
+    case maps:get(project_root, Opts, Default) of
+        "" ->
+            io:format(standard_error, "EMQX_PROJECT_ROOT is not set", []),
+            throw(emqx_project_root_undefined);
+        Dir ->
+            ok = put_project_root(Dir),
+            ok = do_build_source_mapping(Dir, Modules)
+    end.
+
+get_project_root() ->
+    [{_, Dir}] = ets:lookup(?SRC, {root, ?OUTPUT_APPNAME}),
+    Dir.
+
+put_project_root(Dir) ->
+    _ = ets:insert(?SRC, {{root, ?OUTPUT_APPNAME}, Dir}),
+    ok.
+
+do_build_source_mapping(Dir, Modules) ->
+    All = filelib:wildcard("**/*.erl", Dir),
+    lists:foreach(
+      fun(Path) ->
+              ModuleNameStr = filename:basename(Path, ".erl"),
+              Module = list_to_atom(ModuleNameStr),
+              case sets:is_element(Module, Modules) of
+                  true ->
+                      ets:insert(?SRC, {Module, Path});
+                  false ->
+                      ok
+              end
+      end, All),
+    ok.
+
+lookup_source(Module) ->
+    case ets:lookup(?SRC, Module) of
+        [{_, Path}] ->
+            Path;
+        [] ->
+            false
+    end.
+
+maybe_set_secret(Opts) ->
+    Default = os_env("EMQX_DEBUG_SECRET_FILE"),
+    case maps:get(debug_secret_file, Opts, Default) of
         "" ->
             ok;
         File ->
             ok = emqx:set_debug_secret(File)
     end.
 
-cover_compile(_Opts) ->
+modules(_Opts) ->
     %% TODO better filter based on Opts,
     %% e.g. we may want to see coverage info for ehttpc
     Filter = fun is_emqx_module/1,
-    Modules = find_modules(Filter),
+    find_modules(Filter).
+
+cover_compile(Modules) ->
     Results = cover:compile_beam(Modules),
     Errors = lists:filter(fun({ok, _}) -> false;
                              (_) -> true
@@ -66,14 +187,14 @@ cover_compile(_Opts) ->
         [] ->
             ok;
         _ ->
-            io:format(user, "failed_to_cover_compile:~n~p~n", [Errors]),
+            io:format("failed_to_cover_compile:~n~p~n", [Errors]),
             throw(failed_to_cover_compile)
     end.
 
 find_modules(Filter) ->
     All = code:all_loaded(),
-    F = fun({M, _BeamPath}) -> Filter(M) end,
+    F = fun({M, _BeamPath}) -> Filter(M) andalso {true, M} end,
-    lists:filter(F, All).
+    lists:filtermap(F, All).
 
 is_emqx_module(?MODULE) ->
     %% do not cover-compile self
@@ -85,3 +206,6 @@ is_emqx_module(Module) ->
         _ ->
             false
     end.
+
+os_env(Name) ->
+    os:getenv(Name, "").