Merge remote-tracking branch 'origin/main-v4.3' into 1005-sync-v44-upstreams

.github/actions/package-macos/action.yaml

@@ -1,8 +1,5 @@
 name: 'Create MacOS package'
 inputs:
-  profile: # emqx, emqx-enterprise
-    required: true
-    type: string
   otp: # 24.2.1-1, 23.3.4.9-3
     required: true
     type: string
@@ -49,7 +46,7 @@ runs:
         kerl update releases
         kerl build ${{ inputs.otp }}
         kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
-    - name: build ${{ inputs.profile }}
+    - name: build
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
         APPLE_SIGN_BINARIES: 1
@@ -64,14 +61,17 @@ runs:
         . $HOME/.kerl/${{ inputs.otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        make ${{ inputs.profile }}-zip
+        make ${EMQX_NAME}-zip
-    - name: test ${{ inputs.profile }}
+    - name: test
       shell: bash
       run: |
-        pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip)
+        pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip)
-        unzip -q _packages/${{ inputs.profile }}/$pkg_name
+        unzip -q _packages/${EMQX_NAME}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        # test with a spaces in path
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        mv ./emqx "./emqx  home/"
+        cd "./emqx  home/"
+        gsed -i '/emqx_telemetry/d' data/loaded_plugins
+        ./bin/emqx start || cat log/erlang.log.1
         ready='no'
         for i in {1..10}; do
           if curl -fs 127.0.0.1:18083 > /dev/null; then
@@ -82,14 +82,15 @@ runs:
         done
         if [ "$ready" != "yes" ]; then
           echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
+          cat log/erlang.log.1
           exit 1
         fi
-        ./emqx/bin/emqx_ctl status
+        ./bin/emqx_ctl status
-        if ! ./emqx/bin/emqx stop; then
+        if ! ./bin/emqx stop; then
-          cat emqx/log/erlang.log.1 || true
+          cat log/erlang.log.1 || true
-          cat emqx/log/emqx.log.1 || true
+          cat log/emqx.log.1 || true
           echo "failed to stop emqx"
           exit 1
         fi
-        rm -rf emqx
+        cd ..
+        rm -rf "emqx  home"

.github/workflows/build_packages.yaml

@@ -108,11 +108,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
           - 24.1.5-3
-        exclude:
-          - profile: emqx-edge
         os:
           - macos-11
     runs-on: ${{ matrix.os }}
@@ -127,9 +124,12 @@ jobs:
         ln -s . source
         unzip -q source.zip
         rm source source.zip
+    - id: detect-profiles
+      uses: ./.github/actions/detect-profiles
+      with:
+        ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
     - uses: ./.github/actions/package-macos
       with:
-        profile: ${{ matrix.profile }}
         otp: ${{ matrix.otp }}
         os: ${{ matrix.os }}
         apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
@@ -138,8 +138,8 @@ jobs:
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
     - uses: actions/upload-artifact@v1
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${EMQX_NAME}-${{ matrix.otp }}
-        path: _packages/${{ matrix.profile }}/.
+        path: _packages/${EMQX_NAME}/.
 
   linux:
     runs-on: ubuntu-20.04

.github/workflows/build_slim_packages.yaml

@@ -110,8 +110,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile:
-          - emqx
         otp:
           - 24.1.5-3
         os:
@@ -124,39 +122,18 @@ jobs:
         ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
     - uses: ./.github/actions/package-macos
       with:
-        profile: ${{ matrix.profile }}
         otp: ${{ matrix.otp }}
         os: ${{ matrix.os }}
         apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - name: test
+    - uses: actions/upload-artifact@v1
-      run: |
+      if: failure()
-        pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip)
+      with:
-        unzip -q $pkg_name
+        name: rebar3.crashdump
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        path: ./rebar3.crashdump
-        # test with a spaces in path
+
-        mv ./emqx "./emqx  home/"
-        cd "./emqx  home/"
-        ./bin/emqx start || cat log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat log/erlang.log.1
-          exit 1
-        fi
-        ./bin/emqx_ctl status
-        ./bin/emqx stop
-        cd ..
-        rm -rf "emqx  home"
     - uses: actions/upload-artifact@v2
       with:
         name: macos

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -115,7 +115,7 @@ init_per_suite(Config) ->
 end_per_suite(_Cfg) ->
     deinit_mongo_data(),
     %% avoid inter-suite flakiness
-    ok = emqx_mod_acl_internal:load([]),
+    emqx_mod_acl_internal:load([]),
     emqx_ct_helpers:stop_apps([emqx_auth_mongo]).
 
 set_special_confs(emqx) ->
@@ -186,6 +186,8 @@ end_per_testcase(TestCase, Config)
  when TestCase =:= t_available_acl_query_timeout;
       TestCase =:= t_acl_superuser_timeout;
       TestCase =:= t_authn_no_connection;
+      TestCase =:= t_available_authn_query_timeout;
+      TestCase =:= t_authn_timeout;
       TestCase =:= t_available_acl_query_no_connection ->
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),

scripts/rel/sync-remotes.sh

@@ -170,27 +170,29 @@ remote_refs() {
 upstream_branches() {
     local base="$1"
     case "$base" in
-        release-v43|main-v4.3)
+        release-v43)
-            ## no upstream for 4.3 opensource
             remote_ref "$base"
             ;;
         release-v44)
             remote_refs "$base" 'release-v43'
             ;;
-        main-v4.4)
-            remote_refs "$base" 'main-v4.3'
-            ;;
         release-e43)
             remote_refs "$base" 'release-v43'
             ;;
-        main-v4.3-enterprise)
-            remote_refs "$base" 'main-v4.3'
-            ;;
         release-e44)
             remote_refs "$base" 'release-v44' 'release-e43' 'release-v43'
             ;;
+        main-v4.3)
+            remote_refs "$base" 'release-v43'
+            ;;
+        main-v4.4)
+            remote_refs "$base" 'release-v44' 'main-v4.3'
+            ;;
+        main-v4.3-enterprise)
+            remote_refs "$base" 'release-e43' 'main-v4.3'
+            ;;
         main-v4.4-enterprise)
-            remote_refs "$base" 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3'
+            remote_refs "$base" 'release-e44' 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3'
             ;;
     esac
 }

scripts/update-appup.sh

@@ -8,12 +8,20 @@
 set -euo pipefail
 set -x
 
-usage() {
-    echo "$0 PROFILE"
-}
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 
+usage() {
+    echo "$0 PROFILE [options]"
+    echo "options:"
+    echo "--skip-build:      Skip building the profile only to re-generate the appup files."
+    echo "--skip-build-base: This script by default forces a git clean before rebuilding on the base version "
+    echo "                   this option is useful when you are sure the past builds can be trusted,"
+    echo "                   that is, there were no re-tags or anything."
+    echo "--check:           Exit with non-zero code if there is git diff after the execution."
+    echo "                   Mostly used in CI."
+}
+
 PROFILE="${1:-}"
 case "$PROFILE" in
     emqx-ee)
@@ -49,7 +57,7 @@ ESCRIPT_ARGS=( '' )
 while [ "$#" -gt 0 ]; do
     case $1 in
     -h|--help)
-        help
+        usage
         exit 0
         ;;
     --skip-build)
@@ -101,7 +109,7 @@ else
     pushd "${PREV_DIR_BASE}/${PREV_TAG}"
     if [ "$NEW_COPY" = 'no' ]; then
         REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')"
-        git fetch "$REMOTE"
+        git fetch "$REMOTE" --tags -f
     fi
     git reset --hard
     git clean -ffdx