ci: use workflow_call

.github/actions/pr-sanity-checks/action.yaml

@@ -17,69 +17,58 @@ runs:
         BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
         AFTER_REF: ${{ github.sha }}
       run: |
+        echo "==> gitlint --commits $BEFORE_REF..$AFTER_REF"
         pip install gitlint
         gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint
+        echo "==> gitlint ok"
     - name: Check line-break at EOF
       shell: bash
-      run: ./scripts/check-nl-at-eof.sh
+      run: |
+        echo "==> ./scripts/check-nl-at-eof.sh"
+        ./scripts/check-nl-at-eof.sh
+        echo "==> ./scripts/check-nl-at-eof.sh ok"
     - name: Run shellcheck
       shell: bash
       run: |
+        echo "==> ./scripts/shellcheck.sh"
         apt-get update -y
         apt-get install -y shellcheck
         ./scripts/shellcheck.sh
-        echo "success"
+        echo "==> ./scripts/shellcheck.sh ok"
-    - name: Ensure rebar
+    - name: Check apps version
       shell: bash
-      run: make ensure-rebar3
+      run: |
+        echo "==> ./scripts/apps-version-check.sh"
+        ./scripts/apps-version-check.sh
+        echo "==> ./scripts/apps-version-check.sh ok"
     - name: Check Erlang code formatting
       shell: bash
-      run: ./scripts/check-format.sh
+      run: |
-    - name: Apps version check
+        echo "==> ./scripts/check-format.sh"
-      shell: bash
+        ./scripts/check-format.sh
-      run: ./scripts/apps-version-check.sh
+        echo "==> check-format ok"
     - name: Run elvis check
       shell: bash
-      run: ./scripts/elvis-check.sh $GITHUB_BASE_REF
+      run: |
+        echo "==> ./scripts/elvis-check.sh $GITHUB_BASE_REF"
+        ./scripts/elvis-check.sh $GITHUB_BASE_REF
+        echo "==> ./scripts/elvis-check.sh $GITHUB_BASE_REF ok"
     - name: Setup mix
       env:
         MIX_ENV: emqx-enterprise
         PROFILE: emqx-enterprise
       shell: bash
       run: |
+        echo "==> mix deps.get"
         mix local.hex --force
         mix local.rebar --force
-        mix deps.get
+        echo "==> mix deps.get ok"
-    - name: Run check-deps-integrity.escript
-      env:
-        MIX_ENV: emqx-enterprise
-        PROFILE: emqx-enterprise
-      shell: bash
-      run: ./scripts/check-deps-integrity.escript
     - name: Check Elixir code formatting
       env:
         MIX_ENV: emqx-enterprise
         PROFILE: emqx-enterprise
       shell: bash
-      run: mix format --check-formatted
+      run: |
-    - name: Check applications
+        echo "==> mix format --check-formatted"
-      env:
+        mix format --check-formatted
-        MIX_ENV: emqx-enterprise
+        echo "==> mix format ok"
-        PROFILE: emqx-enterprise
-      shell: bash
-      run: ./scripts/check-elixir-applications.exs
-    - name: Check elixir deps
-      env:
-        MIX_ENV: emqx-enterprise
-        PROFILE: emqx-enterprise
-      shell: bash
-      run: ./scripts/check-elixir-deps-discrepancies.exs
-    - name: Upload produced lock files
-      uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: produced_lock_files
-        path: |
-          mix.lock
-          rebar.lock
-        retention-days: 1

.github/workflows/_pr_entrypoint.yaml

@@ -3,13 +3,117 @@ name: PR Entrypoint
 on:
   pull_request:
 
+env:
+  IS_CI: "yes"
+
 jobs:
   sanity-checks:
-    runs-on: aws-amd64
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-    container: ghcr.io/emqx/emqx-builder/5.1-1:1.14.5-25.3.2-1-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+      host: ${{ steps.matrix.outputs.host }}
+      docker: ${{ steps.matrix.outputs.docker }}
 
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - uses: ./.github/actions/pr-sanity-checks
+      - name: Build matrix
+        id: matrix
+        run: |
+          APPS="$(./scripts/find-apps.sh --ci)"
+          MATRIX="$(echo "${APPS}" | jq -c '
+            [
+              (.[] | select(.profile == "emqx") | . + {
+                builder: "5.1-3",
+                otp: "25.3.2-1",
+                elixir: "1.14.5"
+              }),
+              (.[] | select(.profile == "emqx-enterprise") | . + {
+                builder: "5.1-3",
+                otp: ["25.3.2-1"][],
+                elixir: "1.14.5"
+              })
+            ]
+          ')"
+          echo "${MATRIX}" | jq
+          MATRIX_PREPARED="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
+          MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
+          MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
+          echo "matrix=${MATRIX_PREPARED}" | tee -a $GITHUB_OUTPUT
+          echo "ct-host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
+          echo "ct-docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
+
+  compile:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
+    needs: [sanity-checks]
+    strategy:
+      matrix:
+        profile:
+          - emqx
+          - emqx-enterprise
+    outputs:
+      version-emqx: ${{ steps.compile.outputs.version-emqx }}
+      version-emqx-enterprise: ${{ steps.compile.outputs.version-emqx-enterprise }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Work around https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - id: compile
+        env:
+          PROFILE: ${{ matrix.profile }}
+        run: |
+          make $PROFILE
+          ./pkg-vsn.sh $PROFILE > VSN
+          echo "version-$PROFILE=$(cat VSN)" | tee -a $GITHUB_OUTPUT
+          zip -ryq $PROFILE.zip .
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.profile }}
+          path: ${{ matrix.profile }}.zip
+          retention-days: 1
+
+  # run_test_cases:
+  #   needs: [sanity-checks, compile]
+  #   uses: ./.github/workflows/run_test_cases.yaml
+  #   with:
+  #     matrix: ${{ needs.sanity-checks.outputs.matrix }}
+  #     ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
+  #     ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
+
+  # build_slim_packages:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/build_slim_packages.yaml
+
+  call_conf_tests:
+    needs:
+      - sanity-checks
+      - compile
+    uses: ./.github/workflows/run_conf_tests.yaml
+
+  # check_deps_integrity:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/check_deps_integrity.yaml
+
+  # elixir_release:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/elixir_release.yaml
+
+  # run_relup_tests:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/run_relup_tests.yaml
+
+  # run_jmeter_tests:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/run_jmeter_tests.yaml
+
+  # run_fvt_tests:
+  #   needs: [sanity-checks]
+  #   uses: ./.github/workflows/run_fvt_tests.yaml

.github/workflows/apps_version_check.yaml

@@ -1,18 +0,0 @@
-name: Check Apps Version
-
-on:
-  workflow_run:
-    workflows:
-      - PR Entrypoint
-    types:
-      - completed
-
-jobs:
-  check_apps_version:
-    runs-on: aws-amd64
-    if: github.event.workflow_run.conclusion == 'success'
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check apps version
-        run: ./scripts/apps-version-check.sh

.github/workflows/build_packages.yaml

@@ -88,7 +88,7 @@ jobs:
         fetch-depth: 0
 
     - uses: ilammy/msvc-dev-cmd@v1.12.0
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - name: build

.github/workflows/build_slim_packages.yaml

@@ -4,6 +4,8 @@ concurrency:
   group: slim-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
 # on:
 #   push:
 #     branches:
@@ -24,7 +26,7 @@ concurrency:
 
 jobs:
   linux:
-    runs-on: aws-amd64
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
 
     strategy:
       fail-fast: false
@@ -96,7 +98,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ilammy/msvc-dev-cmd@v1.12.0
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: ${{ matrix.otp }}
     - name: build
@@ -165,7 +167,7 @@ jobs:
         path: _packages/**/*
 
   docker:
-    runs-on: aws-amd64
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
 
     strategy:
       fail-fast: false
@@ -234,7 +236,7 @@ jobs:
         profile:
         - emqx
         - emqx-enterprise
-    runs-on: aws-amd64
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     steps:
     - uses: actions/download-artifact@v3
       name: Download schema dump

.github/workflows/check_deps_integrity.yaml

@@ -0,0 +1,39 @@
+name: Check integrity of rebar and mix dependencies
+
+on:
+  workflow_call:
+
+jobs:
+  check_deps_integrity:
+    runs-on: aws-amd64
+    container: ghcr.io/emqx/emqx-builder/5.1-1:1.14.5-25.3.2-1-ubuntu22.04
+    steps:
+      - uses: actions/checkout@v3
+      - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - run: make ensure-rebar3
+      - run: ./scripts/check-deps-integrity.escript
+      - name: Setup mix
+        env:
+          MIX_ENV: emqx-enterprise
+          PROFILE: emqx-enterprise
+        run: |
+          mix local.hex --force
+          mix local.rebar --force
+          mix deps.get
+      - run: ./scripts/check-elixir-deps-discrepancies.exs
+        env:
+          MIX_ENV: emqx-enterprise
+          PROFILE: emqx-enterprise
+      - run: ./scripts/check-elixir-applications.exs
+        env:
+          MIX_ENV: emqx-enterprise
+          PROFILE: emqx-enterprise
+      - name: Upload produced lock files
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: produced_lock_files
+          path: |
+            mix.lock
+            rebar.lock
+          retention-days: 1

.github/workflows/elixir_release.yaml

@@ -5,13 +5,15 @@ concurrency:
   group: mix-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
 # on:
 #   pull_request:
 #   workflow_dispatch:
 
 jobs:
   elixir_release_build:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     strategy:
       matrix:
         profile:

.github/workflows/run_conf_tests.yaml

@@ -4,6 +4,9 @@ concurrency:
   group: conftest-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
+
 # on:
 #   push:
 #     branches:
@@ -19,7 +22,7 @@ env:
 
 jobs:
   run_conf_tests:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     strategy:
       fail-fast: false
       matrix:
@@ -29,25 +32,28 @@ jobs:
     container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
     steps:
       - uses: AutoModality/action-clean@v1
-      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
         with:
-          path: source
+          name: ${{ matrix.profile }}
-      - name: build_package
+      - name: extract artifact
-        working-directory: source
         run: |
-          make ${{ matrix.profile }}
+          unzip -o -q ${{ matrix.profile }}.zip
-      - name: run_tests
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-        working-directory: source
+      - name: check example config files
         env:
           PROFILE: ${{ matrix.profile }}
-        run: |
+        run: ./scripts/test/check-example-configs.sh
-          ./scripts/conf-test/run.sh
+      - name: run conf tests
+        env:
+          PROFILE: ${{ matrix.profile }}
+        run: ./scripts/conf-test/run.sh
       - name: print_erlang_log
         if: failure()
         run: |
-          cat source/_build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
+          cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs-${{ matrix.profile }}
-          path: source/_build/${{ matrix.profile }}/rel/emqx/logs
+          path: _build/${{ matrix.profile }}/rel/emqx/logs
+

.github/workflows/run_emqx_app_tests.yaml

@@ -1,80 +0,0 @@
-name: Check emqx app standalone
-
-## apps/emqx can be used as a rebar/mix dependency
-## in other project, so we need to make sure apps/emqx
-## as an Erlang/Elixir app works standalone
-
-# on:
-#   pull_request:
-
-jobs:
-  run_emqx_app_tests:
-    strategy:
-      matrix:
-        builder:
-          - 5.1-3
-        otp:
-          - 25.3.2-1
-        # no need to use more than 1 version of Elixir, since tests
-        # run using only Erlang code.  This is needed just to specify
-        # the base image.
-        elixir:
-          - 1.14.5
-        os:
-          - ubuntu22.04
-        arch:
-          - amd64
-        runs-on:
-          - aws-amd64
-          - ubuntu-22.04
-        use-self-hosted:
-          - ${{ github.repository_owner == 'emqx' }}
-        exclude:
-          - runs-on: ubuntu-22.04
-            use-self-hosted: true
-          - runs-on: aws-amd64
-            use-self-hosted: false
-
-    runs-on: ${{ matrix.runs-on }}
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
-
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: run
-      run: |
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
-        echo "git diff base: $GITHUB_BASE_REF"
-        if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then
-          # base is a commit sha1
-          compare_base="$GITHUB_BASE_REF"
-        else
-          repo="${GITHUB_REPOSITORY}"
-          git remote -v
-          remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')"
-          git fetch "$remote" "$GITHUB_BASE_REF"
-          compare_base="$remote/$GITHUB_BASE_REF"
-        fi
-        changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)"
-        if [ "$changed_files" = '' ]; then
-          echo "nothing changed in apps/emqx, ignored."
-          exit 0
-        fi
-        make ensure-rebar3
-        cp rebar3 apps/emqx/
-        cd apps/emqx
-        ./rebar3 xref
-        ./rebar3 dialyzer
-        ./rebar3 eunit -v
-        ./rebar3 ct --name 'test@127.0.0.1' -v --readable=true
-        ./rebar3 proper -d test/props
-    - uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: logs-${{ matrix.runs-on }}
-        path: apps/emqx/_build/test/logs

.github/workflows/run_fvt_tests.yaml

@@ -4,6 +4,9 @@ concurrency:
   group: fvt-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
+
 # on:
 #   push:
 #     branches:
@@ -15,7 +18,7 @@ concurrency:
 
 jobs:
   prepare:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     # prepare source with any OTP version, no need for a matrix
     container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11
 
@@ -34,7 +37,9 @@ jobs:
           path: source.zip
 
   docker_test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
     needs: prepare
 
     strategy:
@@ -58,15 +63,16 @@ jobs:
         arch:
           - amd64
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
+    - uses: AutoModality/action-clean@v1
     - uses: actions/download-artifact@v3
       with:
         name: source
         path: .
     - name: unzip source code
-      run: unzip -q source.zip
+      run: unzip -o -q source.zip
 
     - name: make docker image
       working-directory: source
@@ -108,7 +114,9 @@ jobs:
         docker exec node1.emqx.io node_dump
 
   helm_test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
     needs: prepare
 
     strategy:
@@ -133,15 +141,16 @@ jobs:
       # - emqx-enterprise # TODO test enterprise
 
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
+    - uses: AutoModality/action-clean@v1
     - uses: actions/download-artifact@v3
       with:
         name: source
         path: .
     - name: unzip source code
-      run: unzip -q source.zip
+      run: unzip -o -q source.zip
 
     - name: make docker image
       working-directory: source

.github/workflows/run_jmeter_tests.yaml

@@ -1,5 +1,7 @@
 name: JMeter integration tests
 
+on:
+  workflow_call:
 # on:
 #   push:
 #     tags:
@@ -14,7 +16,7 @@ jobs:
     outputs:
       version: ${{ steps.build_docker.outputs.version}}
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - name: Cache Jmeter
@@ -59,7 +61,9 @@ jobs:
         path: ./emqx.tar
 
   advanced_feat:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
 
     strategy:
       fail-fast: false
@@ -72,7 +76,7 @@ jobs:
 
     needs: build_emqx_for_jmeter_tests
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3
@@ -151,9 +155,10 @@ jobs:
         path: ./jmeter_logs
 
   pgsql_authn_authz:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     env:
       _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
+      ImageOS: ubuntu22
 
     strategy:
       fail-fast: false
@@ -170,7 +175,7 @@ jobs:
 
     needs: build_emqx_for_jmeter_tests
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3
@@ -265,7 +270,9 @@ jobs:
         path: ./jmeter_logs
 
   mysql_authn_authz:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
 
     strategy:
       fail-fast: false
@@ -279,7 +286,7 @@ jobs:
 
     needs: build_emqx_for_jmeter_tests
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3
@@ -371,7 +378,9 @@ jobs:
         path: ./jmeter_logs
 
   JWT_authn:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
 
     strategy:
       fail-fast: false
@@ -381,7 +390,7 @@ jobs:
 
     needs: build_emqx_for_jmeter_tests
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3
@@ -469,7 +478,9 @@ jobs:
         path: ./jmeter_logs
 
   built_in_database_authn_authz:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      ImageOS: ubuntu22
 
     strategy:
       fail-fast: false
@@ -480,7 +491,7 @@ jobs:
 
     needs: build_emqx_for_jmeter_tests
     steps:
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3
@@ -561,7 +572,7 @@ jobs:
         path: ./jmeter_logs
 
   delete-artifact:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz]
     steps:
     - uses: geekyeggo/delete-artifact@v2

.github/workflows/run_relup_tests.yaml

@@ -4,6 +4,8 @@ concurrency:
   group: relup-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
 # on:
 #   push:
 #     branches:
@@ -14,7 +16,7 @@ concurrency:
 
 jobs:
   relup_test_plan:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
     outputs:
       CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
@@ -58,7 +60,7 @@ jobs:
     needs:
       - relup_test_plan
     if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]'
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     strategy:
       fail-fast: false
       matrix:
@@ -71,7 +73,7 @@ jobs:
         shell: bash
     steps:
       # setup Erlang to run lux
-    - uses: erlef/setup-beam@v1.15.4
+    - uses: erlef/setup-beam@v1.16.0
       with:
         otp-version: 25.3.2
     - uses: actions/checkout@v3

.github/workflows/run_test_cases.yaml

@@ -4,6 +4,19 @@ concurrency:
   group: test-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
+on:
+  workflow_call:
+    inputs:
+      matrix:
+        required: true
+        type: string
+      ct-host:
+        required: true
+        type: string
+      ct-docker:
+        required: true
+        type: string
+
 # on:
 #   push:
 #     branches:
@@ -18,139 +31,38 @@ env:
   IS_CI: "yes"
 
 jobs:
-  build-matrix:
-    runs-on: ubuntu-22.04
-    outputs:
-      prepare: ${{ steps.matrix.outputs.prepare }}
-      host: ${{ steps.matrix.outputs.host }}
-      docker: ${{ steps.matrix.outputs.docker }}
-      runs-on: ${{ steps.runner.outputs.runs-on }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build matrix
-        id: matrix
-        run: |
-          APPS="$(./scripts/find-apps.sh --ci)"
-          MATRIX="$(echo "${APPS}" | jq -c '
-            [
-              (.[] | select(.profile == "emqx") | . + {
-                builder: "5.1-3",
-                otp: "25.3.2-1",
-                elixir: "1.14.5"
-              }),
-              (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.1-3",
-                otp: ["25.3.2-1"][],
-                elixir: "1.14.5"
-              })
-            ]
-          ')"
-          echo "${MATRIX}" | jq
-          MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
-          MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
-          MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
-          echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT
-          echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
-          echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
-      - name: Choose runner host
-        id: runner
-        run: |
-          RUNS_ON="ubuntu-22.04"
-          ${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64"
-          echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
-
-  prepare:
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
-    needs: [build-matrix]
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
-    steps:
-      - uses: AutoModality/action-clean@v1
-      - uses: actions/checkout@v3
-        with:
-          path: source
-      - name: get_all_deps
-        working-directory: source
-        env:
-          PROFILE: ${{ matrix.profile }}
-        run: |
-          make ensure-rebar3
-          # fetch all deps and compile
-          make ${{ matrix.profile }}-compile
-          make test-compile
-          cd ..
-          zip -ryq source.zip source/* source/.[^.]*
-      - uses: actions/upload-artifact@v3
-        with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
-          path: source.zip
-
-  check_examples:
-    needs:
-      - build-matrix
-      - prepare
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
-    steps:
-      - uses: AutoModality/action-clean@v1
-      - uses: actions/download-artifact@v3
-        with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
-          path: .
-      - name: unzip source code
-        run: unzip -o -q source.zip
-      - name: check example config files
-        env:
-          PROFILE: ${{ matrix.profile }}
-        working-directory: source
-        run: ./scripts/test/check-example-configs.sh
-
   static_checks:
-    needs:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      - build-matrix
-      - prepare
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
+        include: ${{ fromJson(inputs.matrix) }}
     container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
+          name: ${{ matrix.profile }}-${{ github.sha }}
-          path: .
+          path: ${{ matrix.profile }}.zip
       - name: unzip source code
-        run: unzip -o -q source.zip
+        run: unzip -o -q ${{ matrix.profile }}.zip
       - uses: actions/cache@v3
         with:
-          path: "source/emqx_dialyzer_${{ matrix.otp }}_plt"
+          path: "emqx_dialyzer_${{ matrix.otp }}_plt"
-          key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }}
+          key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }}
           restore-keys: |
             rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-
       - name: run static checks
         env:
           PROFILE: ${{ matrix.profile }}
-        working-directory: source
         run: make static_checks
 
   eunit_and_proper:
-    needs:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      - build-matrix
-      - prepare
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
+        include: ${{ fromJson(inputs.matrix) }}
 
     defaults:
       run:
@@ -161,16 +73,15 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
+          name: ${{ matrix.profile }}-${{ github.sha }}
-          path: .
+          path: ${{ matrix.profile }}.zip
       - name: unzip source code
-        run: unzip -o -q source.zip
+        run: unzip -o -q ${{ matrix.profile }}.zip
         # produces eunit.coverdata
       - name: eunit
         env:
           PROFILE: ${{ matrix.profile }}
           CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
-        working-directory: source
         run: make eunit
 
         # produces proper.coverdata
@@ -178,23 +89,19 @@ jobs:
         env:
           PROFILE: ${{ matrix.profile }}
           CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
-        working-directory: source
         run: make proper
 
       - uses: actions/upload-artifact@v3
         with:
           name: coverdata
-          path: source/_build/test/cover
+          path: _build/test/cover
 
   ct_docker:
-    needs:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      - build-matrix
-      - prepare
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.docker) }}
+        include: ${{ fromJson(inputs.ct-docker) }}
 
     defaults:
       run:
@@ -204,14 +111,13 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
+          name: ${{ matrix.profile }}-${{ github.sha }}
-          path: .
+          path: ${{ matrix.profile }}.zip
       - name: unzip source code
-        run: unzip -q source.zip
+        run: unzip -o -q ${{ matrix.profile }}.zip
 
         # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
       - name: run common tests
-        working-directory: source
         env:
           DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
           MONGO_TAG: "5"
@@ -229,22 +135,19 @@ jobs:
       - uses: actions/upload-artifact@v3
         with:
           name: coverdata
-          path: source/_build/test/cover
+          path: _build/test/cover
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
-          path: source/_build/test/logs
+          path: _build/test/logs
 
   ct:
-    needs:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      - build-matrix
-      - prepare
-    runs-on: ${{ needs.build-matrix.outputs.runs-on }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.build-matrix.outputs.host) }}
+        include: ${{ fromJson(inputs.ct-host) }}
 
     container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
     defaults:
@@ -255,14 +158,13 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-${{ matrix.profile }}-${{ matrix.otp }}
+          name: ${{ matrix.profile }}-${{ github.sha }}
-          path: .
+          path: ${{ matrix.profile }}.zip
       - name: unzip source code
-        run: unzip -q source.zip
+        run: unzip -o -q ${{ matrix.profile }}.zip
 
         # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
       - name: run common tests
-        working-directory: source
         env:
           PROFILE: ${{ matrix.profile }}
           SUITEGROUP: ${{ matrix.suitegroup }}
@@ -272,58 +174,60 @@ jobs:
       - uses: actions/upload-artifact@v3
         with:
           name: coverdata
-          path: source/_build/test/cover
+          path: _build/test/cover
           if-no-files-found: warn # do not fail if no coverdata found
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
-          path: source/_build/test/logs
+          path: _build/test/logs
 
   make_cover:
     needs:
       - eunit_and_proper
       - ct
       - ct_docker
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
+    strategy:
+      fail-fast: false
+      matrix:
+        profile:
+          - emqx-enterprise
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-emqx-enterprise-25.3.2-1
+          name: ${{ matrix.profile }}-${{ github.sha }}
-          path: .
+          path: ${{ matrix.profile }}.zip
       - name: unzip source code
-        run: unzip -q source.zip
+        run: unzip -o -q ${{ matrix.profile }}.zip
 
       - uses: actions/download-artifact@v3
         name: download coverdata
         with:
           name: coverdata
-          path: source/_build/test/cover
+          path: _build/test/cover
 
       - name: make cover
-        working-directory: source
         env:
           PROFILE: emqx-enterprise
         run: make cover
 
       - name: send to coveralls
-        working-directory: source
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PROFILE: emqx-enterprise
         run: make coveralls
 
       - name: get coveralls logs
-        working-directory: source
         if: failure()
         run: cat rebar3.crashdump
 
   # do this in a separate job
   upload_coverdata:
     needs: make_cover
-    runs-on: ubuntu-22.04
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     steps:
       - name: Coveralls Finished
         env: