ci: call more workflows

.github/workflows/_pr_entrypoint.yaml

@@ -11,9 +11,11 @@ jobs:
     runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
     container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
     outputs:
-      matrix: ${{ steps.matrix.outputs.matrix }}
+      ct-matrix: ${{ steps.matrix.outputs.matrix }}
-      host: ${{ steps.matrix.outputs.host }}
+      ct-host: ${{ steps.matrix.outputs.host }}
-      docker: ${{ steps.matrix.outputs.docker }}
+      ct-docker: ${{ steps.matrix.outputs.docker }}
+      version-emqx: ${{ steps.matrix.outputs.version-emqx }}
+      version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
 
     steps:
       - uses: actions/checkout@v3
@@ -39,12 +41,14 @@ jobs:
             ]
           ')"
           echo "${MATRIX}" | jq
-          MATRIX_PREPARED="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
+          CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
-          MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
+          CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
-          MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
+          CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
-          echo "matrix=${MATRIX_PREPARED}" | tee -a $GITHUB_OUTPUT
+          echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
-          echo "ct-host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
+          echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT
-          echo "ct-docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
+          echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
+          echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT
+          echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
 
   compile:
     runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
@@ -55,9 +59,6 @@ jobs:
         profile:
           - emqx
           - emqx-enterprise
-    outputs:
-      version-emqx: ${{ steps.compile.outputs.version-emqx }}
-      version-emqx-enterprise: ${{ steps.compile.outputs.version-emqx-enterprise }}
 
     steps:
       - uses: actions/checkout@v3
@@ -71,8 +72,6 @@ jobs:
           PROFILE: ${{ matrix.profile }}
         run: |
           make $PROFILE
-          ./pkg-vsn.sh $PROFILE > VSN
-          echo "version-$PROFILE=$(cat VSN)" | tee -a $GITHUB_OUTPUT
           zip -ryq $PROFILE.zip .
       - uses: actions/upload-artifact@v3
         with:
@@ -80,27 +79,45 @@ jobs:
           path: ${{ matrix.profile }}.zip
           retention-days: 1
 
-  # run_test_cases:
+  run_test_cases:
-  #   needs: [sanity-checks, compile]
+    needs:
-  #   uses: ./.github/workflows/run_test_cases.yaml
+      - sanity-checks
-  #   with:
+      - compile
-  #     matrix: ${{ needs.sanity-checks.outputs.matrix }}
+    uses: ./.github/workflows/run_test_cases.yaml
-  #     ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
+    with:
-  #     ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
+      ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
+      ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
+      ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
 
-  # build_slim_packages:
+  build_slim_packages:
-  #   needs: [sanity-checks]
+    needs:
-  #   uses: ./.github/workflows/build_slim_packages.yaml
+      - sanity-checks
+    uses: ./.github/workflows/build_slim_packages.yaml
 
-  call_conf_tests:
+  build_docker_for_test:
+    needs:
+      - sanity-checks
+      - compile
+    uses: ./.github/workflows/build_docker_for_test.yaml
+    with:
+      version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
+      version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
+
+  spellcheck:
+    needs:
+      - sanity-checks
+      - compile
+    uses: ./.github/workflows/spellcheck.yaml
+
+  run_conf_tests:
     needs:
       - sanity-checks
       - compile
     uses: ./.github/workflows/run_conf_tests.yaml
 
-  # check_deps_integrity:
+  check_deps_integrity:
-  #   needs: [sanity-checks]
+    needs: [sanity-checks]
-  #   uses: ./.github/workflows/check_deps_integrity.yaml
+    uses: ./.github/workflows/check_deps_integrity.yaml
 
   # elixir_release:
   #   needs: [sanity-checks]

.github/workflows/build_docker_for_test.yaml

@@ -0,0 +1,90 @@
+name: Build docker image for test
+
+concurrency:
+  group: docker-test-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_call:
+    inputs:
+      version-emqx:
+        required: true
+        type: string
+      version-emqx-enterprise:
+        required: true
+        type: string
+# on:
+#   push:
+#     branches:
+#       - master
+#       - release-51
+#   pull_request:
+#     # GitHub pull_request action is by default triggered when
+#     # opened reopened or synchronize,
+#     # we add labeled and unlabeled to the list because
+#     # the mac job dpends on the PR having a 'Mac' label
+#     types:
+#       - labeled
+#       - unlabeled
+#       - opened
+#       - reopened
+#       - synchronize
+#   workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      EMQX_NAME: ${{ matrix.profile[0] }}
+      EMQX_IMAGE_TAG: emqx/${{ matrix.profile[0] }}:test
+      EMQX_IMAGE_OLD_VERSION_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[1] }}
+      PKG_VSN: ${{ matrix.profile[0] == 'emqx' && inputs.version-emqx || inputs.version-emqx-enterprise }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        profile:
+          - ["emqx", "5.0.16"]
+          - ["emqx-enterprise", "5.0.1"]
+
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.profile[0] }}
+      - name: extract artifact
+        run: |
+          unzip -o -q $EMQX_NAME.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: build and export to Docker
+        run: |
+          docker build -t emqx/$EMQX_NAME:test --build-arg EMQX_NAME=$EMQX_NAME -f ./deploy/docker/Dockerfile .
+      - name: smoke test
+        run: |
+          CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
+          HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
+          ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
+          docker stop $CID
+      - name: dashboard tests
+        working-directory: ./scripts/ui-tests
+        run: |
+          set -eu
+          docker compose up --abort-on-container-exit --exit-code-from selenium
+      - name: test two nodes cluster with proto_dist=inet_tls in docker
+        run: |
+          ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
+          HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
+          ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
+          # cleanup
+          ./scripts/test/start-two-nodes-in-docker.sh -c
+      - name: export docker image
+        run: |
+          docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
+      - uses: actions/upload-artifact@v3
+        with:
+          name: "${{ matrix.profile[0] }}-docker"
+          path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"
+      - name: cleanup
+        if: always()
+        working-directory: ./scripts/ui-tests
+        run: |
+          docker compose rm -fs

.github/workflows/build_slim_packages.yaml

@@ -27,6 +27,8 @@ on:
 jobs:
   linux:
     runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    env:
+      EMQX_NAME: ${{ matrix.profile[0] }}
 
     strategy:
       fail-fast: false
@@ -48,13 +50,10 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: prepare
-      run: |
-        echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV
-        echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
     - name: Work around https://github.com/actions/checkout/issues/766
       run: |
         git config --global --add safe.directory "$GITHUB_WORKSPACE"
+        echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
     - name: build and test tgz package
       if: matrix.profile[3] == 'erlang'
       run: |
@@ -131,118 +130,3 @@ jobs:
       with:
         name: windows
         path: _packages/${{ matrix.profile }}/*
-
-  mac:
-    strategy:
-      fail-fast: false
-      matrix:
-        profile:
-        - emqx
-        - emqx-enterprise
-        otp:
-        - 25.3.2-1
-        os:
-        - macos-11
-        - macos-12-arm64
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: prepare
-      run: |
-        echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
-    - uses: ./.github/actions/package-macos
-      with:
-        profile: ${{ matrix.profile }}
-        otp: ${{ matrix.otp }}
-        os: ${{ matrix.os }}
-        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.os }}
-        path: _packages/**/*
-
-  docker:
-    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        profile:
-          - ["emqx", "5.0.16"]
-          - ["emqx-enterprise", "5.0.1"]
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: prepare
-      run: |
-        EMQX_NAME=${{ matrix.profile[0] }}
-        PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}
-        EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test
-        EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }}
-        echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV
-        echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV
-        echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
-        echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV
-    - uses: docker/setup-buildx-action@v2
-    - name: build and export to Docker
-      uses: docker/build-push-action@v4
-      with:
-        context: .
-        file: ./deploy/docker/Dockerfile
-        load: true
-        tags: ${{ env.EMQX_IMAGE_TAG }}
-        build-args: |
-          EMQX_NAME=${{ env.EMQX_NAME }}
-    - name: smoke test
-      run: |
-        CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
-        HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
-        ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
-        docker stop $CID
-    - name: dashboard tests
-      working-directory: ./scripts/ui-tests
-      run: |
-        set -eu
-        docker compose up --abort-on-container-exit --exit-code-from selenium
-    - name: test two nodes cluster with proto_dist=inet_tls in docker
-      run: |
-        ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
-        HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
-        ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
-        # cleanup
-        ./scripts/test/start-two-nodes-in-docker.sh -c
-    - name: export docker image
-      run: |
-        docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
-    - uses: actions/upload-artifact@v3
-      with:
-        name: "${{ matrix.profile[0] }}-docker"
-        path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz"
-    - name: cleanup
-      if: always()
-      working-directory: ./scripts/ui-tests
-      run: |
-        docker compose rm -fs
-
-  spellcheck:
-    needs: linux
-    strategy:
-      matrix:
-        profile:
-        - emqx
-        - emqx-enterprise
-    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-    steps:
-    - uses: actions/download-artifact@v3
-      name: Download schema dump
-      with:
-        name: "${{ matrix.profile }}_schema_dump"
-        path: /tmp/
-    - name: Run spellcheck
-      run: |
-        bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json

.github/workflows/run_conf_tests.yaml

@@ -47,7 +47,7 @@ jobs:
         env:
           PROFILE: ${{ matrix.profile }}
         run: ./scripts/conf-test/run.sh
-      - name: print_erlang_log
+      - name: print erlang log
         if: failure()
         run: |
           cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*

.github/workflows/run_test_cases.yaml

@@ -7,7 +7,7 @@ concurrency:
 on:
   workflow_call:
     inputs:
-      matrix:
+      ct-matrix:
         required: true
         type: string
       ct-host:
@@ -36,16 +36,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(inputs.matrix) }}
+        include: ${{ fromJson(inputs.ct-matrix) }}
     container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.profile }}-${{ github.sha }}
+          name: ${{ matrix.profile }}
-          path: ${{ matrix.profile }}.zip
+      - name: extract artifact
-      - name: unzip source code
+        run: |
-        run: unzip -o -q ${{ matrix.profile }}.zip
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - uses: actions/cache@v3
         with:
           path: "emqx_dialyzer_${{ matrix.otp }}_plt"
@@ -62,7 +63,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(inputs.matrix) }}
+        include: ${{ fromJson(inputs.ct-matrix) }}
 
     defaults:
       run:
@@ -73,10 +74,11 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.profile }}-${{ github.sha }}
+          name: ${{ matrix.profile }}
-          path: ${{ matrix.profile }}.zip
+      - name: extract artifact
-      - name: unzip source code
+        run: |
-        run: unzip -o -q ${{ matrix.profile }}.zip
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
         # produces eunit.coverdata
       - name: eunit
         env:
@@ -111,10 +113,11 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.profile }}-${{ github.sha }}
+          name: ${{ matrix.profile }}
-          path: ${{ matrix.profile }}.zip
+      - name: extract artifact
-      - name: unzip source code
+        run: |
-        run: unzip -o -q ${{ matrix.profile }}.zip
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
         # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
       - name: run common tests
@@ -158,10 +161,11 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.profile }}-${{ github.sha }}
+          name: ${{ matrix.profile }}
-          path: ${{ matrix.profile }}.zip
+      - name: extract artifact
-      - name: unzip source code
+        run: |
-        run: unzip -o -q ${{ matrix.profile }}.zip
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
         # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
       - name: run common tests
@@ -198,10 +202,11 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.profile }}-${{ github.sha }}
+          name: ${{ matrix.profile }}
-          path: ${{ matrix.profile }}.zip
+      - name: extract artifact
-      - name: unzip source code
+        run: |
-        run: unzip -o -q ${{ matrix.profile }}.zip
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
       - uses: actions/download-artifact@v3
         name: download coverdata

.github/workflows/spellcheck.yaml

@@ -0,0 +1,45 @@
+name: Build slim packages
+
+concurrency:
+  group: spellcheck-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_call:
+# on:
+#   push:
+#     branches:
+#       - master
+#       - release-51
+#   pull_request:
+#     # GitHub pull_request action is by default triggered when
+#     # opened reopened or synchronize,
+#     # we add labeled and unlabeled to the list because
+#     # the mac job dpends on the PR having a 'Mac' label
+#     types:
+#       - labeled
+#       - unlabeled
+#       - opened
+#       - reopened
+#       - synchronize
+#   workflow_dispatch:
+
+jobs:
+  spellcheck:
+    strategy:
+      matrix:
+        profile:
+        - emqx
+        - emqx-enterprise
+    runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.profile }}
+      - name: extract artifact
+        run: |
+          unzip -o -q ${{ matrix.profile }}.zip
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Run spellcheck
+        run: |
+          ./scripts/spellcheck/spellcheck.sh _build/docgen/${{ matrix.profile }}/schema-en.json