ci: test workflow

.github/workflows/build_packages.yaml

@@ -27,6 +27,7 @@ jobs:
     outputs:
       BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
       IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
+      VERSION: ${{ steps.get_profile.outputs.VERSION }}
 
     steps:
       - uses: actions/checkout@v3
@@ -75,6 +76,7 @@ jobs:
               ;;
           esac
           echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT
+          echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT
       - name: get_all_deps
         run: |
           make -C source deps-all
@@ -84,114 +86,9 @@ jobs:
           name: source
           path: source.zip
 
-  windows:
-    runs-on: windows-2019
-    if: startsWith(github.ref_name, 'v')
-    needs: prepare
-    strategy:
-      fail-fast: false
-      matrix:
-        profile: # for now only CE for windows
-          - emqx
-    steps:
-    - uses: actions/download-artifact@v3
-      with:
-        name: source
-        path: .
-    - name: unzip source code
-      run: Expand-Archive -Path source.zip -DestinationPath ./
-    - uses: ilammy/msvc-dev-cmd@v1.12.0
-    - uses: erlef/setup-beam@v1.15.2
-      with:
-        otp-version: 24.3.4.6
-    - name: build
-      env:
-        PYTHON: python
-        DIAGNOSTIC: 1
-      working-directory: source
-      run: |
-        # ensure crypto app (openssl)
-        erl -eval "erlang:display(crypto:info_lib())" -s init stop
-        make ${{ matrix.profile }}-tgz
-    - name: run emqx
-      timeout-minutes: 5
-      working-directory: source
-      run: |
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
-        Start-Sleep -s 5
-        echo "EMQX started"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
-        echo "EMQX stopped"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
-        echo "EMQX installed"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
-        echo "EMQX uninstalled"
-    - uses: actions/upload-artifact@v3
-      if: success()
-      with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/
-    - name: Send notification to Slack
-      uses: slackapi/slack-github-action@v1.23.0
-      if: failure() && github.event_name == 'schedule'
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      with:
-        payload: |
-          {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
-
-  mac:
-    needs: prepare
-    strategy:
-      fail-fast: false
-      matrix:
-        profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-        otp:
-          - 24.3.4.2-1
-        os:
-          - macos-11
-          - macos-12-arm64
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: emqx/self-hosted-cleanup-action@v1.0.3
-    - uses: actions/download-artifact@v3
-      with:
-        name: source
-        path: .
-    - name: unzip source code
-      run: |
-        ln -s . source
-        unzip -o -q source.zip
-        rm source source.zip
-    - uses: ./.github/actions/package-macos
-      with:
-        profile: ${{ matrix.profile }}
-        otp: ${{ matrix.otp }}
-        os: ${{ matrix.os }}
-        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@v3
-      if: success()
-      with:
-        name: ${{ matrix.profile }}
-        path: _packages/${{ matrix.profile }}/
-    - name: Send notification to Slack
-      uses: slackapi/slack-github-action@v1.23.0
-      if: failure() && github.event_name == 'schedule'
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      with:
-        payload: |
-          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
-
   linux:
     needs: prepare
     runs-on: ${{ matrix.build_machine }}
-    container:
-      image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     strategy:
       fail-fast: false
@@ -204,13 +101,13 @@ jobs:
           - amd64
           - arm64
         os:
-          - ubuntu20.04
+          - ["ubuntu20.04", "ubuntu/focal", "deb"]
-          - ubuntu18.04
+          - ["ubuntu18.04", "ubuntu/bionic", "deb"]
-          - debian11
+          - ["debian11", "debian/bullseye", "deb"]
-          - debian10
+          - ["debian10", "debian/buster", "deb"]
-          - el8
+          - ["el8", "el/8", "rpm"]
-          - el7
+          - ["el7", "el/7", "rpm"]
-          - amzn2
+          - ["amzn2", "el/6", "rpm"]
         build_machine:
           - aws-arm64
           - ubuntu-20.04
@@ -227,7 +124,7 @@ jobs:
           - profile: emqx
             otp: 25.1.2-2
             arch: amd64
-            os: ubuntu20.04
+            os: ["ubuntu20.04", "ubuntu/focal", "deb"]
             build_machine: ubuntu-20.04
             builder: 5.0-26
             elixir: 1.13.4
@@ -235,7 +132,7 @@ jobs:
           - profile: emqx
             otp: 25.1.2-2
             arch: amd64
-            os: amzn2
+            os: ["amzn2", "el/6", "rpm"]
             build_machine: ubuntu-20.04
             builder: 5.0-26
             elixir: 1.13.4
@@ -262,16 +159,9 @@ jobs:
         OTP: ${{ matrix.otp }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
+        SYSTEM: ${{ matrix.os[0] }}
       run: |
         set -eu
-        git config --global --add safe.directory "/__w/emqx/emqx"
-        # Align path for CMake caches
-        if [ ! "$PWD" = "/emqx" ]; then
-          ln -s $PWD /emqx
-          cd /emqx
-        fi
-        echo "pwd is $PWD"
         PKGTYPES="tgz pkg"
         IS_ELIXIR="no"
         if [ ${{ matrix.release_with }} == 'elixir' ]; then
@@ -288,63 +178,23 @@ jobs:
             --elixir "${IS_ELIXIR}" \
             --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}"
         done
-    - uses: actions/upload-artifact@v3
-      if: success()
-      with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/
-    - name: Send notification to Slack
-      uses: slackapi/slack-github-action@v1.23.0
-      if: failure() && github.event_name == 'schedule'
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      with:
-        payload: |
-          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
 
-  publish_artifacts:
+    - name: Push to packagecloud.io
-    runs-on: ubuntu-20.04
+      if: success()
-    needs: [prepare, mac, linux]
+      working-directory: source
-    if: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
-    strategy:
-      fail-fast: false
-      matrix:
-        profile:
-          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-    steps:
-    - uses: actions/download-artifact@v3
-      with:
-        name: ${{ matrix.profile }}
-        path: packages/${{ matrix.profile }}
-    - name: install dos2unix
-      run: sudo apt-get update && sudo apt install -y dos2unix
-    - name: get packages
-      run: |
-        set -e -u
-        cd packages/${{ matrix.profile }}
-        # fix the .sha256 file format
-        for var in $(ls | grep emqx | grep -v sha256); do
-          dos2unix $var.sha256
-          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
-        done
-        cd -
-    - uses: aws-actions/configure-aws-credentials@v1-node16
-      with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-    - name: upload to aws s3
       env:
         PROFILE: ${{ matrix.profile }}
+        ARCH: ${{ matrix.arch }}
+        OS: ${{ matrix.os[0] }}
+        OS_CODENAME: ${{ matrix.os[1] }}
+        PKG_EXT: ${{ matrix.os[2] }}
+        VERSION: ${{ needs.prepare.outputs.VERSION }}
+        PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
       run: |
-        set -e -u
+        set -eu
-        if [ $PROFILE = 'emqx' ]; then
+        REPO=$PROFILE
-            s3dir='emqx-ce'
+        if [ $PROFILE = 'emqx-enterprise' ]; then
-        elif [ $PROFILE = 'emqx-enterprise' ]; then
+            REPO='emqx-enterprise5'
-            s3dir='emqx-ee'
-        else
-            echo "unknown profile $PROFILE"
-            exit 1
         fi
-        aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }}
+        docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/_packages/$PROFILE/:/w -w /w / ghcr.io/emqx/package_cloud push id/$REPO/$OS_CODENAME $PROFILE-$VERSION-$OS-$ARCH.$PKG_EXT
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*"
+

.github/workflows/release.yaml

@@ -79,14 +79,3 @@ jobs:
                -X POST \
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
-      - name: update homebrew packages
-        if: steps.profile.outputs.profile == 'emqx' && (github.event_name == 'release' || inputs.publish_release_artefacts)
-        run: |
-          if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
-              curl --silent --show-error \
-                -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-                -H "Accept: application/vnd.github.v3+json" \
-                -X POST \
-                -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
-                "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
-          fi

scripts/buildx.sh

@@ -145,8 +145,13 @@ elif docker info; then
     if [[ "${IS_NATIVE_ARCH}" == 'no' ]]; then
         docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"
     fi
+    cat <<EOF >.gitconfig
+[safe]
+  directory = /emqx
+EOF
     docker run -i --rm \
-        -v "$(pwd)":/emqx \
+        -v "$(pwd):/emqx" \
+        -v "$(pwd)/.gitconfig:/root/.gitconfig" \
         --workdir /emqx \
         --platform="linux/$ARCH" \
         --env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \

scripts/pkg-tests.sh

@@ -46,6 +46,7 @@ export SCRIPTS="${CODE_PATH}/scripts"
 export EMQX_NAME
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
 export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
+export PAHO_TESTS_PATH="${CODE_PATH}/paho_tests"
 
 SYSTEM="$("$SCRIPTS"/get-distro.sh)"
 
@@ -75,8 +76,8 @@ fi
 emqx_prepare(){
     mkdir -p "${PACKAGE_PATH}"
 
-    if [ ! -d "/paho-mqtt-testing" ]; then
+    if [ ! -d "${PAHO_TESTS_PATH}" ]; then
-        git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing
+        git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git "${PAHO_TESTS_PATH}"
     fi
     pip3 install pytest
 }
@@ -113,7 +114,7 @@ emqx_test(){
                 sleep 10
                 IDLE_TIME=$((IDLE_TIME+1))
             done
-            pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
+            pytest -v "${PAHO_TESTS_PATH}"/interoperability/test_client/V5/test_connect.py::test_basic
             if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then
                 cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
                 cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
@@ -218,7 +219,7 @@ EOF
         sleep 10
         IDLE_TIME=$((IDLE_TIME+1))
     done
-    pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
+    pytest -v "${PAHO_TESTS_PATH}"/interoperability/test_client/V5/test_connect.py::test_basic
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     ps -ef | grep -E '\-progname\s.+emqx\s'
     if ! emqx 'stop'; then