From cca4ea1d13291745ff2d818f7af7b9c3f252488d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 24 Jan 2023 13:54:27 +0100 Subject: [PATCH] ci: test workflow --- .github/workflows/build_packages.yaml | 178 +------------------------- scripts/buildx.sh | 7 +- scripts/pkg-tests.sh | 9 +- 3 files changed, 14 insertions(+), 180 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 558c4c666..f9872dc59 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -86,114 +86,9 @@ jobs: name: source path: source.zip - windows: - runs-on: windows-2019 - if: startsWith(github.ref_name, 'v') - needs: prepare - strategy: - fail-fast: false - matrix: - profile: # for now only CE for windows - - emqx - steps: - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: Expand-Archive -Path source.zip -DestinationPath ./ - - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.2 - with: - otp-version: 24.3.4.6 - - name: build - env: - PYTHON: python - DIAGNOSTIC: 1 - working-directory: source - run: | - # ensure crypto app (openssl) - erl -eval "erlang:display(crypto:info_lib())" -s init stop - make ${{ matrix.profile }}-tgz - - name: run emqx - timeout-minutes: 5 - working-directory: source - run: | - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start - Start-Sleep -s 5 - echo "EMQX started" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop - echo "EMQX stopped" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install - echo "EMQX installed" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - echo "EMQX uninstalled" - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - - mac: - needs: prepare - strategy: - fail-fast: false - matrix: - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - otp: - - 24.3.4.2-1 - os: - - macos-11 - - macos-12-arm64 - runs-on: ${{ matrix.os }} - steps: - - uses: emqx/self-hosted-cleanup-action@v1.0.3 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: | - ln -s . source - unzip -o -q source.zip - rm source source.zip - - uses: ./.github/actions/package-macos - with: - profile: ${{ matrix.profile }} - otp: ${{ matrix.otp }} - os: ${{ matrix.os }} - apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} - apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} - apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} - apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: _packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - linux: needs: prepare runs-on: ${{ matrix.build_machine }} - container: - image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false @@ -267,13 +162,6 @@ jobs: SYSTEM: ${{ matrix.os[0] }} run: | set -eu - git config --global --add safe.directory "/__w/emqx/emqx" - # Align path for CMake caches - if [ ! "$PWD" = "/emqx" ]; then - ln -s $PWD /emqx - cd /emqx - fi - echo "pwd is $PWD" PKGTYPES="tgz pkg" IS_ELIXIR="no" if [ ${{ matrix.release_with }} == 'elixir' ]; then @@ -291,13 +179,8 @@ jobs: --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" done - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - name: Push to packagecloud.io - if: success() && ${{ needs.prepare.outputs.IS_EXACT_TAG }} + if: success() working-directory: source env: PROFILE: ${{ matrix.profile }} @@ -312,61 +195,6 @@ jobs: REPO=$PROFILE if [ $PROFILE = 'emqx-enterprise' ]; then REPO='emqx-enterprise5' - docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN \ - ghcr.io/emqx/package_cloud push emqx/$REPO/$OS_CODENAME \ - _packages/$PROFILE/$PROFILE-$VERSION-$OS-$ARCH.$PKG_EXT - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - - publish_artifacts: - runs-on: ubuntu-20.04 - needs: [prepare, mac, linux] - if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} - strategy: - fail-fast: false - matrix: - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.profile }} - path: packages/${{ matrix.profile }} - - name: install dos2unix - run: sudo apt-get update && sudo apt install -y dos2unix - - name: get packages - run: | - set -e -u - cd packages/${{ matrix.profile }} - # fix the .sha256 file format - for var in $(ls | grep emqx | grep -v sha256); do - dos2unix $var.sha256 - echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 - done - cd - - - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: upload to aws s3 - env: - PROFILE: ${{ matrix.profile }} - run: | - set -e -u - if [ $PROFILE = 'emqx' ]; then - s3dir='emqx-ce' - elif [ $PROFILE = 'emqx-enterprise' ]; then - s3dir='emqx-ee' - else - echo "unknown profile $PROFILE" - exit 1 fi - aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" + docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/_packages/$PROFILE/:/w -w /w / ghcr.io/emqx/package_cloud push id/$REPO/$OS_CODENAME $PROFILE-$VERSION-$OS-$ARCH.$PKG_EXT + diff --git a/scripts/buildx.sh b/scripts/buildx.sh index ca7f812f6..bf9cf7408 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -145,8 +145,13 @@ elif docker info; then if [[ "${IS_NATIVE_ARCH}" == 'no' ]]; then docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" fi + cat <.gitconfig +[safe] + directory = /emqx +EOF docker run -i --rm \ - -v "$(pwd)":/emqx \ + -v "$(pwd):/emqx" \ + -v "$(pwd)/.gitconfig:/root/.gitconfig" \ --workdir /emqx \ --platform="linux/$ARCH" \ --env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \ diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index 768a152c0..137152436 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -46,6 +46,7 @@ export SCRIPTS="${CODE_PATH}/scripts" export EMQX_NAME export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" +export PAHO_TESTS_PATH="${CODE_PATH}/paho_tests" SYSTEM="$("$SCRIPTS"/get-distro.sh)" @@ -75,8 +76,8 @@ fi emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" - if [ ! -d "/paho-mqtt-testing" ]; then - git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing + if [ ! -d "${PAHO_TESTS_PATH}" ]; then + git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git "${PAHO_TESTS_PATH}" fi pip3 install pytest } @@ -113,7 +114,7 @@ emqx_test(){ sleep 10 IDLE_TIME=$((IDLE_TIME+1)) done - pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic + pytest -v "${PAHO_TESTS_PATH}"/interoperability/test_client/V5/test_connect.py::test_basic if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true @@ -218,7 +219,7 @@ EOF sleep 10 IDLE_TIME=$((IDLE_TIME+1)) done - pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic + pytest -v "${PAHO_TESTS_PATH}"/interoperability/test_client/V5/test_connect.py::test_basic # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions ps -ef | grep -E '\-progname\s.+emqx\s' if ! emqx 'stop'; then