From ab0aedd501d1b32d1c2d6e3c08f4dd37745718e6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 20 Jul 2023 17:04:21 +0200 Subject: [PATCH] ci: call more workflows --- .github/workflows/_pr_entrypoint.yaml | 73 ++++++----- .github/workflows/build_docker_for_test.yaml | 90 ++++++++++++++ .github/workflows/build_slim_packages.yaml | 122 +------------------ .github/workflows/run_conf_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 51 ++++---- .github/workflows/spellcheck.yaml | 45 +++++++ 6 files changed, 212 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/build_docker_for_test.yaml create mode 100644 .github/workflows/spellcheck.yaml diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index c056ebd8e..11c9aee39 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -11,9 +11,11 @@ jobs: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - host: ${{ steps.matrix.outputs.host }} - docker: ${{ steps.matrix.outputs.docker }} + ct-matrix: ${{ steps.matrix.outputs.matrix }} + ct-host: ${{ steps.matrix.outputs.host }} + ct-docker: ${{ steps.matrix.outputs.docker }} + version-emqx: ${{ steps.matrix.outputs.version-emqx }} + version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} steps: - uses: actions/checkout@v3 @@ -39,12 +41,14 @@ jobs: ] ')" echo "${MATRIX}" | jq - MATRIX_PREPARED="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" - MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" - MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" - echo "matrix=${MATRIX_PREPARED}" | tee -a $GITHUB_OUTPUT - echo "ct-host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT - echo "ct-docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT + CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT + echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT + echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT + echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT + echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT compile: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} @@ -55,9 +59,6 @@ jobs: profile: - emqx - emqx-enterprise - outputs: - version-emqx: ${{ steps.compile.outputs.version-emqx }} - version-emqx-enterprise: ${{ steps.compile.outputs.version-emqx-enterprise }} steps: - uses: actions/checkout@v3 @@ -71,8 +72,6 @@ jobs: PROFILE: ${{ matrix.profile }} run: | make $PROFILE - ./pkg-vsn.sh $PROFILE > VSN - echo "version-$PROFILE=$(cat VSN)" | tee -a $GITHUB_OUTPUT zip -ryq $PROFILE.zip . - uses: actions/upload-artifact@v3 with: @@ -80,27 +79,45 @@ jobs: path: ${{ matrix.profile }}.zip retention-days: 1 - # run_test_cases: - # needs: [sanity-checks, compile] - # uses: ./.github/workflows/run_test_cases.yaml - # with: - # matrix: ${{ needs.sanity-checks.outputs.matrix }} - # ct-host: ${{ needs.sanity-checks.outputs.ct-host }} - # ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} + run_test_cases: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_test_cases.yaml + with: + ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + ct-host: ${{ needs.sanity-checks.outputs.ct-host }} + ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} - # build_slim_packages: - # needs: [sanity-checks] - # uses: ./.github/workflows/build_slim_packages.yaml + build_slim_packages: + needs: + - sanity-checks + uses: ./.github/workflows/build_slim_packages.yaml - call_conf_tests: + build_docker_for_test: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/build_docker_for_test.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} + + spellcheck: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/spellcheck.yaml + + run_conf_tests: needs: - sanity-checks - compile uses: ./.github/workflows/run_conf_tests.yaml - # check_deps_integrity: - # needs: [sanity-checks] - # uses: ./.github/workflows/check_deps_integrity.yaml + check_deps_integrity: + needs: [sanity-checks] + uses: ./.github/workflows/check_deps_integrity.yaml # elixir_release: # needs: [sanity-checks] diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml new file mode 100644 index 000000000..95a2737bf --- /dev/null +++ b/.github/workflows/build_docker_for_test.yaml @@ -0,0 +1,90 @@ +name: Build docker image for test + +concurrency: + group: docker-test-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + version-emqx: + required: true + type: string + version-emqx-enterprise: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + docker: + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + env: + EMQX_NAME: ${{ matrix.profile[0] }} + EMQX_IMAGE_TAG: emqx/${{ matrix.profile[0] }}:test + EMQX_IMAGE_OLD_VERSION_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[1] }} + PKG_VSN: ${{ matrix.profile[0] == 'emqx' && inputs.version-emqx || inputs.version-emqx-enterprise }} + + strategy: + fail-fast: false + matrix: + profile: + - ["emqx", "5.0.16"] + - ["emqx-enterprise", "5.0.1"] + + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile[0] }} + - name: extract artifact + run: | + unzip -o -q $EMQX_NAME.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: build and export to Docker + run: | + docker build -t emqx/$EMQX_NAME:test --build-arg EMQX_NAME=$EMQX_NAME -f ./deploy/docker/Dockerfile . + - name: smoke test + run: | + CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + docker stop $CID + - name: dashboard tests + working-directory: ./scripts/ui-tests + run: | + set -eu + docker compose up --abort-on-container-exit --exit-code-from selenium + - name: test two nodes cluster with proto_dist=inet_tls in docker + run: | + ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + # cleanup + ./scripts/test/start-two-nodes-in-docker.sh -c + - name: export docker image + run: | + docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: "${{ matrix.profile[0] }}-docker" + path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" + - name: cleanup + if: always() + working-directory: ./scripts/ui-tests + run: | + docker compose rm -fs diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6da92332d..ba581098b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,6 +27,8 @@ on: jobs: linux: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + env: + EMQX_NAME: ${{ matrix.profile[0] }} strategy: fail-fast: false @@ -48,13 +50,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV - echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Work around https://github.com/actions/checkout/issues/766 run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: build and test tgz package if: matrix.profile[3] == 'erlang' run: | @@ -131,118 +130,3 @@ jobs: with: name: windows path: _packages/${{ matrix.profile }}/* - - mac: - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - otp: - - 25.3.2-1 - os: - - macos-11 - - macos-12-arm64 - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v3 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - - uses: ./.github/actions/package-macos - with: - profile: ${{ matrix.profile }} - otp: ${{ matrix.otp }} - os: ${{ matrix.os }} - apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} - apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} - apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} - apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.os }} - path: _packages/**/* - - docker: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - - strategy: - fail-fast: false - matrix: - profile: - - ["emqx", "5.0.16"] - - ["emqx-enterprise", "5.0.1"] - - steps: - - uses: actions/checkout@v3 - - name: prepare - run: | - EMQX_NAME=${{ matrix.profile[0] }} - PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)} - EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test - EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }} - echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV - echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV - echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV - echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 - - name: build and export to Docker - uses: docker/build-push-action@v4 - with: - context: . - file: ./deploy/docker/Dockerfile - load: true - tags: ${{ env.EMQX_IMAGE_TAG }} - build-args: | - EMQX_NAME=${{ env.EMQX_NAME }} - - name: smoke test - run: | - CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - docker stop $CID - - name: dashboard tests - working-directory: ./scripts/ui-tests - run: | - set -eu - docker compose up --abort-on-container-exit --exit-code-from selenium - - name: test two nodes cluster with proto_dist=inet_tls in docker - run: | - ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - # cleanup - ./scripts/test/start-two-nodes-in-docker.sh -c - - name: export docker image - run: | - docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz - - uses: actions/upload-artifact@v3 - with: - name: "${{ matrix.profile[0] }}-docker" - path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz" - - name: cleanup - if: always() - working-directory: ./scripts/ui-tests - run: | - docker compose rm -fs - - spellcheck: - needs: linux - strategy: - matrix: - profile: - - emqx - - emqx-enterprise - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - steps: - - uses: actions/download-artifact@v3 - name: Download schema dump - with: - name: "${{ matrix.profile }}_schema_dump" - path: /tmp/ - - name: Run spellcheck - run: | - bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index a96f64229..cc77162ee 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -47,7 +47,7 @@ jobs: env: PROFILE: ${{ matrix.profile }} run: ./scripts/conf-test/run.sh - - name: print_erlang_log + - name: print erlang log if: failure() run: | cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.* diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index ce5fd28ae..e0fc75ab6 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -7,7 +7,7 @@ concurrency: on: workflow_call: inputs: - matrix: + ct-matrix: required: true type: string ct-host: @@ -36,16 +36,17 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJson(inputs.matrix) }} + include: ${{ fromJson(inputs.ct-matrix) }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: ${{ matrix.profile }}-${{ github.sha }} - path: ${{ matrix.profile }}.zip - - name: unzip source code - run: unzip -o -q ${{ matrix.profile }}.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/cache@v3 with: path: "emqx_dialyzer_${{ matrix.otp }}_plt" @@ -62,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJson(inputs.matrix) }} + include: ${{ fromJson(inputs.ct-matrix) }} defaults: run: @@ -73,10 +74,11 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: ${{ matrix.profile }}-${{ github.sha }} - path: ${{ matrix.profile }}.zip - - name: unzip source code - run: unzip -o -q ${{ matrix.profile }}.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces eunit.coverdata - name: eunit env: @@ -111,10 +113,11 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: ${{ matrix.profile }}-${{ github.sha }} - path: ${{ matrix.profile }}.zip - - name: unzip source code - run: unzip -o -q ${{ matrix.profile }}.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests @@ -158,10 +161,11 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: ${{ matrix.profile }}-${{ github.sha }} - path: ${{ matrix.profile }}.zip - - name: unzip source code - run: unzip -o -q ${{ matrix.profile }}.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests @@ -198,10 +202,11 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: ${{ matrix.profile }}-${{ github.sha }} - path: ${{ matrix.profile }}.zip - - name: unzip source code - run: unzip -o -q ${{ matrix.profile }}.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/download-artifact@v3 name: download coverdata diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml new file mode 100644 index 000000000..e32a5b2c6 --- /dev/null +++ b/.github/workflows/spellcheck.yaml @@ -0,0 +1,45 @@ +name: Build slim packages + +concurrency: + group: spellcheck-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + spellcheck: + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Run spellcheck + run: | + ./scripts/spellcheck/spellcheck.sh _build/docgen/${{ matrix.profile }}/schema-en.json