From 54aed80e597f431cdb519c7b3f7dfa28c6ea2a2e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 2 Aug 2023 14:13:03 +0200 Subject: [PATCH] ci: move the rest of workflows to _pr_entrypoint.yaml --- .github/workflows/_pr_entrypoint.yaml | 65 +++++--- .github/workflows/build_docker_for_test.yaml | 31 ++-- .github/workflows/elixir_release.yaml | 14 +- .github/workflows/run_docker_tests.yaml | 106 +++++++++++++ ...run_fvt_tests.yaml => run_helm_tests.yaml} | 148 +++--------------- .github/workflows/run_jmeter_tests.yaml | 108 +++++-------- .github/workflows/run_relup_tests.yaml | 33 +--- build | 14 +- 8 files changed, 242 insertions(+), 277 deletions(-) create mode 100644 .github/workflows/run_docker_tests.yaml rename .github/workflows/{run_fvt_tests.yaml => run_helm_tests.yaml} (55%) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 638f2e9ea..744261c7f 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -80,15 +80,15 @@ jobs: path: ${{ matrix.profile }}.zip retention-days: 1 - run_test_cases: - needs: - - sanity-checks - - compile - uses: ./.github/workflows/run_test_cases.yaml - with: - ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} - ct-host: ${{ needs.sanity-checks.outputs.ct-host }} - ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} + # run_test_cases: + # needs: + # - sanity-checks + # - compile + # uses: ./.github/workflows/run_test_cases.yaml + # with: + # ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + # ct-host: ${{ needs.sanity-checks.outputs.ct-host }} + # ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} static_checks: needs: @@ -123,21 +123,42 @@ jobs: uses: ./.github/workflows/run_conf_tests.yaml check_deps_integrity: - needs: [sanity-checks] + needs: + - sanity-checks uses: ./.github/workflows/check_deps_integrity.yaml - # elixir_release: - # needs: [sanity-checks] - # uses: ./.github/workflows/elixir_release.yaml + elixir_release: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/elixir_release.yaml - # run_relup_tests: - # needs: [sanity-checks] - # uses: ./.github/workflows/run_relup_tests.yaml + run_relup_tests: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_relup_tests.yaml - # run_jmeter_tests: - # needs: [sanity-checks] - # uses: ./.github/workflows/run_jmeter_tests.yaml + run_jmeter_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_jmeter_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} - # run_fvt_tests: - # needs: [sanity-checks] - # uses: ./.github/workflows/run_fvt_tests.yaml + run_docker_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_docker_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + + run_helm_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_helm_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index fa717825c..8aeea7d61 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -1,7 +1,7 @@ name: Build docker image for test concurrency: - group: docker-test-${{ github.event_name }}-${{ github.sha }} + group: docker-test-build-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: @@ -36,48 +36,43 @@ jobs: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} env: EMQX_NAME: ${{ matrix.profile[0] }} - EMQX_IMAGE_TAG: emqx/${{ matrix.profile[0] }}:test - EMQX_IMAGE_OLD_VERSION_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[1] }} PKG_VSN: ${{ matrix.profile[0] == 'emqx' && inputs.version-emqx || inputs.version-emqx-enterprise }} + EMQX_IMAGE_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[0] == 'emqx' && inputs.version-emqx || inputs.version-emqx-enterprise }} + EMQX_IMAGE_OLD_VERSION_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[1] }} strategy: fail-fast: false matrix: profile: - ["emqx", "5.0.16"] + - ["emqx-elixir", "5.0.16"] - ["emqx-enterprise", "5.0.1"] steps: - uses: actions/checkout@v3 - name: build and export to Docker + id: build run: | - docker build -t $EMQX_IMAGE_TAG --build-arg EMQX_NAME=$EMQX_NAME -f ./deploy/docker/Dockerfile . + make ${{ matrix.profile[0] }}-docker + echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_OUTPUT - name: smoke test + env: + EMQX_IMAGE_TAG: ${{ steps.build.outputs.EMQX_IMAGE_TAG }} run: | CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT docker stop $CID - - name: dashboard tests - working-directory: ./scripts/ui-tests - run: | - set -eu - docker compose up --abort-on-container-exit --exit-code-from selenium - - name: test two nodes cluster with proto_dist=inet_tls in docker - run: | - ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - # cleanup - ./scripts/test/start-two-nodes-in-docker.sh -c - name: export docker image + env: + EMQX_IMAGE_TAG: ${{ steps.build.outputs.EMQX_IMAGE_TAG }} run: | docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz - uses: actions/upload-artifact@v3 with: - name: "${{ matrix.profile[0] }}-docker" + name: "${{ env.EMQX_NAME }}-docker" path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" - retention-days: 7 + retention-days: 3 - name: cleanup if: always() working-directory: ./scripts/ui-tests diff --git a/.github/workflows/elixir_release.yaml b/.github/workflows/elixir_release.yaml index 58edac176..43b2c5db0 100644 --- a/.github/workflows/elixir_release.yaml +++ b/.github/workflows/elixir_release.yaml @@ -21,12 +21,12 @@ jobs: - emqx-enterprise container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 steps: - - name: Checkout - uses: actions/checkout@v3 - - name: install tools - run: apt update && apt install netcat-openbsd - - name: Work around https://github.com/actions/checkout/issues/766 + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact run: | + unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: elixir release run: make ${{ matrix.profile }}-elixir @@ -36,8 +36,10 @@ jobs: bin/emqx start - name: check if started run: | + set -e sleep 10 - nc -zv localhost 1883 + timeout 1 bash -c '> $GITHUB_ENV + - name: test two nodes cluster with proto_dist=inet_tls in docker + run: | + ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + # cleanup + ./scripts/test/start-two-nodes-in-docker.sh -c + - name: dashboard tests + working-directory: ./scripts/ui-tests + run: | + set -eu + docker compose up --abort-on-container-exit --exit-code-from selenium + + pytest: + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + env: + DB_BACKEND: ${{ matrix.cluster_db_backend }} + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + - emqx-elixir + cluster_db_backend: + - mnesia + - rlog + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }}-docker + path: /tmp + - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} + run: | + EMQX_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - name: run emqx + timeout-minutes: 5 + run: | + ./.ci/docker-compose-file/scripts/run-emqx.sh $EMQX_IMAGE_TAG $DB_BACKEND + - name: make paho tests + run: | + if ! docker exec -i python /scripts/pytest.sh "$DB_BACKEND"; then + echo "DUMP_CONTAINER_LOGS_BGN" + echo "============== haproxy ==============" + docker logs haproxy + echo "============== node1 ==============" + docker logs node1.emqx.io + echo "============== node2 ==============" + docker logs node2.emqx.io + echo "DUMP_CONTAINER_LOGS_END" + exit 1 + fi + # simple smoke test for node_dump + - name: test node_dump + run: | + docker exec node1.emqx.io node_dump diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_helm_tests.yaml similarity index 55% rename from .github/workflows/run_fvt_tests.yaml rename to .github/workflows/run_helm_tests.yaml index 63de7fecf..24a4d6957 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -1,11 +1,15 @@ -name: Functional Verification Tests +name: Helm tests concurrency: - group: fvt-${{ github.event_name }}-${{ github.ref }} + group: helm-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: workflow_call: + inputs: + version-emqx: + required: true + type: string # on: # push: @@ -17,107 +21,8 @@ on: # pull_request: jobs: - prepare: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11 - - steps: - - uses: actions/checkout@v3 - with: - path: source - fetch-depth: 0 - - name: get deps - run: | - make -C source deps-all - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip - - docker_test: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - env: - ImageOS: ubuntu22 - needs: prepare - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - - emqx-elixir - cluster_db_backend: - - mnesia - - rlog - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -o -q source.zip - - - name: make docker image - working-directory: source - env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} - run: | - make ${{ matrix.profile }}-docker - - name: run emqx - timeout-minutes: 5 - working-directory: source - run: | - set -x - if [[ "${{ matrix.profile }}" = *-elixir ]] - then - export IS_ELIXIR=yes - PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g") - IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir - else - IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) - fi - ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - - name: make paho tests - run: | - if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then - echo "DUMP_CONTAINER_LOGS_BGN" - echo "============== haproxy ==============" - docker logs haproxy - echo "============== node1 ==============" - docker logs node1.emqx.io - echo "============== node2 ==============" - docker logs node2.emqx.io - echo "DUMP_CONTAINER_LOGS_END" - exit 1 - fi - # simple smoke test for node_dump - - name: test node_dump - run: | - docker exec node1.emqx.io node_dump - helm_test: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - env: - ImageOS: ubuntu22 - needs: prepare strategy: fail-fast: false @@ -128,44 +33,27 @@ jobs: profile: - emqx - emqx-enterprise - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - # - emqx-enterprise # TODO test enterprise steps: - - uses: erlef/setup-beam@v1.16.0 + - uses: actions/checkout@v3 with: - otp-version: 25.3.2 - - uses: AutoModality/action-clean@v1 + path: source - uses: actions/download-artifact@v3 with: - name: source - path: . - - name: unzip source code - run: unzip -o -q source.zip - - - name: make docker image - working-directory: source + name: "${{ matrix.profile }}-docker" + path: /tmp + - name: load docker image env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} + PKG_VSN: ${{ inputs.version-emqx }} run: | - make ${{ matrix.profile }}-docker + EMQX_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "EMQX_TAG=$EMQX_TAG" >> $GITHUB_ENV echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - run: minikube start - run: minikube image load $TARGET:$EMQX_TAG - - name: run emqx on chart - working-directory: source + - name: run emqx on chart (k8s) if: matrix.discovery == 'k8s' + working-directory: source run: | helm install ${{ matrix.profile }} \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ @@ -182,9 +70,9 @@ jobs: --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ deploy/charts/${{ matrix.profile }} \ --debug - - name: run emqx on chart - working-directory: source + - name: run emqx on chart (dns) if: matrix.discovery == 'dns' + working-directory: source run: | helm install ${{ matrix.profile }} \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 2a4168d2f..200f89591 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -2,6 +2,10 @@ name: JMeter integration tests on: workflow_call: + inputs: + version-emqx: + required: true + type: string # on: # push: # tags: @@ -11,14 +15,9 @@ on: # - "master" jobs: - build_emqx_for_jmeter_tests: + jmeter_artifact: runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.build_docker.outputs.version}} steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - name: Cache Jmeter id: cache-jmeter uses: actions/cache@v3 @@ -44,21 +43,6 @@ jobs: with: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz - - uses: actions/checkout@v3 - - name: zip emqx docker image - id: build_docker - if: endsWith(github.repository, 'emqx') - run: | - ## TODO: make profile a matrix dimension - PROFILE='emqx' - make "${PROFILE}-docker" - VSN="$(./pkg-vsn.sh $PROFILE)" - echo "version=${VSN}" >> $GITHUB_OUTPUT - docker save -o emqx.tar emqx/emqx:${VSN} - - uses: actions/upload-artifact@v3 - with: - name: emqx.tar - path: ./emqx.tar advanced_feat: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} @@ -74,23 +58,20 @@ jobs: - mqtt_topic_rewrite # - mqtt_retainer - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: emqx.tar + name: emqx-docker path: /tmp - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} run: | - docker load < /tmp/emqx.tar + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | docker-compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ @@ -157,7 +138,6 @@ jobs: pgsql_authn_authz: runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} ImageOS: ubuntu22 strategy: @@ -173,19 +153,18 @@ jobs: - pgsql_authn - pgsql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: emqx.tar + name: emqx-docker path: /tmp - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} run: | - docker load < /tmp/emqx.tar + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV - name: docker compose up timeout-minutes: 5 env: @@ -284,23 +263,21 @@ jobs: - mysql_authn - mysql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: emqx.tar + name: emqx-docker path: /tmp - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} run: | - docker load < /tmp/emqx.tar + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV - name: docker compose up timeout-minutes: 5 env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} PGSQL_TAG: ${{ matrix.mysql_tag }} run: | docker-compose \ @@ -388,23 +365,20 @@ jobs: scripts_type: - jwt_authn - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: emqx.tar + name: emqx-docker path: /tmp - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} run: | - docker load < /tmp/emqx.tar + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | docker-compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ @@ -489,24 +463,20 @@ jobs: - built_in_database_authn - built_in_database_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: emqx.tar + name: emqx-docker path: /tmp - name: load docker image + env: + PKG_VSN: ${{ inputs.version-emqx }} run: | - docker load < /tmp/emqx.tar + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} - PGSQL_TAG: ${{ matrix.mysql_tag }} run: | docker-compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ @@ -570,11 +540,3 @@ jobs: with: name: jmeter_logs path: ./jmeter_logs - - delete-artifact: - runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} - needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz] - steps: - - uses: geekyeggo/delete-artifact@v2 - with: - name: emqx.tar diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 699ea76d8..cf7edd08d 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -25,16 +25,17 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@v3 - name: Checkout + - uses: actions/download-artifact@v3 with: - path: emqx - fetch-depth: 0 + name: emqx-enterprise + - name: extract artifact + run: | + unzip -o -q emqx-enterprise.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Find versions id: find-versions run: | set -x - cd emqx ee_vsn="$(./pkg-vsn.sh enterprise)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') @@ -42,8 +43,6 @@ jobs: echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT - name: build emqx run: | - set -x - cd emqx export PROFILE='emqx-enterprise' make emqx-enterprise-tgz - uses: actions/upload-artifact@v3 @@ -65,6 +64,7 @@ jobs: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.OLD_VERSIONS) }} + container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" env: OLD_VSN: "${{ matrix.old_vsn }}" CUR_EE_VSN: "${{ needs.relup_test_plan.outputs.CUR_EE_VSN }}" @@ -72,23 +72,6 @@ jobs: run: shell: bash steps: - # setup Erlang to run lux - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: 25.3.2 - - uses: actions/checkout@v3 - with: - repository: hawk/lux - ref: lux-2.8.1 - path: lux - - name: Install lux - run: | - set -e -u -x - cd lux - autoconf - ./configure - make - echo "$(pwd)/bin" >> $GITHUB_PATH - uses: actions/download-artifact@v3 name: Download built emqx and test scenario with: @@ -96,7 +79,7 @@ jobs: path: . - name: run relup test run: | - set -e -x -u + set -eux chmod a+x scripts/**/*.sh ls -l scripts ls -l scripts/relup-test diff --git a/build b/build index 1ca324120..fb0d213f2 100755 --- a/build +++ b/build @@ -361,10 +361,10 @@ make_tgz() { log "Archive sha256sum: $(cat "${target}.sha256")" } -trap docker_cleanup EXIT - docker_cleanup() { rm -f ./.dockerignore >/dev/null + # shellcheck disable=SC2015 + [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true } ## Build the default docker image based on debian 11. @@ -384,7 +384,14 @@ make_docker() { if [[ "$PROFILE" = *enterprise* ]]; then extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' fi - echo '_build' >> ./.dockerignore + # shellcheck disable=SC2015 + [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true + trap docker_cleanup EXIT + { + echo '/_build' + echo '/deps' + echo '/*.lock' + } >> ./.dockerignore set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ @@ -394,6 +401,7 @@ make_docker() { --tag "${EMQX_IMAGE_TAG}" \ -f "${EMQX_DOCKERFILE}" . [[ "${DEBUG:-}" -eq 1 ]] || set +x + echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag } function join {