From 362235021ce2fdc5790c5fd53e86c3379b4bf981 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 12 Jul 2023 15:55:38 +0200 Subject: [PATCH] ci: refactor workflows to use single entry point on pull request trigger --- .../docker-compose-emqx-cluster.yaml | 6 +- .ci/docker-compose-file/haproxy/haproxy.cfg | 4 +- .ci/pr-sanity-checks.sh | 32 ++ .github/actions/pr-sanity-checks/action.yaml | 68 ++++ .github/actions/prepare-jmeter/action.yaml | 49 +++ .github/workflows/_pr_entrypoint.yaml | 194 ++++++++++ .github/workflows/apps_version_check.yaml | 14 - .../build_and_push_docker_images.yaml | 12 +- .github/workflows/build_docker_for_test.yaml | 78 ++++ .github/workflows/build_packages.yaml | 16 +- .github/workflows/build_slim_packages.yaml | 164 +++----- .github/workflows/check_deps_integrity.yaml | 46 ++- .github/workflows/code_style_check.yaml | 29 -- .github/workflows/elixir_apps_check.yaml | 45 --- .github/workflows/elixir_deps_check.yaml | 49 --- .github/workflows/elixir_release.yaml | 47 +++ .github/workflows/elixir_release.yml | 41 -- .github/workflows/run_conf_tests.yaml | 60 +-- .github/workflows/run_docker_tests.yaml | 127 +++++++ .github/workflows/run_emqx_app_tests.yaml | 80 ---- .github/workflows/run_fvt_tests.yaml | 255 ------------- .github/workflows/run_gitlint.yaml | 16 - .github/workflows/run_helm_tests.yaml | 160 ++++++++ .github/workflows/run_jmeter_tests.yaml | 358 +++--------------- .github/workflows/run_relup_tests.yaml | 45 ++- .github/workflows/run_test_cases.yaml | 248 ++++-------- .github/workflows/shellcheck.yaml | 19 - .github/workflows/spellcheck.yaml | 46 +++ .github/workflows/static_checks.yaml | 59 +++ build | 14 +- 30 files changed, 1158 insertions(+), 1223 deletions(-) create mode 100755 .ci/pr-sanity-checks.sh create mode 100644 .github/actions/pr-sanity-checks/action.yaml create mode 100644 .github/actions/prepare-jmeter/action.yaml create mode 100644 .github/workflows/_pr_entrypoint.yaml delete mode 100644 .github/workflows/apps_version_check.yaml create mode 100644 .github/workflows/build_docker_for_test.yaml delete mode 100644 .github/workflows/code_style_check.yaml delete mode 100644 .github/workflows/elixir_apps_check.yaml delete mode 100644 .github/workflows/elixir_deps_check.yaml create mode 100644 .github/workflows/elixir_release.yaml delete mode 100644 .github/workflows/elixir_release.yml create mode 100644 .github/workflows/run_docker_tests.yaml delete mode 100644 .github/workflows/run_emqx_app_tests.yaml delete mode 100644 .github/workflows/run_fvt_tests.yaml delete mode 100644 .github/workflows/run_gitlint.yaml create mode 100644 .github/workflows/run_helm_tests.yaml delete mode 100644 .github/workflows/shellcheck.yaml create mode 100644 .github/workflows/spellcheck.yaml create mode 100644 .github/workflows/static_checks.yaml diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 65bf4faf8..5745dcf5b 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -19,7 +19,9 @@ services: - emqx2 volumes: - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - - ../../apps/emqx/etc/certs:/usr/local/etc/haproxy/certs + - ../../apps/emqx/etc/certs/cert.pem:/usr/local/etc/haproxy/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/usr/local/etc/haproxy/certs/key.pem + - ../../apps/emqx/etc/certs/cacert.pem:/usr/local/etc/haproxy/certs/cacert.pem ports: - "18083:18083" # - "1883:1883" @@ -34,7 +36,7 @@ services: - -c - | set -x - cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /tmp/emqx.pem + cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /var/lib/haproxy/emqx.pem haproxy -f /usr/local/etc/haproxy/haproxy.cfg emqx1: diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg index 1f10c4f9e..8a863c2d3 100644 --- a/.ci/docker-compose-file/haproxy/haproxy.cfg +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -83,13 +83,13 @@ backend emqx_ws_back frontend emqx_ssl mode tcp option tcplog - bind *:8883 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + bind *:8883 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 default_backend emqx_ssl_back frontend emqx_wss mode tcp option tcplog - bind *:8084 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + bind *:8084 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 default_backend emqx_wss_back backend emqx_ssl_back diff --git a/.ci/pr-sanity-checks.sh b/.ci/pr-sanity-checks.sh new file mode 100755 index 000000000..bdd4b3d48 --- /dev/null +++ b/.ci/pr-sanity-checks.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if ! type "docker" > /dev/null; then + echo "docker is not installed" + exit 1 +fi + +if ! type "yq" > /dev/null; then + echo "yq is not installed" + exit 1 +fi + +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-1} +EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-25.3.2-1} +EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5} +EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} +EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}} + +commands=$(yq ".runs.steps[].run" .github/actions/pr-sanity-checks/action.yaml | grep -v null) + +BEFORE_REF=${BEFORE_REF:-$(git rev-parse master)} +AFTER_REF=${AFTER_REF:-$(git rev-parse HEAD)} +docker run --rm -it -v "$(pwd):/emqx" -w /emqx \ + -e GITHUB_WORKSPACE=/emqx \ + -e BEFORE_REF="$BEFORE_REF" \ + -e AFTER_REF="$AFTER_REF" \ + -e GITHUB_BASE_REF="$BEFORE_REF" \ + -e MIX_ENV=emqx-enterprise \ + -e PROFILE=emqx-enterprise \ + "${EMQX_BUILDER}" /bin/bash -c "${commands}" diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml new file mode 100644 index 000000000..60976e27e --- /dev/null +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -0,0 +1,68 @@ +name: 'Pull request sanity checks' + +defaults: + run: + shell: 'bash -Eeuo pipefail {0}' + +runs: + using: composite + steps: + - name: Work around https://github.com/actions/checkout/issues/766 + shell: bash + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Run gitlint + shell: bash + env: + BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + AFTER_REF: ${{ github.sha }} + run: | + pip install gitlint + gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint + - name: Run shellcheck + shell: bash + run: | + apt-get update -y && apt-get install -y shellcheck + ./scripts/shellcheck.sh + # https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions + - name: Check workflow files + shell: bash + run: | + bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + # TODO: enable shellcheck when all the current issues are fixed + ./actionlint -color \ + -shellcheck= \ + -ignore 'label ".+" is unknown' \ + -ignore 'value "emqx-enterprise" in "exclude"' + - name: Check line-break at EOF + shell: bash + run: | + ./scripts/check-nl-at-eof.sh + - name: Check apps version + shell: bash + run: | + ./scripts/apps-version-check.sh + - name: Check Erlang code formatting + shell: bash + run: | + echo "==> ./scripts/check-format.sh" + ./scripts/check-format.sh + echo "==> check-format ok" + - name: Run elvis check + shell: bash + run: | + ./scripts/elvis-check.sh $GITHUB_BASE_REF + - name: Setup mix + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + shell: bash + run: | + mix local.hex --force && mix local.rebar --force + - name: Check Elixir code formatting + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + shell: bash + run: | + mix format --check-formatted diff --git a/.github/actions/prepare-jmeter/action.yaml b/.github/actions/prepare-jmeter/action.yaml new file mode 100644 index 000000000..23e868212 --- /dev/null +++ b/.github/actions/prepare-jmeter/action.yaml @@ -0,0 +1,49 @@ +name: 'Run jmeter' + +inputs: + version-emqx: + required: true + type: string + +runs: + using: composite + steps: + - uses: actions/download-artifact@v3 + with: + name: emqx-docker + path: /tmp + - name: load docker image + shell: bash + env: + PKG_VSN: ${{ inputs.version-emqx }} + run: | + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV + - uses: actions/checkout@v3 + with: + repository: emqx/emqx-fvt + ref: broker-autotest-v5 + path: scripts + - uses: actions/setup-java@v3 + with: + java-version: '8.0.282' # The JDK version to make available on the path. + java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk + architecture: x64 # (x64 or x86) - defaults to x64 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 + with: + name: apache-jmeter.tgz + - name: install jmeter + shell: bash + env: + JMETER_VERSION: 5.4.3 + run: | + tar -xf apache-jmeter.tgz + ln -s apache-jmeter-$JMETER_VERSION jmeter + echo "jmeter.save.saveservice.output_format=xml" >> jmeter/user.properties + echo "jmeter.save.saveservice.response_data.on_error=true" >> jmeter/user.properties + cd jmeter/lib/ext + wget --no-verbose https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar + wget --no-verbose https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar + wget --no-verbose https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml new file mode 100644 index 000000000..1f1920632 --- /dev/null +++ b/.github/workflows/_pr_entrypoint.yaml @@ -0,0 +1,194 @@ +name: PR Entrypoint + +on: + pull_request: + +env: + IS_CI: "yes" + OTP_VSN: "25.3.2-1" + ELIXIR_VSN: "1.14.5" + BUILDER_VSN: "25.3.2-1" + +jobs: + sanity-checks: + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + outputs: + ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} + ct-host: ${{ steps.matrix.outputs.ct-host }} + ct-docker: ${{ steps.matrix.outputs.ct-docker }} + version-emqx: ${{ steps.matrix.outputs.version-emqx }} + version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} + runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + builder: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + builder_vsn: "5.1-3" + otp_vsn: "25.3.2-1" + elixir_vsn: "1.14.5" + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/pr-sanity-checks + - name: Build matrix + id: matrix + run: | + APPS="$(./scripts/find-apps.sh --ci)" + MATRIX="$(echo "${APPS}" | jq -c ' + [ + (.[] | select(.profile == "emqx") | . + { + builder: "5.1-3", + otp: "25.3.2-1", + elixir: "1.14.5" + }), + (.[] | select(.profile == "emqx-enterprise") | . + { + builder: "5.1-3", + otp: ["25.3.2-1"][], + elixir: "1.14.5" + }) + ] + ')" + echo "${MATRIX}" | jq + CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT + echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT + echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT + echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT + echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT + + compile: + runs-on: ${{ needs.sanity-checks.outputs.runner }} + container: ${{ needs.sanity-checks.outputs.builder }} + needs: + - sanity-checks + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - id: compile + env: + PROFILE: ${{ matrix.profile }} + ENABLE_COVER_COMPILE: 1 + run: | + make $PROFILE + zip -ryq $PROFILE.zip . + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.profile }} + path: ${{ matrix.profile }}.zip + retention-days: 1 + + run_test_cases: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_test_cases.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + ct-host: ${{ needs.sanity-checks.outputs.ct-host }} + ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} + + static_checks: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/static_checks.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + + build_slim_packages: + needs: + - sanity-checks + uses: ./.github/workflows/build_slim_packages.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }} + otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} + elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} + + build_docker_for_test: + needs: + - sanity-checks + uses: ./.github/workflows/build_docker_for_test.yaml + with: + otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} + elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} + + spellcheck: + needs: + - sanity-checks + - build_slim_packages + uses: ./.github/workflows/spellcheck.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + + run_conf_tests: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_conf_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + check_deps_integrity: + needs: + - sanity-checks + uses: ./.github/workflows/check_deps_integrity.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + elixir_release: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/elixir_release.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + run_jmeter_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_jmeter_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + + run_docker_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_docker_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} + + run_helm_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_helm_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml deleted file mode 100644 index 52c467786..000000000 --- a/.github/workflows/apps_version_check.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Check Apps Version - -on: [pull_request] - -jobs: - check_apps_version: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check apps version - run: ./scripts/apps-version-check.sh diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index e8066b8bc..ddcf2c6ac 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -5,11 +5,13 @@ concurrency: cancel-in-progress: true on: - push: - tags: - - v* - - e* - - docker-latest-* + workflow_call: +# on: +# push: +# tags: +# - v* +# - e* +# - docker-latest-* workflow_dispatch: inputs: branch_or_tag: diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml new file mode 100644 index 000000000..756383d7f --- /dev/null +++ b/.github/workflows/build_docker_for_test.yaml @@ -0,0 +1,78 @@ +name: Build docker image for test + +concurrency: + group: docker-test-build-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string + version-emqx: + required: true + type: string + version-emqx-enterprise: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + env: + EMQX_NAME: ${{ matrix.profile }} + PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + OTP_VSN: ${{ inputs.otp_vsn }} + ELIXIR_VSN: ${{ inputs.elixir_vsn }} + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + - emqx-elixir + + steps: + - uses: actions/checkout@v3 + - name: build and export to Docker + id: build + run: | + make ${EMQX_NAME}-docker + echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_ENV + - name: smoke test + run: | + CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + docker stop $CID + - name: export docker image + run: | + docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: "${{ env.EMQX_NAME }}-docker" + path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" + retention-days: 3 + diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index bb6b46612..c65b05677 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -5,12 +5,14 @@ concurrency: cancel-in-progress: true on: - push: - branches: - - 'ci/**' - tags: - - v* - - e* + workflow_call: +# on: +# push: +# branches: +# - 'ci/**' +# tags: +# - v* +# - e* workflow_dispatch: inputs: branch_or_tag: @@ -88,7 +90,7 @@ jobs: fetch-depth: 0 - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: 25.3.2 - name: build diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a955b4a9b..c5430439b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,58 +1,69 @@ name: Build slim packages concurrency: - group: slim-${{ github.event_name }}-${{ github.ref }} + group: slim-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: - push: - branches: - - master - - release-51 - pull_request: - # GitHub pull_request action is by default triggered when - # opened reopened or synchronize, - # we add labeled and unlabeled to the list because - # the mac job dpends on the PR having a 'Mac' label - types: - - labeled - - unlabeled - - opened - - reopened - - synchronize + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + builder_vsn: + required: true + type: string + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize workflow_dispatch: jobs: linux: - runs-on: aws-amd64 + runs-on: ${{ inputs.runner }} + env: + EMQX_NAME: ${{ matrix.profile[0] }} strategy: fail-fast: false matrix: profile: - - ["emqx", "25.3.2-1", "el7", "erlang"] - - ["emqx", "25.3.2-1", "ubuntu22.04", "elixir"] - - ["emqx-enterprise", "25.3.2-1", "amzn2023", "erlang"] + - ["emqx", "25.3.2-1", "ubuntu20.04", "elixir"] - ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"] - builder: - - 5.1-3 - elixir: - - '1.14.5' - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" + container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV - echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Work around https://github.com/actions/checkout/issues/766 run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: build and test tgz package if: matrix.profile[3] == 'erlang' run: | @@ -77,12 +88,14 @@ jobs: with: name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" path: _packages/${{ matrix.profile[0] }}/* + retention-days: 7 - uses: actions/upload-artifact@v3 with: name: "${{ matrix.profile[0] }}_schema_dump" path: | scripts/spellcheck _build/docgen/${{ matrix.profile[0] }}/schema-en.json + retention-days: 7 windows: runs-on: windows-2019 @@ -96,7 +109,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: ${{ matrix.otp }} - name: build @@ -129,6 +142,7 @@ jobs: with: name: windows path: _packages/${{ matrix.profile }}/* + retention-days: 7 mac: strategy: @@ -136,20 +150,18 @@ jobs: matrix: profile: - emqx - - emqx-enterprise otp: - - 25.3.2-1 + - ${{ inputs.otp_vsn }} os: - macos-11 - macos-12-arm64 runs-on: ${{ matrix.os }} + env: + EMQX_NAME: ${{ matrix.profile }} steps: - uses: actions/checkout@v3 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} @@ -163,84 +175,4 @@ jobs: with: name: ${{ matrix.os }} path: _packages/**/* - - docker: - runs-on: aws-amd64 - - strategy: - fail-fast: false - matrix: - profile: - - ["emqx", "5.0.16"] - - ["emqx-enterprise", "5.0.1"] - - steps: - - uses: actions/checkout@v3 - - name: prepare - run: | - EMQX_NAME=${{ matrix.profile[0] }} - PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)} - EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test - EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }} - echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV - echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV - echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV - echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 - - name: build and export to Docker - uses: docker/build-push-action@v4 - with: - context: . - file: ./deploy/docker/Dockerfile - load: true - tags: ${{ env.EMQX_IMAGE_TAG }} - build-args: | - EMQX_NAME=${{ env.EMQX_NAME }} - - name: smoke test - run: | - CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - docker stop $CID - - name: dashboard tests - working-directory: ./scripts/ui-tests - run: | - set -eu - docker compose up --abort-on-container-exit --exit-code-from selenium - - name: test two nodes cluster with proto_dist=inet_tls in docker - run: | - ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - # cleanup - ./scripts/test/start-two-nodes-in-docker.sh -c - - name: export docker image - run: | - docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz - - uses: actions/upload-artifact@v3 - with: - name: "${{ matrix.profile[0] }}-docker" - path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz" - - name: cleanup - if: always() - working-directory: ./scripts/ui-tests - run: | - docker compose rm -fs - - spellcheck: - needs: linux - strategy: - matrix: - profile: - - emqx - - emqx-enterprise - runs-on: aws-amd64 - steps: - - uses: actions/download-artifact@v3 - name: Download schema dump - with: - name: "${{ matrix.profile }}_schema_dump" - path: /tmp/ - - name: Run spellcheck - run: | - bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json + retention-days: 7 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 4a079c570..8ece0746b 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -1,14 +1,46 @@ -name: Check Rebar Dependencies +name: Check integrity of rebar and mix dependencies on: - pull_request: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string jobs: check_deps_integrity: - runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 - + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} steps: - uses: actions/checkout@v3 - - name: Run check-deps-integrity.escript - run: ./scripts/check-deps-integrity.escript + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: make ensure-rebar3 + - run: ./scripts/check-deps-integrity.escript + - name: Setup mix + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + run: | + mix local.hex --force + mix local.rebar --force + mix deps.get + - run: ./scripts/check-elixir-deps-discrepancies.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + - run: ./scripts/check-elixir-applications.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + - name: Upload produced lock files + uses: actions/upload-artifact@v3 + if: failure() + with: + name: produced_lock_files + path: | + mix.lock + rebar.lock + retention-days: 1 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml deleted file mode 100644 index ff1043b81..000000000 --- a/.github/workflows/code_style_check.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Code style check - -on: [pull_request] - -jobs: - code_style_check: - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1000 - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Check line-break at EOF - run: | - ./scripts/check-nl-at-eof.sh - - name: Check Elixir code formatting - run: | - mix format --check-formatted - - - name: Check Erlang code formatting - run: | - ./scripts/check-format.sh - - - name: Run elvis check - run: | - ./scripts/elvis-check.sh $GITHUB_BASE_REF diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml deleted file mode 100644 index 31f70690e..000000000 --- a/.github/workflows/elixir_apps_check.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- - -name: Check Elixir Release Applications - -on: - pull_request: - -jobs: - elixir_apps_check: - runs-on: ubuntu-22.04 - # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - - emqx-pkg - - emqx-enterprise-pkg - - steps: - - name: fix_git_permission - run: git config --global --add safe.directory '/__w/emqx/emqx' - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: ensure rebar - run: ./scripts/ensure-rebar3.sh - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: check applications - run: ./scripts/check-elixir-applications.exs - env: - MIX_ENV: ${{ matrix.profile }} - PROFILE: ${{ matrix.profile }} -# - name: check applications started with emqx_machine -# run: ./scripts/check-elixir-emqx-machine-boot-discrepancies.exs -# env: -# MIX_ENV: ${{ matrix.profile }} - -... diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml deleted file mode 100644 index a7e086bb1..000000000 --- a/.github/workflows/elixir_deps_check.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -name: Elixir Dependency Version Check - -on: - pull_request: - -jobs: - elixir_deps_check: - runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 - - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: ensure rebar - run: ./scripts/ensure-rebar3.sh - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: setup mix - run: | - mix local.hex --force - mix local.rebar --force - mix deps.get - # we check only enterprise because `rebar3 tree`, even if an - # enterprise app is excluded from `project_app_dirs` in - # `rebar.config.erl`, will still list dependencies from it. - # Since the enterprise profile is a superset of the - # community one and thus more complete, we use the former. - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - - name: check elixir deps - run: ./scripts/check-elixir-deps-discrepancies.exs - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - - name: produced lock files - uses: actions/upload-artifact@v3 - if: failure() - with: - name: produced_lock_files - path: | - mix.lock - rebar.lock - retention-days: 1 - -... diff --git a/.github/workflows/elixir_release.yaml b/.github/workflows/elixir_release.yaml new file mode 100644 index 000000000..06b26fc8e --- /dev/null +++ b/.github/workflows/elixir_release.yaml @@ -0,0 +1,47 @@ +# FIXME: temporary workflow for testing; remove later +name: Elixir Build (temporary) + +concurrency: + group: elixir-release-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + +jobs: + elixir_release_build: + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} + strategy: + matrix: + profile: + - emqx + + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: elixir release + run: make ${{ matrix.profile }}-elixir + - name: start release + run: | + _build/${{ matrix.profile }}/rel/emqx/bin/emqx start + - name: check if started + run: | + set -e + sleep 10 + timeout 1 bash -c '/dev/null | sed 's/Loaded image: //g') + echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - name: dashboard tests + working-directory: ./scripts/ui-tests + run: | + set -eu + docker compose up --abort-on-container-exit --exit-code-from selenium + - name: test two nodes cluster with proto_dist=inet_tls in docker + run: | + ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + ./scripts/test/start-two-nodes-in-docker.sh -c + - name: cleanup + if: always() + working-directory: ./scripts/ui-tests + run: | + docker compose rm -fs + + paho-mqtt-testing: + runs-on: ${{ inputs.runner }} + defaults: + run: + shell: bash + env: + EMQX_NAME: ${{ matrix.profile }} + PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + _EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }} + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + - emqx-elixir + cluster_db_backend: + - mnesia + - rlog + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: ${{ env.EMQX_NAME }}-docker + path: /tmp + - name: load docker image + run: | + EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - name: run emqx + timeout-minutes: 5 + run: | + ./.ci/docker-compose-file/scripts/run-emqx.sh $_EMQX_DOCKER_IMAGE_TAG $_EMQX_TEST_DB_BACKEND + - name: make paho tests + run: | + if ! docker exec -i python /scripts/pytest.sh "$_EMQX_TEST_DB_BACKEND"; then + echo "DUMP_CONTAINER_LOGS_BGN" + echo "============== haproxy ==============" + docker logs haproxy + echo "============== node1 ==============" + docker logs node1.emqx.io + echo "============== node2 ==============" + docker logs node2.emqx.io + echo "DUMP_CONTAINER_LOGS_END" + exit 1 + fi + # node_dump requires netstat, which is not available in the container + # simple smoke test for node_dump + # - name: test node_dump + # run: | + # docker exec -u root node1.emqx.io apt update && apt install -y net-tools + # docker exec node1.emqx.io node_dump diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml deleted file mode 100644 index 40a630e76..000000000 --- a/.github/workflows/run_emqx_app_tests.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: Check emqx app standalone - -## apps/emqx can be used as a rebar/mix dependency -## in other project, so we need to make sure apps/emqx -## as an Erlang/Elixir app works standalone - -on: - pull_request: - -jobs: - run_emqx_app_tests: - strategy: - matrix: - builder: - - 5.1-3 - otp: - - 25.3.2-1 - # no need to use more than 1 version of Elixir, since tests - # run using only Erlang code. This is needed just to specify - # the base image. - elixir: - - 1.14.5 - os: - - ubuntu22.04 - arch: - - amd64 - runs-on: - - aws-amd64 - - ubuntu-22.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-22.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false - - runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" - - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: run - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - echo "git diff base: $GITHUB_BASE_REF" - if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then - # base is a commit sha1 - compare_base="$GITHUB_BASE_REF" - else - repo="${GITHUB_REPOSITORY}" - git remote -v - remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')" - git fetch "$remote" "$GITHUB_BASE_REF" - compare_base="$remote/$GITHUB_BASE_REF" - fi - changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)" - if [ "$changed_files" = '' ]; then - echo "nothing changed in apps/emqx, ignored." - exit 0 - fi - make ensure-rebar3 - cp rebar3 apps/emqx/ - cd apps/emqx - ./rebar3 xref - ./rebar3 dialyzer - ./rebar3 eunit -v - ./rebar3 ct --name 'test@127.0.0.1' -v --readable=true - ./rebar3 proper -d test/props - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: logs-${{ matrix.runs-on }} - path: apps/emqx/_build/test/logs diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml deleted file mode 100644 index 0bcdee93a..000000000 --- a/.github/workflows/run_fvt_tests.yaml +++ /dev/null @@ -1,255 +0,0 @@ -name: Functional Verification Tests - -concurrency: - group: fvt-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - master - - 'ci/**' - tags: - - v* - pull_request: - -jobs: - prepare: - runs-on: ubuntu-22.04 - # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11 - - steps: - - uses: actions/checkout@v3 - with: - path: source - fetch-depth: 0 - - name: get deps - run: | - make -C source deps-all - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip - - docker_test: - runs-on: ubuntu-22.04 - needs: prepare - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - - emqx-elixir - cluster_db_backend: - - mnesia - - rlog - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - - name: make docker image - working-directory: source - env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} - run: | - make ${{ matrix.profile }}-docker - - name: run emqx - timeout-minutes: 5 - working-directory: source - run: | - set -x - if [[ "${{ matrix.profile }}" = *-elixir ]] - then - export IS_ELIXIR=yes - PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g") - IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir - else - IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) - fi - ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - - name: make paho tests - run: | - if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then - echo "DUMP_CONTAINER_LOGS_BGN" - echo "============== haproxy ==============" - docker logs haproxy - echo "============== node1 ==============" - docker logs node1.emqx.io - echo "============== node2 ==============" - docker logs node2.emqx.io - echo "DUMP_CONTAINER_LOGS_END" - exit 1 - fi - # simple smoke test for node_dump - - name: test node_dump - run: | - docker exec node1.emqx.io node_dump - - helm_test: - runs-on: ubuntu-22.04 - needs: prepare - - strategy: - fail-fast: false - matrix: - discovery: - - k8s - - dns - profile: - - emqx - - emqx-enterprise - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - # - emqx-enterprise # TODO test enterprise - - steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - - name: make docker image - working-directory: source - env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} - run: | - make ${{ matrix.profile }}-docker - echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - - run: minikube start - - run: minikube image load $TARGET:$EMQX_TAG - - name: run emqx on chart - working-directory: source - if: matrix.discovery == 'k8s' - run: | - helm install ${{ matrix.profile }} \ - --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ - --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ - --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${{ matrix.profile }}-headless" \ - --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ - --set image.repository=$TARGET \ - --set image.pullPolicy=Never \ - --set image.tag=$EMQX_TAG \ - --set emqxAclConfig="" \ - --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ - --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ - --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${{ matrix.profile }} \ - --debug - - name: run emqx on chart - working-directory: source - if: matrix.discovery == 'dns' - run: | - helm install ${{ matrix.profile }} \ - --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ - --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ - --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${{ matrix.profile }}-headless.default.svc.cluster.local" \ - --set image.repository=$TARGET \ - --set image.pullPolicy=Never \ - --set image.tag=$EMQX_TAG \ - --set emqxAclConfig="" \ - --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ - --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ - --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${{ matrix.profile }} \ - --debug - - name: waiting emqx started - timeout-minutes: 10 - run: | - while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.replicas}')" \ - != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do - echo "=============================="; - kubectl get pods; - echo "=============================="; - echo "waiting emqx started"; - sleep 10; - done - - name: Get Token - timeout-minutes: 1 - run: | - kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null & - - while - [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] - do - echo "waiting emqx" - sleep 1 - done - - echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV - - - name: Check cluster - timeout-minutes: 10 - run: | - while - [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; - do - echo "waiting ${{ matrix.profile }} cluster scale" - sleep 1 - done - - uses: actions/checkout@v3 - with: - repository: emqx/paho.mqtt.testing - ref: develop-5.0 - path: paho.mqtt.testing - - name: install pytest - run: | - pip install pytest==7.1.2 pytest-retry - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run paho test - timeout-minutes: 10 - run: | - port_connected () { - local server="$1" - local port="$2" - echo > /dev/tcp/${server}/${port} 2>/dev/null - } - - kubectl port-forward service/${{ matrix.profile }} 1883:1883 > /dev/null & - - while ! port_connected localhost 1883; do - echo server not listening yet... - sleep 10 - done - - pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" - - if: failure() - run: kubectl logs -l "app.kubernetes.io/instance=${{ matrix.profile }}" -c emqx --tail=1000 diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml deleted file mode 100644 index 52082c56e..000000000 --- a/.github/workflows/run_gitlint.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Run gitlint - -on: [pull_request] - -jobs: - run_gitlint: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Run gitlint - shell: bash - run: | - set -ex - docker run --ulimit nofile=1024 -v $(pwd):/repo -w /repo ghcr.io/emqx/gitlint --commits ${{ github.event.pull_request.base.sha }}..$GITHUB_SHA --config .github/workflows/.gitlint diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml new file mode 100644 index 000000000..fbd45bc5b --- /dev/null +++ b/.github/workflows/run_helm_tests.yaml @@ -0,0 +1,160 @@ +name: Helm tests + +concurrency: + group: helm-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + version-emqx: + required: true + type: string + version-emqx-enterprise: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# pull_request: + +jobs: + helm_test: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + env: + EMQX_NAME: ${{ matrix.profile }} + EMQX_TAG: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + REPOSITORY: "emqx/${{ matrix.profile }}" + + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + with: + path: source + - uses: actions/download-artifact@v3 + with: + name: "${{ env.EMQX_NAME }}-docker" + path: /tmp + - run: minikube start + - run: | + img="/tmp/${EMQX_NAME}-docker-${EMQX_TAG}.tar.gz" + if stderr=$(minikube image load "${img}" 2>&1 >/dev/null) && test -n "$stderr"; then + echo "${stderr}"; + exit 1; + fi + - name: run emqx on chart (k8s) + if: matrix.discovery == 'k8s' + working-directory: source + run: | + helm install ${EMQX_NAME} \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ + --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ + --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${EMQX_NAME}-headless" \ + --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ + --set image.repository=$REPOSITORY \ + --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ + --set emqxAclConfig="" \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ + --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ + deploy/charts/${EMQX_NAME} \ + --debug + - name: run emqx on chart (dns) + if: matrix.discovery == 'dns' + working-directory: source + run: | + helm install ${EMQX_NAME} \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${EMQX_NAME}-headless.default.svc.cluster.local" \ + --set image.repository=$REPOSITORY \ + --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ + --set emqxAclConfig="" \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ + --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ + deploy/charts/${EMQX_NAME} \ + --debug + - name: waiting emqx started + timeout-minutes: 5 + run: | + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + echo "=============================="; + kubectl get pods; + echo "=============================="; + echo "waiting emqx started"; + sleep 10; + done + - name: Get Token + timeout-minutes: 1 + run: | + kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null & + + while + [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] + do + echo "waiting emqx" + sleep 1 + done + + echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV + + - name: Check cluster + timeout-minutes: 10 + run: | + while + [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; + do + echo "waiting ${EMQX_NAME} cluster scale" + sleep 1 + done + - uses: actions/checkout@v3 + with: + repository: emqx/paho.mqtt.testing + ref: develop-5.0 + path: paho.mqtt.testing + - name: install pytest + run: | + pip install pytest==7.1.2 pytest-retry + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: run paho test + timeout-minutes: 10 + run: | + port_connected () { + local server="$1" + local port="$2" + echo > /dev/tcp/${server}/${port} 2>/dev/null + } + + kubectl port-forward service/${EMQX_NAME} 1883:1883 > /dev/null & + + while ! port_connected localhost 1883; do + echo server not listening yet... + sleep 10 + done + + pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" + - if: failure() + run: kubectl logs -l "app.kubernetes.io/instance=${EMQX_NAME}" -c emqx --tail=1000 diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 04866c7a9..7ca3363de 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -1,22 +1,23 @@ name: JMeter integration tests on: - push: - tags: - - "v5.*" - pull_request: - branches: - - "master" + workflow_call: + inputs: + version-emqx: + required: true + type: string +# on: +# push: +# tags: +# - "v5.*" +# pull_request: +# branches: +# - "master" jobs: - build_emqx_for_jmeter_tests: + jmeter_artifact: runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.build_docker.outputs.version}} steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - name: Cache Jmeter id: cache-jmeter uses: actions/cache@v3 @@ -42,21 +43,6 @@ jobs: with: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz - - uses: actions/checkout@v3 - - name: zip emqx docker image - id: build_docker - if: endsWith(github.repository, 'emqx') - run: | - ## TODO: make profile a matrix dimension - PROFILE='emqx' - make "${PROFILE}-docker" - VSN="$(./pkg-vsn.sh $PROFILE)" - echo "version=${VSN}" >> $GITHUB_OUTPUT - docker save -o emqx.tar emqx/emqx:${VSN} - - uses: actions/upload-artifact@v3 - with: - name: emqx.tar - path: ./emqx.tar advanced_feat: runs-on: ubuntu-22.04 @@ -70,69 +56,28 @@ jobs: - mqtt_topic_rewrite # - mqtt_retainer - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 + - name: show logs + if: failure() run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter + docker compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + logs - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -152,8 +97,6 @@ jobs: pgsql_authn_authz: runs-on: ubuntu-22.04 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} strategy: fail-fast: false @@ -168,72 +111,26 @@ jobs: - pgsql_authn - pgsql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 env: PGSQL_TAG: ${{ matrix.pgsql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/postgresql-42.2.18.jar https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -257,7 +154,7 @@ jobs: - name: dump docker compose logs if: failure() run: | - docker-compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log + docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log - uses: actions/upload-artifact@v3 if: always() with: @@ -277,73 +174,26 @@ jobs: - mysql_authn - mysql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} PGSQL_TAG: ${{ matrix.mysql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -379,45 +229,19 @@ jobs: scripts_type: - jwt_authn - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - name: run jwks_server timeout-minutes: 10 run: | @@ -426,30 +250,9 @@ jobs: cd target docker run --name jwks_server --network emqx_bridge --ip 172.100.239.88 -d -v $(pwd)/jwkserver-0.0.1.jar:/jwks_server/jwkserver-0.0.1.jar --workdir /jwks_server openjdk:8-jdk bash \ -c "java -jar jwkserver-0.0.1.jar" - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -478,79 +281,30 @@ jobs: - built_in_database_authn - built_in_database_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} - PGSQL_TAG: ${{ matrix.mysql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ - -l jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl \ - -j jmeter_logs/logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.log + -l jmeter_logs/${{ matrix.scripts_type }}.jtl \ + -j jmeter_logs/logs/${{ matrix.scripts_type }}.log - name: check logs run: | - if cat jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl | grep -e 'true' > /dev/null 2>&1; then + if cat jmeter_logs/${{ matrix.scripts_type }}.jtl | grep -e 'true' > /dev/null 2>&1; then echo "check logs failed" exit 1 fi @@ -559,11 +313,3 @@ jobs: with: name: jmeter_logs path: ./jmeter_logs - - delete-artifact: - runs-on: ubuntu-22.04 - needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz] - steps: - - uses: geekyeggo/delete-artifact@v2 - with: - name: emqx.tar diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 0400d0502..166fca513 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -4,6 +4,15 @@ concurrency: group: relup-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string # on: # push: # branches: @@ -14,8 +23,8 @@ concurrency: jobs: relup_test_plan: - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} @@ -23,16 +32,18 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@v3 - name: Checkout + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 with: - path: emqx - fetch-depth: 0 + name: emqx-enterprise + - name: extract artifact + run: | + unzip -o -q emqx-enterprise.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Find versions id: find-versions run: | set -x - cd emqx ee_vsn="$(./pkg-vsn.sh enterprise)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') @@ -40,8 +51,6 @@ jobs: echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT - name: build emqx run: | - set -x - cd emqx export PROFILE='emqx-enterprise' make emqx-enterprise-tgz - uses: actions/upload-artifact@v3 @@ -49,10 +58,10 @@ jobs: with: name: emqx_built path: | - emqx/_upgrade_base - emqx/_packages - emqx/scripts - emqx/.ci + _upgrade_base + _packages + scripts + .ci relup_test_run: needs: @@ -70,8 +79,7 @@ jobs: run: shell: bash steps: - # setup Erlang to run lux - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: 25.3.2 - uses: actions/checkout@v3 @@ -81,7 +89,7 @@ jobs: path: lux - name: Install lux run: | - set -e -u -x + set -eu cd lux autoconf ./configure @@ -94,10 +102,7 @@ jobs: path: . - name: run relup test run: | - set -e -x -u - chmod a+x scripts/**/*.sh - ls -l scripts - ls -l scripts/relup-test + set -eux case "$OLD_VSN" in e*) export CUR_VSN="$CUR_EE_VSN" diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d43079d61..e2d2a0abf 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -1,156 +1,48 @@ name: Run test case concurrency: - group: test-${{ github.event_name }}-${{ github.ref }} + group: test-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: - push: - branches: - - master - - 'ci/**' - tags: - - v* - - e* - pull_request: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + ct-matrix: + required: true + type: string + ct-host: + required: true + type: string + ct-docker: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# - e* +# pull_request: env: IS_CI: "yes" jobs: - build-matrix: - runs-on: ubuntu-22.04 - outputs: - prepare: ${{ steps.matrix.outputs.prepare }} - host: ${{ steps.matrix.outputs.host }} - docker: ${{ steps.matrix.outputs.docker }} - runs-on: ${{ steps.runner.outputs.runs-on }} - steps: - - uses: actions/checkout@v3 - - name: Build matrix - id: matrix - run: | - APPS="$(./scripts/find-apps.sh --ci)" - MATRIX="$(echo "${APPS}" | jq -c ' - [ - (.[] | select(.profile == "emqx") | . + { - builder: "5.1-3", - otp: "25.3.2-1", - elixir: "1.14.5" - }), - (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.1-3", - otp: ["25.3.2-1"][], - elixir: "1.14.5" - }) - ] - ')" - echo "${MATRIX}" | jq - MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" - MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" - MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" - echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT - echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT - echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT - - name: Choose runner host - id: runner - run: | - RUNS_ON="ubuntu-22.04" - ${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64" - echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT - - prepare: - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - needs: [build-matrix] - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 - with: - path: source - - name: get_all_deps - working-directory: source - env: - PROFILE: ${{ matrix.profile }} - run: | - make ensure-rebar3 - # fetch all deps and compile - make ${{ matrix.profile }}-compile - make test-compile - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: source.zip - - check_examples: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip - - name: check example config files - env: - PROFILE: ${{ matrix.profile }} - working-directory: source - run: ./scripts/test/check-example-configs.sh - - static_checks: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip - - uses: actions/cache@v3 - with: - path: "source/emqx_dialyzer_${{ matrix.otp }}_plt" - key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }} - restore-keys: | - rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- - - name: run static checks - env: - PROFILE: ${{ matrix.profile }} - working-directory: source - run: make static_checks - eunit_and_proper: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} + include: ${{ fromJson(inputs.ct-matrix) }} defaults: run: @@ -161,16 +53,16 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces eunit.coverdata - name: eunit env: PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - working-directory: source run: make eunit # produces proper.coverdata @@ -178,23 +70,19 @@ jobs: env: PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - working-directory: source run: make proper - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover ct_docker: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.docker) }} + include: ${{ fromJson(inputs.ct-docker) }} defaults: run: @@ -204,14 +92,14 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests - working-directory: source env: DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" MONGO_TAG: "5" @@ -229,22 +117,19 @@ jobs: - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: source/_build/test/logs + path: _build/test/logs ct: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.host) }} + include: ${{ fromJson(inputs.ct-host) }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" defaults: @@ -255,14 +140,14 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests - working-directory: source env: PROFILE: ${{ matrix.profile }} SUITEGROUP: ${{ matrix.suitegroup }} @@ -272,58 +157,61 @@ jobs: - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover if-no-files-found: warn # do not fail if no coverdata found - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: source/_build/test/logs + path: _build/test/logs make_cover: needs: - eunit_and_proper - ct - ct_docker - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} + strategy: + fail-fast: false + matrix: + profile: + - emqx-enterprise steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise-25.3.2-1 - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/download-artifact@v3 name: download coverdata with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover - name: make cover - working-directory: source env: PROFILE: emqx-enterprise run: make cover - name: send to coveralls - working-directory: source env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PROFILE: emqx-enterprise run: make coveralls - name: get coveralls logs - working-directory: source if: failure() run: cat rebar3.crashdump # do this in a separate job upload_coverdata: needs: make_cover - runs-on: ubuntu-22.04 + runs-on: ${{ inputs.runner }} steps: - name: Coveralls Finished env: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml deleted file mode 100644 index 7f29572b9..000000000 --- a/.github/workflows/shellcheck.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Shellcheck - -on: - pull_request: - -jobs: - shellcheck: - runs-on: ubuntu-22.04 - steps: - - name: Checkout source code - uses: actions/checkout@v3 - - name: Install shellcheck - run: | - sudo apt-get update - sudo apt install shellcheck - - name: Run shellcheck - run: | - ./scripts/shellcheck.sh - echo "success" diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml new file mode 100644 index 000000000..07308e1a6 --- /dev/null +++ b/.github/workflows/spellcheck.yaml @@ -0,0 +1,46 @@ +name: Build slim packages + +concurrency: + group: spellcheck-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + spellcheck: + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/download-artifact@v3 + with: + name: "${{ matrix.profile }}_schema_dump" + path: /tmp/ + - name: Run spellcheck + run: | + bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml new file mode 100644 index 000000000..b7fc2444b --- /dev/null +++ b/.github/workflows/static_checks.yaml @@ -0,0 +1,59 @@ +name: Static checks + +concurrency: + group: static-checks-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + ct-matrix: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# - e* +# pull_request: + +env: + IS_CI: "yes" + +jobs: + static_checks: + runs-on: ${{ inputs.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.ct-matrix) }} + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/cache@v3 + with: + path: "emqx_dialyzer_${{ matrix.otp }}_plt" + key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }} + restore-keys: | + rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- + - name: run static checks + env: + PROFILE: ${{ matrix.profile }} + run: make static_checks diff --git a/build b/build index 1ca324120..fb0d213f2 100755 --- a/build +++ b/build @@ -361,10 +361,10 @@ make_tgz() { log "Archive sha256sum: $(cat "${target}.sha256")" } -trap docker_cleanup EXIT - docker_cleanup() { rm -f ./.dockerignore >/dev/null + # shellcheck disable=SC2015 + [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true } ## Build the default docker image based on debian 11. @@ -384,7 +384,14 @@ make_docker() { if [[ "$PROFILE" = *enterprise* ]]; then extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' fi - echo '_build' >> ./.dockerignore + # shellcheck disable=SC2015 + [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true + trap docker_cleanup EXIT + { + echo '/_build' + echo '/deps' + echo '/*.lock' + } >> ./.dockerignore set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ @@ -394,6 +401,7 @@ make_docker() { --tag "${EMQX_IMAGE_TAG}" \ -f "${EMQX_DOCKERFILE}" . [[ "${DEBUG:-}" -eq 1 ]] || set +x + echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag } function join {