From 362235021ce2fdc5790c5fd53e86c3379b4bf981 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 12 Jul 2023 15:55:38 +0200 Subject: [PATCH 01/22] ci: refactor workflows to use single entry point on pull request trigger --- .../docker-compose-emqx-cluster.yaml | 6 +- .ci/docker-compose-file/haproxy/haproxy.cfg | 4 +- .ci/pr-sanity-checks.sh | 32 ++ .github/actions/pr-sanity-checks/action.yaml | 68 ++++ .github/actions/prepare-jmeter/action.yaml | 49 +++ .github/workflows/_pr_entrypoint.yaml | 194 ++++++++++ .github/workflows/apps_version_check.yaml | 14 - .../build_and_push_docker_images.yaml | 12 +- .github/workflows/build_docker_for_test.yaml | 78 ++++ .github/workflows/build_packages.yaml | 16 +- .github/workflows/build_slim_packages.yaml | 164 +++----- .github/workflows/check_deps_integrity.yaml | 46 ++- .github/workflows/code_style_check.yaml | 29 -- .github/workflows/elixir_apps_check.yaml | 45 --- .github/workflows/elixir_deps_check.yaml | 49 --- .github/workflows/elixir_release.yaml | 47 +++ .github/workflows/elixir_release.yml | 41 -- .github/workflows/run_conf_tests.yaml | 60 +-- .github/workflows/run_docker_tests.yaml | 127 +++++++ .github/workflows/run_emqx_app_tests.yaml | 80 ---- .github/workflows/run_fvt_tests.yaml | 255 ------------- .github/workflows/run_gitlint.yaml | 16 - .github/workflows/run_helm_tests.yaml | 160 ++++++++ .github/workflows/run_jmeter_tests.yaml | 358 +++--------------- .github/workflows/run_relup_tests.yaml | 45 ++- .github/workflows/run_test_cases.yaml | 248 ++++-------- .github/workflows/shellcheck.yaml | 19 - .github/workflows/spellcheck.yaml | 46 +++ .github/workflows/static_checks.yaml | 59 +++ build | 14 +- 30 files changed, 1158 insertions(+), 1223 deletions(-) create mode 100755 .ci/pr-sanity-checks.sh create mode 100644 .github/actions/pr-sanity-checks/action.yaml create mode 100644 .github/actions/prepare-jmeter/action.yaml create mode 100644 .github/workflows/_pr_entrypoint.yaml delete mode 100644 .github/workflows/apps_version_check.yaml create mode 100644 .github/workflows/build_docker_for_test.yaml delete mode 100644 .github/workflows/code_style_check.yaml delete mode 100644 .github/workflows/elixir_apps_check.yaml delete mode 100644 .github/workflows/elixir_deps_check.yaml create mode 100644 .github/workflows/elixir_release.yaml delete mode 100644 .github/workflows/elixir_release.yml create mode 100644 .github/workflows/run_docker_tests.yaml delete mode 100644 .github/workflows/run_emqx_app_tests.yaml delete mode 100644 .github/workflows/run_fvt_tests.yaml delete mode 100644 .github/workflows/run_gitlint.yaml create mode 100644 .github/workflows/run_helm_tests.yaml delete mode 100644 .github/workflows/shellcheck.yaml create mode 100644 .github/workflows/spellcheck.yaml create mode 100644 .github/workflows/static_checks.yaml diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 65bf4faf8..5745dcf5b 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -19,7 +19,9 @@ services: - emqx2 volumes: - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - - ../../apps/emqx/etc/certs:/usr/local/etc/haproxy/certs + - ../../apps/emqx/etc/certs/cert.pem:/usr/local/etc/haproxy/certs/cert.pem + - ../../apps/emqx/etc/certs/key.pem:/usr/local/etc/haproxy/certs/key.pem + - ../../apps/emqx/etc/certs/cacert.pem:/usr/local/etc/haproxy/certs/cacert.pem ports: - "18083:18083" # - "1883:1883" @@ -34,7 +36,7 @@ services: - -c - | set -x - cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /tmp/emqx.pem + cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /var/lib/haproxy/emqx.pem haproxy -f /usr/local/etc/haproxy/haproxy.cfg emqx1: diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg index 1f10c4f9e..8a863c2d3 100644 --- a/.ci/docker-compose-file/haproxy/haproxy.cfg +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -83,13 +83,13 @@ backend emqx_ws_back frontend emqx_ssl mode tcp option tcplog - bind *:8883 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + bind *:8883 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 default_backend emqx_ssl_back frontend emqx_wss mode tcp option tcplog - bind *:8084 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + bind *:8084 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 default_backend emqx_wss_back backend emqx_ssl_back diff --git a/.ci/pr-sanity-checks.sh b/.ci/pr-sanity-checks.sh new file mode 100755 index 000000000..bdd4b3d48 --- /dev/null +++ b/.ci/pr-sanity-checks.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if ! type "docker" > /dev/null; then + echo "docker is not installed" + exit 1 +fi + +if ! type "yq" > /dev/null; then + echo "yq is not installed" + exit 1 +fi + +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-1} +EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-25.3.2-1} +EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5} +EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} +EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}} + +commands=$(yq ".runs.steps[].run" .github/actions/pr-sanity-checks/action.yaml | grep -v null) + +BEFORE_REF=${BEFORE_REF:-$(git rev-parse master)} +AFTER_REF=${AFTER_REF:-$(git rev-parse HEAD)} +docker run --rm -it -v "$(pwd):/emqx" -w /emqx \ + -e GITHUB_WORKSPACE=/emqx \ + -e BEFORE_REF="$BEFORE_REF" \ + -e AFTER_REF="$AFTER_REF" \ + -e GITHUB_BASE_REF="$BEFORE_REF" \ + -e MIX_ENV=emqx-enterprise \ + -e PROFILE=emqx-enterprise \ + "${EMQX_BUILDER}" /bin/bash -c "${commands}" diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml new file mode 100644 index 000000000..60976e27e --- /dev/null +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -0,0 +1,68 @@ +name: 'Pull request sanity checks' + +defaults: + run: + shell: 'bash -Eeuo pipefail {0}' + +runs: + using: composite + steps: + - name: Work around https://github.com/actions/checkout/issues/766 + shell: bash + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Run gitlint + shell: bash + env: + BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + AFTER_REF: ${{ github.sha }} + run: | + pip install gitlint + gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint + - name: Run shellcheck + shell: bash + run: | + apt-get update -y && apt-get install -y shellcheck + ./scripts/shellcheck.sh + # https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions + - name: Check workflow files + shell: bash + run: | + bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + # TODO: enable shellcheck when all the current issues are fixed + ./actionlint -color \ + -shellcheck= \ + -ignore 'label ".+" is unknown' \ + -ignore 'value "emqx-enterprise" in "exclude"' + - name: Check line-break at EOF + shell: bash + run: | + ./scripts/check-nl-at-eof.sh + - name: Check apps version + shell: bash + run: | + ./scripts/apps-version-check.sh + - name: Check Erlang code formatting + shell: bash + run: | + echo "==> ./scripts/check-format.sh" + ./scripts/check-format.sh + echo "==> check-format ok" + - name: Run elvis check + shell: bash + run: | + ./scripts/elvis-check.sh $GITHUB_BASE_REF + - name: Setup mix + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + shell: bash + run: | + mix local.hex --force && mix local.rebar --force + - name: Check Elixir code formatting + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + shell: bash + run: | + mix format --check-formatted diff --git a/.github/actions/prepare-jmeter/action.yaml b/.github/actions/prepare-jmeter/action.yaml new file mode 100644 index 000000000..23e868212 --- /dev/null +++ b/.github/actions/prepare-jmeter/action.yaml @@ -0,0 +1,49 @@ +name: 'Run jmeter' + +inputs: + version-emqx: + required: true + type: string + +runs: + using: composite + steps: + - uses: actions/download-artifact@v3 + with: + name: emqx-docker + path: /tmp + - name: load docker image + shell: bash + env: + PKG_VSN: ${{ inputs.version-emqx }} + run: | + EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV + - uses: actions/checkout@v3 + with: + repository: emqx/emqx-fvt + ref: broker-autotest-v5 + path: scripts + - uses: actions/setup-java@v3 + with: + java-version: '8.0.282' # The JDK version to make available on the path. + java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk + architecture: x64 # (x64 or x86) - defaults to x64 + # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md + distribution: 'zulu' + - uses: actions/download-artifact@v3 + with: + name: apache-jmeter.tgz + - name: install jmeter + shell: bash + env: + JMETER_VERSION: 5.4.3 + run: | + tar -xf apache-jmeter.tgz + ln -s apache-jmeter-$JMETER_VERSION jmeter + echo "jmeter.save.saveservice.output_format=xml" >> jmeter/user.properties + echo "jmeter.save.saveservice.response_data.on_error=true" >> jmeter/user.properties + cd jmeter/lib/ext + wget --no-verbose https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar + wget --no-verbose https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar + wget --no-verbose https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml new file mode 100644 index 000000000..1f1920632 --- /dev/null +++ b/.github/workflows/_pr_entrypoint.yaml @@ -0,0 +1,194 @@ +name: PR Entrypoint + +on: + pull_request: + +env: + IS_CI: "yes" + OTP_VSN: "25.3.2-1" + ELIXIR_VSN: "1.14.5" + BUILDER_VSN: "25.3.2-1" + +jobs: + sanity-checks: + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + outputs: + ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} + ct-host: ${{ steps.matrix.outputs.ct-host }} + ct-docker: ${{ steps.matrix.outputs.ct-docker }} + version-emqx: ${{ steps.matrix.outputs.version-emqx }} + version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} + runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + builder: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + builder_vsn: "5.1-3" + otp_vsn: "25.3.2-1" + elixir_vsn: "1.14.5" + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/pr-sanity-checks + - name: Build matrix + id: matrix + run: | + APPS="$(./scripts/find-apps.sh --ci)" + MATRIX="$(echo "${APPS}" | jq -c ' + [ + (.[] | select(.profile == "emqx") | . + { + builder: "5.1-3", + otp: "25.3.2-1", + elixir: "1.14.5" + }), + (.[] | select(.profile == "emqx-enterprise") | . + { + builder: "5.1-3", + otp: ["25.3.2-1"][], + elixir: "1.14.5" + }) + ] + ')" + echo "${MATRIX}" | jq + CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT + echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT + echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT + echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT + echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT + + compile: + runs-on: ${{ needs.sanity-checks.outputs.runner }} + container: ${{ needs.sanity-checks.outputs.builder }} + needs: + - sanity-checks + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - id: compile + env: + PROFILE: ${{ matrix.profile }} + ENABLE_COVER_COMPILE: 1 + run: | + make $PROFILE + zip -ryq $PROFILE.zip . + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.profile }} + path: ${{ matrix.profile }}.zip + retention-days: 1 + + run_test_cases: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_test_cases.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + ct-host: ${{ needs.sanity-checks.outputs.ct-host }} + ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }} + + static_checks: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/static_checks.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }} + + build_slim_packages: + needs: + - sanity-checks + uses: ./.github/workflows/build_slim_packages.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }} + otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} + elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} + + build_docker_for_test: + needs: + - sanity-checks + uses: ./.github/workflows/build_docker_for_test.yaml + with: + otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }} + elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }} + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} + + spellcheck: + needs: + - sanity-checks + - build_slim_packages + uses: ./.github/workflows/spellcheck.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + + run_conf_tests: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_conf_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + check_deps_integrity: + needs: + - sanity-checks + uses: ./.github/workflows/check_deps_integrity.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + elixir_release: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/elixir_release.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + + run_jmeter_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_jmeter_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + + run_docker_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_docker_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} + + run_helm_tests: + needs: + - sanity-checks + - build_docker_for_test + uses: ./.github/workflows/run_helm_tests.yaml + with: + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml deleted file mode 100644 index 52c467786..000000000 --- a/.github/workflows/apps_version_check.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Check Apps Version - -on: [pull_request] - -jobs: - check_apps_version: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check apps version - run: ./scripts/apps-version-check.sh diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index e8066b8bc..ddcf2c6ac 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -5,11 +5,13 @@ concurrency: cancel-in-progress: true on: - push: - tags: - - v* - - e* - - docker-latest-* + workflow_call: +# on: +# push: +# tags: +# - v* +# - e* +# - docker-latest-* workflow_dispatch: inputs: branch_or_tag: diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml new file mode 100644 index 000000000..756383d7f --- /dev/null +++ b/.github/workflows/build_docker_for_test.yaml @@ -0,0 +1,78 @@ +name: Build docker image for test + +concurrency: + group: docker-test-build-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string + version-emqx: + required: true + type: string + version-emqx-enterprise: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + env: + EMQX_NAME: ${{ matrix.profile }} + PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + OTP_VSN: ${{ inputs.otp_vsn }} + ELIXIR_VSN: ${{ inputs.elixir_vsn }} + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + - emqx-elixir + + steps: + - uses: actions/checkout@v3 + - name: build and export to Docker + id: build + run: | + make ${EMQX_NAME}-docker + echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_ENV + - name: smoke test + run: | + CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + docker stop $CID + - name: export docker image + run: | + docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: "${{ env.EMQX_NAME }}-docker" + path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" + retention-days: 3 + diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index bb6b46612..c65b05677 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -5,12 +5,14 @@ concurrency: cancel-in-progress: true on: - push: - branches: - - 'ci/**' - tags: - - v* - - e* + workflow_call: +# on: +# push: +# branches: +# - 'ci/**' +# tags: +# - v* +# - e* workflow_dispatch: inputs: branch_or_tag: @@ -88,7 +90,7 @@ jobs: fetch-depth: 0 - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: 25.3.2 - name: build diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a955b4a9b..c5430439b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,58 +1,69 @@ name: Build slim packages concurrency: - group: slim-${{ github.event_name }}-${{ github.ref }} + group: slim-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: - push: - branches: - - master - - release-51 - pull_request: - # GitHub pull_request action is by default triggered when - # opened reopened or synchronize, - # we add labeled and unlabeled to the list because - # the mac job dpends on the PR having a 'Mac' label - types: - - labeled - - unlabeled - - opened - - reopened - - synchronize + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + builder_vsn: + required: true + type: string + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize workflow_dispatch: jobs: linux: - runs-on: aws-amd64 + runs-on: ${{ inputs.runner }} + env: + EMQX_NAME: ${{ matrix.profile[0] }} strategy: fail-fast: false matrix: profile: - - ["emqx", "25.3.2-1", "el7", "erlang"] - - ["emqx", "25.3.2-1", "ubuntu22.04", "elixir"] - - ["emqx-enterprise", "25.3.2-1", "amzn2023", "erlang"] + - ["emqx", "25.3.2-1", "ubuntu20.04", "elixir"] - ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"] - builder: - - 5.1-3 - elixir: - - '1.14.5' - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" + container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV - echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Work around https://github.com/actions/checkout/issues/766 run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: build and test tgz package if: matrix.profile[3] == 'erlang' run: | @@ -77,12 +88,14 @@ jobs: with: name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" path: _packages/${{ matrix.profile[0] }}/* + retention-days: 7 - uses: actions/upload-artifact@v3 with: name: "${{ matrix.profile[0] }}_schema_dump" path: | scripts/spellcheck _build/docgen/${{ matrix.profile[0] }}/schema-en.json + retention-days: 7 windows: runs-on: windows-2019 @@ -96,7 +109,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: ${{ matrix.otp }} - name: build @@ -129,6 +142,7 @@ jobs: with: name: windows path: _packages/${{ matrix.profile }}/* + retention-days: 7 mac: strategy: @@ -136,20 +150,18 @@ jobs: matrix: profile: - emqx - - emqx-enterprise otp: - - 25.3.2-1 + - ${{ inputs.otp_vsn }} os: - macos-11 - macos-12-arm64 runs-on: ${{ matrix.os }} + env: + EMQX_NAME: ${{ matrix.profile }} steps: - uses: actions/checkout@v3 - - name: prepare - run: | - echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} @@ -163,84 +175,4 @@ jobs: with: name: ${{ matrix.os }} path: _packages/**/* - - docker: - runs-on: aws-amd64 - - strategy: - fail-fast: false - matrix: - profile: - - ["emqx", "5.0.16"] - - ["emqx-enterprise", "5.0.1"] - - steps: - - uses: actions/checkout@v3 - - name: prepare - run: | - EMQX_NAME=${{ matrix.profile[0] }} - PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)} - EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test - EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }} - echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV - echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV - echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV - echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 - - name: build and export to Docker - uses: docker/build-push-action@v4 - with: - context: . - file: ./deploy/docker/Dockerfile - load: true - tags: ${{ env.EMQX_IMAGE_TAG }} - build-args: | - EMQX_NAME=${{ env.EMQX_NAME }} - - name: smoke test - run: | - CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - docker stop $CID - - name: dashboard tests - working-directory: ./scripts/ui-tests - run: | - set -eu - docker compose up --abort-on-container-exit --exit-code-from selenium - - name: test two nodes cluster with proto_dist=inet_tls in docker - run: | - ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG - HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) - ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT - # cleanup - ./scripts/test/start-two-nodes-in-docker.sh -c - - name: export docker image - run: | - docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz - - uses: actions/upload-artifact@v3 - with: - name: "${{ matrix.profile[0] }}-docker" - path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz" - - name: cleanup - if: always() - working-directory: ./scripts/ui-tests - run: | - docker compose rm -fs - - spellcheck: - needs: linux - strategy: - matrix: - profile: - - emqx - - emqx-enterprise - runs-on: aws-amd64 - steps: - - uses: actions/download-artifact@v3 - name: Download schema dump - with: - name: "${{ matrix.profile }}_schema_dump" - path: /tmp/ - - name: Run spellcheck - run: | - bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json + retention-days: 7 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 4a079c570..8ece0746b 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -1,14 +1,46 @@ -name: Check Rebar Dependencies +name: Check integrity of rebar and mix dependencies on: - pull_request: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string jobs: check_deps_integrity: - runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 - + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} steps: - uses: actions/checkout@v3 - - name: Run check-deps-integrity.escript - run: ./scripts/check-deps-integrity.escript + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: make ensure-rebar3 + - run: ./scripts/check-deps-integrity.escript + - name: Setup mix + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + run: | + mix local.hex --force + mix local.rebar --force + mix deps.get + - run: ./scripts/check-elixir-deps-discrepancies.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + - run: ./scripts/check-elixir-applications.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + - name: Upload produced lock files + uses: actions/upload-artifact@v3 + if: failure() + with: + name: produced_lock_files + path: | + mix.lock + rebar.lock + retention-days: 1 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml deleted file mode 100644 index ff1043b81..000000000 --- a/.github/workflows/code_style_check.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Code style check - -on: [pull_request] - -jobs: - code_style_check: - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1000 - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Check line-break at EOF - run: | - ./scripts/check-nl-at-eof.sh - - name: Check Elixir code formatting - run: | - mix format --check-formatted - - - name: Check Erlang code formatting - run: | - ./scripts/check-format.sh - - - name: Run elvis check - run: | - ./scripts/elvis-check.sh $GITHUB_BASE_REF diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml deleted file mode 100644 index 31f70690e..000000000 --- a/.github/workflows/elixir_apps_check.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- - -name: Check Elixir Release Applications - -on: - pull_request: - -jobs: - elixir_apps_check: - runs-on: ubuntu-22.04 - # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - - emqx-pkg - - emqx-enterprise-pkg - - steps: - - name: fix_git_permission - run: git config --global --add safe.directory '/__w/emqx/emqx' - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: ensure rebar - run: ./scripts/ensure-rebar3.sh - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: check applications - run: ./scripts/check-elixir-applications.exs - env: - MIX_ENV: ${{ matrix.profile }} - PROFILE: ${{ matrix.profile }} -# - name: check applications started with emqx_machine -# run: ./scripts/check-elixir-emqx-machine-boot-discrepancies.exs -# env: -# MIX_ENV: ${{ matrix.profile }} - -... diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml deleted file mode 100644 index a7e086bb1..000000000 --- a/.github/workflows/elixir_deps_check.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -name: Elixir Dependency Version Check - -on: - pull_request: - -jobs: - elixir_deps_check: - runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 - - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: ensure rebar - run: ./scripts/ensure-rebar3.sh - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: setup mix - run: | - mix local.hex --force - mix local.rebar --force - mix deps.get - # we check only enterprise because `rebar3 tree`, even if an - # enterprise app is excluded from `project_app_dirs` in - # `rebar.config.erl`, will still list dependencies from it. - # Since the enterprise profile is a superset of the - # community one and thus more complete, we use the former. - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - - name: check elixir deps - run: ./scripts/check-elixir-deps-discrepancies.exs - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - - name: produced lock files - uses: actions/upload-artifact@v3 - if: failure() - with: - name: produced_lock_files - path: | - mix.lock - rebar.lock - retention-days: 1 - -... diff --git a/.github/workflows/elixir_release.yaml b/.github/workflows/elixir_release.yaml new file mode 100644 index 000000000..06b26fc8e --- /dev/null +++ b/.github/workflows/elixir_release.yaml @@ -0,0 +1,47 @@ +# FIXME: temporary workflow for testing; remove later +name: Elixir Build (temporary) + +concurrency: + group: elixir-release-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + +jobs: + elixir_release_build: + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} + strategy: + matrix: + profile: + - emqx + + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: elixir release + run: make ${{ matrix.profile }}-elixir + - name: start release + run: | + _build/${{ matrix.profile }}/rel/emqx/bin/emqx start + - name: check if started + run: | + set -e + sleep 10 + timeout 1 bash -c '/dev/null | sed 's/Loaded image: //g') + echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - name: dashboard tests + working-directory: ./scripts/ui-tests + run: | + set -eu + docker compose up --abort-on-container-exit --exit-code-from selenium + - name: test two nodes cluster with proto_dist=inet_tls in docker + run: | + ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + ./scripts/test/start-two-nodes-in-docker.sh -c + - name: cleanup + if: always() + working-directory: ./scripts/ui-tests + run: | + docker compose rm -fs + + paho-mqtt-testing: + runs-on: ${{ inputs.runner }} + defaults: + run: + shell: bash + env: + EMQX_NAME: ${{ matrix.profile }} + PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + _EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }} + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + - emqx-elixir + cluster_db_backend: + - mnesia + - rlog + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: ${{ env.EMQX_NAME }}-docker + path: /tmp + - name: load docker image + run: | + EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g') + echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - name: run emqx + timeout-minutes: 5 + run: | + ./.ci/docker-compose-file/scripts/run-emqx.sh $_EMQX_DOCKER_IMAGE_TAG $_EMQX_TEST_DB_BACKEND + - name: make paho tests + run: | + if ! docker exec -i python /scripts/pytest.sh "$_EMQX_TEST_DB_BACKEND"; then + echo "DUMP_CONTAINER_LOGS_BGN" + echo "============== haproxy ==============" + docker logs haproxy + echo "============== node1 ==============" + docker logs node1.emqx.io + echo "============== node2 ==============" + docker logs node2.emqx.io + echo "DUMP_CONTAINER_LOGS_END" + exit 1 + fi + # node_dump requires netstat, which is not available in the container + # simple smoke test for node_dump + # - name: test node_dump + # run: | + # docker exec -u root node1.emqx.io apt update && apt install -y net-tools + # docker exec node1.emqx.io node_dump diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml deleted file mode 100644 index 40a630e76..000000000 --- a/.github/workflows/run_emqx_app_tests.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: Check emqx app standalone - -## apps/emqx can be used as a rebar/mix dependency -## in other project, so we need to make sure apps/emqx -## as an Erlang/Elixir app works standalone - -on: - pull_request: - -jobs: - run_emqx_app_tests: - strategy: - matrix: - builder: - - 5.1-3 - otp: - - 25.3.2-1 - # no need to use more than 1 version of Elixir, since tests - # run using only Erlang code. This is needed just to specify - # the base image. - elixir: - - 1.14.5 - os: - - ubuntu22.04 - arch: - - amd64 - runs-on: - - aws-amd64 - - ubuntu-22.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-22.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false - - runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" - - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: run - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - echo "git diff base: $GITHUB_BASE_REF" - if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then - # base is a commit sha1 - compare_base="$GITHUB_BASE_REF" - else - repo="${GITHUB_REPOSITORY}" - git remote -v - remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')" - git fetch "$remote" "$GITHUB_BASE_REF" - compare_base="$remote/$GITHUB_BASE_REF" - fi - changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)" - if [ "$changed_files" = '' ]; then - echo "nothing changed in apps/emqx, ignored." - exit 0 - fi - make ensure-rebar3 - cp rebar3 apps/emqx/ - cd apps/emqx - ./rebar3 xref - ./rebar3 dialyzer - ./rebar3 eunit -v - ./rebar3 ct --name 'test@127.0.0.1' -v --readable=true - ./rebar3 proper -d test/props - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: logs-${{ matrix.runs-on }} - path: apps/emqx/_build/test/logs diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml deleted file mode 100644 index 0bcdee93a..000000000 --- a/.github/workflows/run_fvt_tests.yaml +++ /dev/null @@ -1,255 +0,0 @@ -name: Functional Verification Tests - -concurrency: - group: fvt-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - master - - 'ci/**' - tags: - - v* - pull_request: - -jobs: - prepare: - runs-on: ubuntu-22.04 - # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11 - - steps: - - uses: actions/checkout@v3 - with: - path: source - fetch-depth: 0 - - name: get deps - run: | - make -C source deps-all - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip - - docker_test: - runs-on: ubuntu-22.04 - needs: prepare - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - - emqx-elixir - cluster_db_backend: - - mnesia - - rlog - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - - name: make docker image - working-directory: source - env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} - run: | - make ${{ matrix.profile }}-docker - - name: run emqx - timeout-minutes: 5 - working-directory: source - run: | - set -x - if [[ "${{ matrix.profile }}" = *-elixir ]] - then - export IS_ELIXIR=yes - PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g") - IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir - else - IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) - fi - ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - - name: make paho tests - run: | - if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then - echo "DUMP_CONTAINER_LOGS_BGN" - echo "============== haproxy ==============" - docker logs haproxy - echo "============== node1 ==============" - docker logs node1.emqx.io - echo "============== node2 ==============" - docker logs node2.emqx.io - echo "DUMP_CONTAINER_LOGS_END" - exit 1 - fi - # simple smoke test for node_dump - - name: test node_dump - run: | - docker exec node1.emqx.io node_dump - - helm_test: - runs-on: ubuntu-22.04 - needs: prepare - - strategy: - fail-fast: false - matrix: - discovery: - - k8s - - dns - profile: - - emqx - - emqx-enterprise - os: - - ["debian11", "debian:11-slim"] - builder: - - 5.1-3 - otp: - - 25.3.2-1 - elixir: - - 1.14.5 - arch: - - amd64 - # - emqx-enterprise # TODO test enterprise - - steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - - name: make docker image - working-directory: source - env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - EMQX_RUNNER: ${{ matrix.os[1] }} - run: | - make ${{ matrix.profile }}-docker - echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - - run: minikube start - - run: minikube image load $TARGET:$EMQX_TAG - - name: run emqx on chart - working-directory: source - if: matrix.discovery == 'k8s' - run: | - helm install ${{ matrix.profile }} \ - --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ - --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ - --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${{ matrix.profile }}-headless" \ - --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ - --set image.repository=$TARGET \ - --set image.pullPolicy=Never \ - --set image.tag=$EMQX_TAG \ - --set emqxAclConfig="" \ - --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ - --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ - --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${{ matrix.profile }} \ - --debug - - name: run emqx on chart - working-directory: source - if: matrix.discovery == 'dns' - run: | - helm install ${{ matrix.profile }} \ - --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ - --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ - --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${{ matrix.profile }}-headless.default.svc.cluster.local" \ - --set image.repository=$TARGET \ - --set image.pullPolicy=Never \ - --set image.tag=$EMQX_TAG \ - --set emqxAclConfig="" \ - --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ - --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ - --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${{ matrix.profile }} \ - --debug - - name: waiting emqx started - timeout-minutes: 10 - run: | - while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.replicas}')" \ - != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do - echo "=============================="; - kubectl get pods; - echo "=============================="; - echo "waiting emqx started"; - sleep 10; - done - - name: Get Token - timeout-minutes: 1 - run: | - kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null & - - while - [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] - do - echo "waiting emqx" - sleep 1 - done - - echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV - - - name: Check cluster - timeout-minutes: 10 - run: | - while - [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; - do - echo "waiting ${{ matrix.profile }} cluster scale" - sleep 1 - done - - uses: actions/checkout@v3 - with: - repository: emqx/paho.mqtt.testing - ref: develop-5.0 - path: paho.mqtt.testing - - name: install pytest - run: | - pip install pytest==7.1.2 pytest-retry - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run paho test - timeout-minutes: 10 - run: | - port_connected () { - local server="$1" - local port="$2" - echo > /dev/tcp/${server}/${port} 2>/dev/null - } - - kubectl port-forward service/${{ matrix.profile }} 1883:1883 > /dev/null & - - while ! port_connected localhost 1883; do - echo server not listening yet... - sleep 10 - done - - pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" - - if: failure() - run: kubectl logs -l "app.kubernetes.io/instance=${{ matrix.profile }}" -c emqx --tail=1000 diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml deleted file mode 100644 index 52082c56e..000000000 --- a/.github/workflows/run_gitlint.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Run gitlint - -on: [pull_request] - -jobs: - run_gitlint: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Run gitlint - shell: bash - run: | - set -ex - docker run --ulimit nofile=1024 -v $(pwd):/repo -w /repo ghcr.io/emqx/gitlint --commits ${{ github.event.pull_request.base.sha }}..$GITHUB_SHA --config .github/workflows/.gitlint diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml new file mode 100644 index 000000000..fbd45bc5b --- /dev/null +++ b/.github/workflows/run_helm_tests.yaml @@ -0,0 +1,160 @@ +name: Helm tests + +concurrency: + group: helm-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + version-emqx: + required: true + type: string + version-emqx-enterprise: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# pull_request: + +jobs: + helm_test: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + env: + EMQX_NAME: ${{ matrix.profile }} + EMQX_TAG: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} + REPOSITORY: "emqx/${{ matrix.profile }}" + + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + with: + path: source + - uses: actions/download-artifact@v3 + with: + name: "${{ env.EMQX_NAME }}-docker" + path: /tmp + - run: minikube start + - run: | + img="/tmp/${EMQX_NAME}-docker-${EMQX_TAG}.tar.gz" + if stderr=$(minikube image load "${img}" 2>&1 >/dev/null) && test -n "$stderr"; then + echo "${stderr}"; + exit 1; + fi + - name: run emqx on chart (k8s) + if: matrix.discovery == 'k8s' + working-directory: source + run: | + helm install ${EMQX_NAME} \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ + --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ + --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${EMQX_NAME}-headless" \ + --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ + --set image.repository=$REPOSITORY \ + --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ + --set emqxAclConfig="" \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ + --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ + deploy/charts/${EMQX_NAME} \ + --debug + - name: run emqx on chart (dns) + if: matrix.discovery == 'dns' + working-directory: source + run: | + helm install ${EMQX_NAME} \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${EMQX_NAME}-headless.default.svc.cluster.local" \ + --set image.repository=$REPOSITORY \ + --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ + --set emqxAclConfig="" \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ + --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ + deploy/charts/${EMQX_NAME} \ + --debug + - name: waiting emqx started + timeout-minutes: 5 + run: | + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + echo "=============================="; + kubectl get pods; + echo "=============================="; + echo "waiting emqx started"; + sleep 10; + done + - name: Get Token + timeout-minutes: 1 + run: | + kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null & + + while + [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] + do + echo "waiting emqx" + sleep 1 + done + + echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV + + - name: Check cluster + timeout-minutes: 10 + run: | + while + [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; + do + echo "waiting ${EMQX_NAME} cluster scale" + sleep 1 + done + - uses: actions/checkout@v3 + with: + repository: emqx/paho.mqtt.testing + ref: develop-5.0 + path: paho.mqtt.testing + - name: install pytest + run: | + pip install pytest==7.1.2 pytest-retry + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: run paho test + timeout-minutes: 10 + run: | + port_connected () { + local server="$1" + local port="$2" + echo > /dev/tcp/${server}/${port} 2>/dev/null + } + + kubectl port-forward service/${EMQX_NAME} 1883:1883 > /dev/null & + + while ! port_connected localhost 1883; do + echo server not listening yet... + sleep 10 + done + + pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" + - if: failure() + run: kubectl logs -l "app.kubernetes.io/instance=${EMQX_NAME}" -c emqx --tail=1000 diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 04866c7a9..7ca3363de 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -1,22 +1,23 @@ name: JMeter integration tests on: - push: - tags: - - "v5.*" - pull_request: - branches: - - "master" + workflow_call: + inputs: + version-emqx: + required: true + type: string +# on: +# push: +# tags: +# - "v5.*" +# pull_request: +# branches: +# - "master" jobs: - build_emqx_for_jmeter_tests: + jmeter_artifact: runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.build_docker.outputs.version}} steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - name: Cache Jmeter id: cache-jmeter uses: actions/cache@v3 @@ -42,21 +43,6 @@ jobs: with: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz - - uses: actions/checkout@v3 - - name: zip emqx docker image - id: build_docker - if: endsWith(github.repository, 'emqx') - run: | - ## TODO: make profile a matrix dimension - PROFILE='emqx' - make "${PROFILE}-docker" - VSN="$(./pkg-vsn.sh $PROFILE)" - echo "version=${VSN}" >> $GITHUB_OUTPUT - docker save -o emqx.tar emqx/emqx:${VSN} - - uses: actions/upload-artifact@v3 - with: - name: emqx.tar - path: ./emqx.tar advanced_feat: runs-on: ubuntu-22.04 @@ -70,69 +56,28 @@ jobs: - mqtt_topic_rewrite # - mqtt_retainer - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 + - name: show logs + if: failure() run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter + docker compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + logs - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -152,8 +97,6 @@ jobs: pgsql_authn_authz: runs-on: ubuntu-22.04 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} strategy: fail-fast: false @@ -168,72 +111,26 @@ jobs: - pgsql_authn - pgsql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 env: PGSQL_TAG: ${{ matrix.pgsql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/postgresql-42.2.18.jar https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -257,7 +154,7 @@ jobs: - name: dump docker compose logs if: failure() run: | - docker-compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log + docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log - uses: actions/upload-artifact@v3 if: always() with: @@ -277,73 +174,26 @@ jobs: - mysql_authn - mysql_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} PGSQL_TAG: ${{ matrix.mysql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql-tls) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -379,45 +229,19 @@ jobs: scripts_type: - jwt_authn - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - name: run jwks_server timeout-minutes: 10 run: | @@ -426,30 +250,9 @@ jobs: cd target docker run --name jwks_server --network emqx_bridge --ip 172.100.239.88 -d -v $(pwd)/jwkserver-0.0.1.jar:/jwks_server/jwkserver-0.0.1.jar --workdir /jwks_server openjdk:8-jdk bash \ -c "java -jar jwkserver-0.0.1.jar" - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ @@ -478,79 +281,30 @@ jobs: - built_in_database_authn - built_in_database_authz - needs: build_emqx_for_jmeter_tests + needs: jmeter_artifact steps: - - uses: erlef/setup-beam@v1.15.4 - with: - otp-version: 25.3.2 - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: ./.github/actions/prepare-jmeter with: - name: emqx.tar - path: /tmp - - name: load docker image - run: | - docker load < /tmp/emqx.tar + version-emqx: ${{ inputs.version-emqx }} - name: docker compose up timeout-minutes: 5 - env: - _EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }} - PGSQL_TAG: ${{ matrix.mysql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - up -d --build - - name: wait docker compose up - timeout-minutes: 5 - run: | - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx"; - sleep 5; - done - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 1 - done - docker ps -a + up --wait --build echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV - - uses: actions/checkout@v3 - with: - repository: emqx/emqx-fvt - ref: broker-autotest-v5 - path: scripts - - uses: actions/setup-java@v3 - with: - java-version: '8.0.282' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md - distribution: 'zulu' - - uses: actions/download-artifact@v3 - with: - name: apache-jmeter.tgz - path: /tmp - - name: install jmeter - timeout-minutes: 10 - env: - JMETER_VERSION: 5.4.3 - run: | - cd /tmp && tar -xvf apache-jmeter.tgz - echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar - ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run jmeter run: | - /opt/jmeter/bin/jmeter.sh \ + jmeter/bin/jmeter.sh \ -Jjmeter.save.saveservice.output_format=xml -n \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -Demqx_ip=$HAPROXY_IP \ - -l jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl \ - -j jmeter_logs/logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.log + -l jmeter_logs/${{ matrix.scripts_type }}.jtl \ + -j jmeter_logs/logs/${{ matrix.scripts_type }}.log - name: check logs run: | - if cat jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl | grep -e 'true' > /dev/null 2>&1; then + if cat jmeter_logs/${{ matrix.scripts_type }}.jtl | grep -e 'true' > /dev/null 2>&1; then echo "check logs failed" exit 1 fi @@ -559,11 +313,3 @@ jobs: with: name: jmeter_logs path: ./jmeter_logs - - delete-artifact: - runs-on: ubuntu-22.04 - needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz] - steps: - - uses: geekyeggo/delete-artifact@v2 - with: - name: emqx.tar diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 0400d0502..166fca513 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -4,6 +4,15 @@ concurrency: group: relup-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string # on: # push: # branches: @@ -14,8 +23,8 @@ concurrency: jobs: relup_test_plan: - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} @@ -23,16 +32,18 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@v3 - name: Checkout + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 with: - path: emqx - fetch-depth: 0 + name: emqx-enterprise + - name: extract artifact + run: | + unzip -o -q emqx-enterprise.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Find versions id: find-versions run: | set -x - cd emqx ee_vsn="$(./pkg-vsn.sh enterprise)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') @@ -40,8 +51,6 @@ jobs: echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT - name: build emqx run: | - set -x - cd emqx export PROFILE='emqx-enterprise' make emqx-enterprise-tgz - uses: actions/upload-artifact@v3 @@ -49,10 +58,10 @@ jobs: with: name: emqx_built path: | - emqx/_upgrade_base - emqx/_packages - emqx/scripts - emqx/.ci + _upgrade_base + _packages + scripts + .ci relup_test_run: needs: @@ -70,8 +79,7 @@ jobs: run: shell: bash steps: - # setup Erlang to run lux - - uses: erlef/setup-beam@v1.15.4 + - uses: erlef/setup-beam@v1.16.0 with: otp-version: 25.3.2 - uses: actions/checkout@v3 @@ -81,7 +89,7 @@ jobs: path: lux - name: Install lux run: | - set -e -u -x + set -eu cd lux autoconf ./configure @@ -94,10 +102,7 @@ jobs: path: . - name: run relup test run: | - set -e -x -u - chmod a+x scripts/**/*.sh - ls -l scripts - ls -l scripts/relup-test + set -eux case "$OLD_VSN" in e*) export CUR_VSN="$CUR_EE_VSN" diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d43079d61..e2d2a0abf 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -1,156 +1,48 @@ name: Run test case concurrency: - group: test-${{ github.event_name }}-${{ github.ref }} + group: test-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: - push: - branches: - - master - - 'ci/**' - tags: - - v* - - e* - pull_request: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + ct-matrix: + required: true + type: string + ct-host: + required: true + type: string + ct-docker: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# - e* +# pull_request: env: IS_CI: "yes" jobs: - build-matrix: - runs-on: ubuntu-22.04 - outputs: - prepare: ${{ steps.matrix.outputs.prepare }} - host: ${{ steps.matrix.outputs.host }} - docker: ${{ steps.matrix.outputs.docker }} - runs-on: ${{ steps.runner.outputs.runs-on }} - steps: - - uses: actions/checkout@v3 - - name: Build matrix - id: matrix - run: | - APPS="$(./scripts/find-apps.sh --ci)" - MATRIX="$(echo "${APPS}" | jq -c ' - [ - (.[] | select(.profile == "emqx") | . + { - builder: "5.1-3", - otp: "25.3.2-1", - elixir: "1.14.5" - }), - (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.1-3", - otp: ["25.3.2-1"][], - elixir: "1.14.5" - }) - ] - ')" - echo "${MATRIX}" | jq - MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" - MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" - MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" - echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT - echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT - echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT - - name: Choose runner host - id: runner - run: | - RUNS_ON="ubuntu-22.04" - ${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64" - echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT - - prepare: - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - needs: [build-matrix] - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 - with: - path: source - - name: get_all_deps - working-directory: source - env: - PROFILE: ${{ matrix.profile }} - run: | - make ensure-rebar3 - # fetch all deps and compile - make ${{ matrix.profile }}-compile - make test-compile - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: source.zip - - check_examples: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip - - name: check example config files - env: - PROFILE: ${{ matrix.profile }} - working-directory: source - run: ./scripts/test/check-example-configs.sh - - static_checks: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip - - uses: actions/cache@v3 - with: - path: "source/emqx_dialyzer_${{ matrix.otp }}_plt" - key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }} - restore-keys: | - rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- - - name: run static checks - env: - PROFILE: ${{ matrix.profile }} - working-directory: source - run: make static_checks - eunit_and_proper: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} + include: ${{ fromJson(inputs.ct-matrix) }} defaults: run: @@ -161,16 +53,16 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -o -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces eunit.coverdata - name: eunit env: PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - working-directory: source run: make eunit # produces proper.coverdata @@ -178,23 +70,19 @@ jobs: env: PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - working-directory: source run: make proper - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover ct_docker: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.docker) }} + include: ${{ fromJson(inputs.ct-docker) }} defaults: run: @@ -204,14 +92,14 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests - working-directory: source env: DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" MONGO_TAG: "5" @@ -229,22 +117,19 @@ jobs: - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: source/_build/test/logs + path: _build/test/logs ct: - needs: - - build-matrix - - prepare - runs-on: ${{ needs.build-matrix.outputs.runs-on }} + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.build-matrix.outputs.host) }} + include: ${{ fromJson(inputs.ct-host) }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" defaults: @@ -255,14 +140,14 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }}-${{ matrix.otp }} - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" # produces $PROFILE---sg.coverdata - name: run common tests - working-directory: source env: PROFILE: ${{ matrix.profile }} SUITEGROUP: ${{ matrix.suitegroup }} @@ -272,58 +157,61 @@ jobs: - uses: actions/upload-artifact@v3 with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover if-no-files-found: warn # do not fail if no coverdata found - uses: actions/upload-artifact@v3 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} - path: source/_build/test/logs + path: _build/test/logs make_cover: needs: - eunit_and_proper - ct - ct_docker - runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" + runs-on: ${{ inputs.runner }} + container: ${{ inputs.builder }} + strategy: + fail-fast: false + matrix: + profile: + - emqx-enterprise steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise-25.3.2-1 - path: . - - name: unzip source code - run: unzip -q source.zip + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/download-artifact@v3 name: download coverdata with: name: coverdata - path: source/_build/test/cover + path: _build/test/cover - name: make cover - working-directory: source env: PROFILE: emqx-enterprise run: make cover - name: send to coveralls - working-directory: source env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PROFILE: emqx-enterprise run: make coveralls - name: get coveralls logs - working-directory: source if: failure() run: cat rebar3.crashdump # do this in a separate job upload_coverdata: needs: make_cover - runs-on: ubuntu-22.04 + runs-on: ${{ inputs.runner }} steps: - name: Coveralls Finished env: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml deleted file mode 100644 index 7f29572b9..000000000 --- a/.github/workflows/shellcheck.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Shellcheck - -on: - pull_request: - -jobs: - shellcheck: - runs-on: ubuntu-22.04 - steps: - - name: Checkout source code - uses: actions/checkout@v3 - - name: Install shellcheck - run: | - sudo apt-get update - sudo apt install shellcheck - - name: Run shellcheck - run: | - ./scripts/shellcheck.sh - echo "success" diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml new file mode 100644 index 000000000..07308e1a6 --- /dev/null +++ b/.github/workflows/spellcheck.yaml @@ -0,0 +1,46 @@ +name: Build slim packages + +concurrency: + group: spellcheck-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string +# on: +# push: +# branches: +# - master +# - release-51 +# pull_request: +# # GitHub pull_request action is by default triggered when +# # opened reopened or synchronize, +# # we add labeled and unlabeled to the list because +# # the mac job dpends on the PR having a 'Mac' label +# types: +# - labeled +# - unlabeled +# - opened +# - reopened +# - synchronize +# workflow_dispatch: + +jobs: + spellcheck: + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/download-artifact@v3 + with: + name: "${{ matrix.profile }}_schema_dump" + path: /tmp/ + - name: Run spellcheck + run: | + bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml new file mode 100644 index 000000000..b7fc2444b --- /dev/null +++ b/.github/workflows/static_checks.yaml @@ -0,0 +1,59 @@ +name: Static checks + +concurrency: + group: static-checks-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + ct-matrix: + required: true + type: string + +# on: +# push: +# branches: +# - master +# - 'ci/**' +# tags: +# - v* +# - e* +# pull_request: + +env: + IS_CI: "yes" + +jobs: + static_checks: + runs-on: ${{ inputs.runner }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.ct-matrix) }} + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.profile }} + - name: extract artifact + run: | + unzip -o -q ${{ matrix.profile }}.zip + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/cache@v3 + with: + path: "emqx_dialyzer_${{ matrix.otp }}_plt" + key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }} + restore-keys: | + rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- + - name: run static checks + env: + PROFILE: ${{ matrix.profile }} + run: make static_checks diff --git a/build b/build index 1ca324120..fb0d213f2 100755 --- a/build +++ b/build @@ -361,10 +361,10 @@ make_tgz() { log "Archive sha256sum: $(cat "${target}.sha256")" } -trap docker_cleanup EXIT - docker_cleanup() { rm -f ./.dockerignore >/dev/null + # shellcheck disable=SC2015 + [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true } ## Build the default docker image based on debian 11. @@ -384,7 +384,14 @@ make_docker() { if [[ "$PROFILE" = *enterprise* ]]; then extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' fi - echo '_build' >> ./.dockerignore + # shellcheck disable=SC2015 + [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true + trap docker_cleanup EXIT + { + echo '/_build' + echo '/deps' + echo '/*.lock' + } >> ./.dockerignore set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ @@ -394,6 +401,7 @@ make_docker() { --tag "${EMQX_IMAGE_TAG}" \ -f "${EMQX_DOCKERFILE}" . [[ "${DEBUG:-}" -eq 1 ]] || set +x + echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag } function join { From dc8487ef7154cd8c7bdc1409cc1d3e1b36ca4683 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 3 Aug 2023 08:56:06 +0200 Subject: [PATCH 02/22] ci: delete elixir_release workflow as it is covered by slim_packages --- .github/workflows/_pr_entrypoint.yaml | 9 ----- .github/workflows/elixir_release.yaml | 47 --------------------------- 2 files changed, 56 deletions(-) delete mode 100644 .github/workflows/elixir_release.yaml diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 1f1920632..4b30304e6 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -157,15 +157,6 @@ jobs: runner: ${{ needs.sanity-checks.outputs.runner }} builder: ${{ needs.sanity-checks.outputs.builder }} - elixir_release: - needs: - - sanity-checks - - compile - uses: ./.github/workflows/elixir_release.yaml - with: - runner: ${{ needs.sanity-checks.outputs.runner }} - builder: ${{ needs.sanity-checks.outputs.builder }} - run_jmeter_tests: needs: - sanity-checks diff --git a/.github/workflows/elixir_release.yaml b/.github/workflows/elixir_release.yaml deleted file mode 100644 index 06b26fc8e..000000000 --- a/.github/workflows/elixir_release.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# FIXME: temporary workflow for testing; remove later -name: Elixir Build (temporary) - -concurrency: - group: elixir-release-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_call: - inputs: - runner: - required: true - type: string - builder: - required: true - type: string - -jobs: - elixir_release_build: - runs-on: ${{ inputs.runner }} - container: ${{ inputs.builder }} - strategy: - matrix: - profile: - - emqx - - steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.profile }} - - name: extract artifact - run: | - unzip -o -q ${{ matrix.profile }}.zip - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: elixir release - run: make ${{ matrix.profile }}-elixir - - name: start release - run: | - _build/${{ matrix.profile }}/rel/emqx/bin/emqx start - - name: check if started - run: | - set -e - sleep 10 - timeout 1 bash -c ' Date: Thu, 3 Aug 2023 08:56:43 +0200 Subject: [PATCH 03/22] ci: move windows build from slim_packages to scheduled daily build --- .github/workflows/build_packages_cron.yaml | 61 +++++++++++++++++++++- .github/workflows/build_slim_packages.yaml | 47 ----------------- 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 09f68c256..b245078da 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -29,7 +29,7 @@ jobs: arch: - amd64 os: - - debian10 + - ubuntu22.04 - amzn2023 builder: - 5.1-3 @@ -94,7 +94,7 @@ jobs: otp: - 25.3.2-1 os: - - macos-12 + - macos-13 - macos-12-arm64 steps: @@ -117,6 +117,7 @@ jobs: with: name: ${{ matrix.profile }} path: _packages/${{ matrix.profile }}/ + retention-days: 7 - name: Send notification to Slack uses: slackapi/slack-github-action@v1.23.0 if: failure() @@ -125,3 +126,59 @@ jobs: with: payload: | {"text": "Scheduled build of ${{ matrix.profile }} package for ${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + + windows: + if: github.repository_owner == 'emqx' + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + profile: + - emqx + otp: + - 25.3.2 + steps: + - uses: actions/checkout@v3 + - uses: ilammy/msvc-dev-cmd@v1.12.0 + - uses: erlef/setup-beam@v1.16.0 + with: + otp-version: ${{ matrix.otp }} + - name: build + env: + PYTHON: python + DIAGNOSTIC: 1 + run: | + # ensure crypto app (openssl) + erl -eval "erlang:display(crypto:info_lib())" -s init stop + make ${{ matrix.profile }}-tgz + - name: run emqx + timeout-minutes: 5 + run: | + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + Start-Sleep -s 10 + $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping + if ($pingOutput = 'pong') { + echo "EMQX started OK" + } else { + echo "Failed to ping EMQX $pingOutput" + Exit 1 + } + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EMQX installed" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EMQX uninstalled" + - uses: actions/upload-artifact@v3 + with: + name: windows + path: _packages/${{ matrix.profile }}/* + retention-days: 7 + - name: Send notification to Slack + uses: slackapi/slack-github-action@v1.23.0 + if: failure() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + payload: | + {"text": "Scheduled build of ${{ matrix.profile }} package for Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index c5430439b..a1de84728 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -97,53 +97,6 @@ jobs: _build/docgen/${{ matrix.profile[0] }}/schema-en.json retention-days: 7 - windows: - runs-on: windows-2019 - strategy: - fail-fast: false - matrix: - profile: - - emqx - otp: - - 25.3.2 - steps: - - uses: actions/checkout@v3 - - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.16.0 - with: - otp-version: ${{ matrix.otp }} - - name: build - env: - PYTHON: python - DIAGNOSTIC: 1 - run: | - # ensure crypto app (openssl) - erl -eval "erlang:display(crypto:info_lib())" -s init stop - make ${{ matrix.profile }}-tgz - - name: run emqx - timeout-minutes: 5 - run: | - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start - Start-Sleep -s 10 - $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping - if ($pingOutput = 'pong') { - echo "EMQX started OK" - } else { - echo "Failed to ping EMQX $pingOutput" - Exit 1 - } - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop - echo "EMQX stopped" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install - echo "EMQX installed" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - echo "EMQX uninstalled" - - uses: actions/upload-artifact@v3 - with: - name: windows - path: _packages/${{ matrix.profile }}/* - retention-days: 7 - mac: strategy: fail-fast: false From a32b8fd21fcc7e827abd0e963ed9406f0ad36d21 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 3 Aug 2023 17:48:59 +0200 Subject: [PATCH 04/22] ci: use single entry point for push event --- .github/actions/pr-sanity-checks/action.yaml | 8 +- .github/workflows/_pr_entrypoint.yaml | 19 +- .github/workflows/_push-entrypoint.yaml | 179 +++++++++++++++++ .../build_and_push_docker_images.yaml | 188 +++++++----------- .github/workflows/build_docker_for_test.yaml | 17 -- .github/workflows/build_packages.yaml | 144 ++++++-------- .github/workflows/build_slim_packages.yaml | 40 ++-- .github/workflows/run_conf_tests.yaml | 10 - .github/workflows/run_docker_tests.yaml | 9 - .github/workflows/run_helm_tests.yaml | 9 - .github/workflows/run_jmeter_tests.yaml | 7 - .github/workflows/run_relup_tests.yaml | 7 - .github/workflows/run_test_cases.yaml | 10 - .github/workflows/spellcheck.yaml | 17 -- .github/workflows/static_checks.yaml | 10 - scripts/parse-git-ref.sh | 47 +++++ {.ci => scripts}/pr-sanity-checks.sh | 2 +- 17 files changed, 402 insertions(+), 321 deletions(-) create mode 100644 .github/workflows/_push-entrypoint.yaml create mode 100755 scripts/parse-git-ref.sh rename {.ci => scripts}/pr-sanity-checks.sh (95%) diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml index 60976e27e..3114f8a94 100644 --- a/.github/actions/pr-sanity-checks/action.yaml +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -7,10 +7,6 @@ defaults: runs: using: composite steps: - - name: Work around https://github.com/actions/checkout/issues/766 - shell: bash - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Run gitlint shell: bash env: @@ -22,7 +18,7 @@ runs: - name: Run shellcheck shell: bash run: | - apt-get update -y && apt-get install -y shellcheck + DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shellcheck ./scripts/shellcheck.sh # https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions - name: Check workflow files @@ -45,9 +41,7 @@ runs: - name: Check Erlang code formatting shell: bash run: | - echo "==> ./scripts/check-format.sh" ./scripts/check-format.sh - echo "==> check-format ok" - name: Run elvis check shell: bash run: | diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 4b30304e6..67cedcfb8 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -1,13 +1,18 @@ name: PR Entrypoint +concurrency: + group: pr-entrypoint-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: + workflow_dispatch: + inputs: + ref: + required: false env: IS_CI: "yes" - OTP_VSN: "25.3.2-1" - ELIXIR_VSN: "1.14.5" - BUILDER_VSN: "25.3.2-1" jobs: sanity-checks: @@ -28,7 +33,11 @@ jobs: steps: - uses: actions/checkout@v3 with: + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: ./.github/actions/pr-sanity-checks - name: Build matrix id: matrix @@ -50,10 +59,10 @@ jobs: ')" echo "${MATRIX}" | jq CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" - CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT - echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT + echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml new file mode 100644 index 000000000..af014e74d --- /dev/null +++ b/.github/workflows/_push-entrypoint.yaml @@ -0,0 +1,179 @@ +name: Push Entrypoint + +concurrency: + group: push-entrypoint-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +on: + push: + tags: + - 'v*' + - 'e*' + branches: + - 'master' + - 'release-51' + - 'ci/**' + +env: + IS_CI: 'yes' + +jobs: + prepare: + runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + container: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04' + outputs: + profile: ${{ steps.parse-git-ref.outputs.profile }} + edition: ${{ steps.parse-git-ref.outputs.edition }} + release: ${{ steps.parse-git-ref.outputs.release }} + latest: ${{ steps.parse-git-ref.outputs.latest }} + version: ${{ steps.parse-git-ref.outputs.version }} + ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} + ct-host: ${{ steps.matrix.outputs.ct-host }} + ct-docker: ${{ steps.matrix.outputs.ct-docker }} + runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }} + builder: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04' + builder_vsn: '5.1-3' + otp_vsn: '25.3.2-1' + elixir_vsn: '1.14.5' + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.ref }} + fetch-depth: 0 + - name: Work around https://github.com/actions/checkout/issues/766 + shell: bash + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Detect emqx profile and version + id: parse-git-ref + run: | + JSON="$(./scripts/parse-git-ref.sh $GITHUB_REF)" + PROFILE=$(echo "$JSON" | jq -cr '.profile') + EDITION=$(echo "$JSON" | jq -cr '.edition') + RELEASE=$(echo "$JSON" | jq -cr '.release') + LATEST=$(echo "$JSON" | jq -cr '.latest') + VERSION="$(./pkg-vsn.sh "$PROFILE")" + echo "profile=$PROFILE" | tee -a $GITHUB_OUTPUT + echo "edition=$EDITION" | tee -a $GITHUB_OUTPUT + echo "release=$RELEASE" | tee -a $GITHUB_OUTPUT + echo "latest=$LATEST" | tee -a $GITHUB_OUTPUT + echo "version=$VERSION" | tee -a $GITHUB_OUTPUT + - name: Build matrix + id: matrix + run: | + APPS="$(./scripts/find-apps.sh --ci)" + MATRIX="$(echo "${APPS}" | jq -c ' + [ + (.[] | select(.profile == "emqx") | . + { + builder: "5.1-3", + otp: "25.3.2-1", + elixir: "1.14.5" + }), + (.[] | select(.profile == "emqx-enterprise") | . + { + builder: "5.1-3", + otp: ["25.3.2-1"][], + elixir: "1.14.5" + }) + ] + ')" + echo "${MATRIX}" | jq + CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT + echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT + echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT + + build_packages: + if: ${{ needs.prepare.outputs.release == 'true' }} + needs: + - prepare + uses: ./.github/workflows/build_packages.yaml + with: + profile: ${{ needs.prepare.outputs.profile }} + publish: ${{ needs.prepare.outputs.release }} + otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} + elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} + runner: ${{ needs.prepare.outputs.runner }} + builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + + build_and_push_docker_images: + if: ${{ needs.prepare.outputs.release == 'true' }} + needs: + - prepare + uses: ./.github/workflows/build_and_push_docker_images.yaml + with: + profile: ${{ needs.prepare.outputs.profile }} + edition: ${{ needs.prepare.outputs.edition }} + version: ${{ needs.prepare.outputs.version }} + latest: ${{ needs.prepare.outputs.latest }} + publish: ${{ needs.prepare.outputs.release }} + otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} + elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} + runner: ${{ needs.prepare.outputs.runner }} + builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + + compile: + runs-on: ${{ needs.prepare.outputs.runner }} + container: ${{ needs.prepare.outputs.builder }} + needs: + - prepare + strategy: + matrix: + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.ref }} + fetch-depth: 0 + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - id: compile + env: + PROFILE: ${{ matrix.profile }} + ENABLE_COVER_COMPILE: 1 + run: | + make $PROFILE + zip -ryq $PROFILE.zip . + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.profile }} + path: ${{ matrix.profile }}.zip + retention-days: 1 + + run_test_cases: + needs: + - prepare + - compile + uses: ./.github/workflows/run_test_cases.yaml + with: + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} + ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} + ct-host: ${{ needs.prepare.outputs.ct-host }} + ct-docker: ${{ needs.prepare.outputs.ct-docker }} + + run_conf_tests: + needs: + - prepare + - compile + uses: ./.github/workflows/run_conf_tests.yaml + with: + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} + + static_checks: + needs: + - prepare + - compile + uses: ./.github/workflows/static_checks.yaml + with: + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} + ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} + diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index ddcf2c6ac..63225eec3 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -1,144 +1,106 @@ name: Build and push docker images concurrency: - group: docker-build-${{ github.event_name }}-${{ github.ref }} + group: docker-build-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: workflow_call: -# on: -# push: -# tags: -# - v* -# - e* -# - docker-latest-* + inputs: + profile: + required: true + type: string + edition: + required: true + type: string + version: + required: true + type: string + latest: + required: true + type: string + publish: + required: true + type: string + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string + runner: + required: true + type: string + builder_vsn: + required: true + type: string workflow_dispatch: inputs: - branch_or_tag: + ref: required: false + version: + required: true + type: string profile: required: false + type: string default: 'emqx' - is_latest: + edition: required: false + type: string + default: 'Opensource' + latest: + required: false + type: boolean default: false + publish: + required: false + type: boolean + default: false + otp_vsn: + required: false + type: string + default: '25.3.2-1' + elixir_vsn: + required: false + type: string + default: '1.14.5' + runner: + required: false + type: string + default: 'ubuntu-22.04' + builder_vsn: + required: false + type: string + default: '5.1-3' jobs: - prepare: - runs-on: ubuntu-22.04 - # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" - - outputs: - PROFILE: ${{ steps.get_profile.outputs.PROFILE }} - EDITION: ${{ steps.get_profile.outputs.EDITION }} - IS_LATEST: ${{ steps.get_profile.outputs.IS_LATEST }} - IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} - VERSION: ${{ steps.get_profile.outputs.VERSION }} - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used - path: source - fetch-depth: 0 - - - name: Get profiles to build - id: get_profile - env: - INPUTS_PROFILE: ${{ github.event.inputs.profile }} - run: | - cd source - # tag docker-latest-ce or docker-latest-ee - if git describe --tags --exact --match 'docker-latest-*' 2>/dev/null; then - echo 'is_latest=true due to docker-latest-* tag' - is_latest=true - elif [ "${{ inputs.is_latest }}" = "true" ]; then - echo 'is_latest=true due to manual input from workflow_dispatch' - is_latest=true - else - echo 'is_latest=false' - is_latest=false - fi - # resolve profile - if git describe --tags --match "v*" --exact; then - echo "This is an exact git tag, will publish images" - is_exact='true' - PROFILE=emqx - elif git describe --tags --match "e*" --exact; then - echo "This is an exact git tag, will publish images" - is_exact='true' - PROFILE=emqx-enterprise - else - echo "This is NOT an exact git tag, will not publish images" - is_exact='false' - fi - - case "${PROFILE:-$INPUTS_PROFILE}" in - emqx) - EDITION='Opensource' - ;; - emqx-enterprise) - EDITION='Enterprise' - ;; - *) - echo "ERROR: Failed to resolve build profile" - exit 1 - ;; - esac - - VSN="$(./pkg-vsn.sh "$PROFILE")" - echo "Building emqx/$PROFILE:$VSN image (latest=$is_latest)" - echo "Push = $is_exact" - echo "IS_LATEST=$is_latest" >> $GITHUB_OUTPUT - echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT - echo "PROFILE=$PROFILE" >> $GITHUB_OUTPUT - echo "EDITION=$EDITION" >> $GITHUB_OUTPUT - echo "VERSION=$VSN" >> $GITHUB_OUTPUT - - name: get_all_deps - env: - PROFILE: ${{ steps.get_profile.outputs.PROFILE }} - run: | - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip - docker: - runs-on: ubuntu-22.04 - needs: prepare + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: profile: - - "${{ needs.prepare.outputs.PROFILE }}" + - ${{ inputs.profile }} registry: - 'docker.io' - 'public.ecr.aws' os: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - # NOTE: 'otp' and 'elixir' are to configure emqx-builder image - # only support latest otp and elixir, not a matrix builder: - - 5.1-3 # update to latest + - ${{ inputs.builder_vsn }} otp: - - 25.3.2-1 + - ${{ inputs.otp_vsn }} elixir: - 'no_elixir' - - '1.14.5' # update to latest - exclude: # TODO: publish enterprise to ecr too? - - registry: 'public.ecr.aws' - profile: emqx-enterprise + - ${{ inputs.elixir_vsn }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip + ref: ${{ github.event.inputs.ref }} + fetch-depth: 0 - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 @@ -187,18 +149,18 @@ jobs: latest=${{ matrix.elixir == 'no_elixir' }} suffix=${{ steps.pre-meta.outputs.img_suffix }} tags: | - type=semver,pattern={{major}}.{{minor}},value=${{ needs.prepare.outputs.VERSION }} - type=semver,pattern={{version}},value=${{ needs.prepare.outputs.VERSION }} - type=raw,value=${{ needs.prepare.outputs.VERSION }} - type=raw,value=latest,enable=${{ needs.prepare.outputs.IS_LATEST }} + type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }} + type=semver,pattern={{version}},value=${{ inputs.version }} + type=raw,value=${{ inputs.version }} + type=raw,value=latest,enable=${{ inputs.latest }} labels: | org.opencontainers.image.otp.version=${{ matrix.otp }} - org.opencontainers.image.edition=${{ needs.prepare.outputs.EDITION }} + org.opencontainers.image.edition=${{ inputs.edition }} ${{ steps.pre-meta.outputs.extra_labels }} - uses: docker/build-push-action@v3 with: - push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }} + push: ${{ inputs.publish == 'true' || github.repository_owner != 'emqx' }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 @@ -208,4 +170,4 @@ jobs: EMQX_NAME=${{ matrix.profile }}${{ steps.pre-meta.outputs.img_suffix }} EXTRA_DEPS=${{ steps.pre-meta.outputs.extra_deps }} file: source/${{ matrix.os[2] }} - context: source + diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index 756383d7f..e43a651f7 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -19,23 +19,6 @@ on: version-emqx-enterprise: required: true type: string -# on: -# push: -# branches: -# - master -# - release-51 -# pull_request: -# # GitHub pull_request action is by default triggered when -# # opened reopened or synchronize, -# # we add labeled and unlabeled to the list because -# # the mac job dpends on the PR having a 'Mac' label -# types: -# - labeled -# - unlabeled -# - opened -# - reopened -# - synchronize -# workflow_dispatch: jobs: docker: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c65b05677..20b64038d 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,83 +1,61 @@ name: Cross build packages concurrency: - group: build-${{ github.event_name }}-${{ github.ref }} + group: build-packages-${{ github.event_name }}-${{ github.sha }} cancel-in-progress: true on: workflow_call: -# on: -# push: -# branches: -# - 'ci/**' -# tags: -# - v* -# - e* + inputs: + profile: + required: true + type: string + publish: + required: true + type: string + otp_vsn: + required: true + type: string + elixir_vsn: + required: true + type: string + runner: + required: true + type: string + builder_vsn: + required: true + type: string workflow_dispatch: inputs: - branch_or_tag: + ref: required: false profile: required: false + publish: + required: false + type: boolean + default: false + otp_vsn: + required: false + type: string + default: '25.3.2-1' + elixir_vsn: + required: false + type: string + default: '1.14.5' + runner: + required: false + type: string + default: 'ubuntu-22.04' + builder_vsn: + required: false + type: string + default: '5.1-3' jobs: - prepare: - runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 - outputs: - BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} - IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} - VERSION: ${{ steps.get_profile.outputs.VERSION }} - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used - fetch-depth: 0 - - - name: Get profile to build - id: get_profile - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - tag=${{ github.ref }} - if git describe --tags --match "[v|e]*" --exact; then - echo "WARN: This is an exact git tag, will publish release" - is_exact_tag='true' - else - echo "WARN: This is NOT an exact git tag, will not publish release" - is_exact_tag='false' - fi - echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT - case $tag in - refs/tags/v*) - PROFILE='emqx' - ;; - refs/tags/e*) - PROFILE=emqx-enterprise - ;; - *) - PROFILE=${{ github.event.inputs.profile }} - case "$PROFILE" in - emqx) - true - ;; - emqx-enterprise) - true - ;; - *) - # maybe triggered from schedule - echo "WARN: \"$PROFILE\" is not a valid profile." - echo "building the default profile 'emqx' instead" - PROFILE='emqx' - ;; - esac - ;; - esac - echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT - echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT - windows: runs-on: windows-2019 - if: startsWith(github.ref_name, 'v') + if: inputs.profile == 'emqx' strategy: fail-fast: false matrix: @@ -86,7 +64,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.branch_or_tag }} + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - uses: ilammy/msvc-dev-cmd@v1.12.0 @@ -127,14 +105,13 @@ jobs: path: _packages/${{ matrix.profile }}/ mac: - needs: prepare strategy: fail-fast: false matrix: profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} + - ${{ inputs.profile }} otp: - - 25.3.2-1 + - ${{ inputs.otp_vsn }} os: - macos-11 - macos-12 @@ -144,7 +121,7 @@ jobs: - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.branch_or_tag }} + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - uses: ./.github/actions/package-macos with: @@ -162,7 +139,6 @@ jobs: path: _packages/${{ matrix.profile }}/ linux: - needs: prepare runs-on: ${{ matrix.build_machine }} # always run in builder container because the host might have the wrong OTP version etc. # otherwise buildx.sh does not run docker if arch and os matches the target arch and os. @@ -173,9 +149,9 @@ jobs: fail-fast: false matrix: profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} + - ${{ inputs.profile }} otp: - - 25.3.2-1 + - ${{ inputs.otp_vsn }} arch: - amd64 - arm64 @@ -195,9 +171,9 @@ jobs: - aws-arm64 - aws-amd64 builder: - - 5.1-3 + - ${{ inputs.builder_vsn }} elixir: - - 1.14.5 + - ${{ inputs.elixir_vsn }} with_elixir: - 'no' exclude: @@ -207,12 +183,12 @@ jobs: build_machine: aws-arm64 include: - profile: emqx - otp: 25.3.2-1 + otp: ${{ inputs.otp_vsn }} arch: amd64 os: ubuntu22.04 build_machine: aws-amd64 - builder: 5.1-3 - elixir: 1.14.5 + builder: ${{ inputs.builder_vsn }} + elixir: ${{ inputs.elixir_vsn }} with_elixir: 'yes' defaults: @@ -224,7 +200,7 @@ jobs: - uses: actions/checkout@v3 with: - ref: ${{ github.event.inputs.branch_or_tag }} + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - name: fix workdir @@ -269,14 +245,16 @@ jobs: path: _packages/${{ matrix.profile }}/ publish_artifacts: - runs-on: ubuntu-22.04 - needs: [prepare, mac, linux] - if: needs.prepare.outputs.IS_EXACT_TAG == 'true' + runs-on: ${{ inputs.runner }} + needs: + - mac + - linux + if: ${{ inputs.publish == 'true' }} strategy: fail-fast: false matrix: profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} + - ${{ inputs.profile }} steps: - uses: actions/download-artifact@v3 with: @@ -286,7 +264,7 @@ jobs: run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | - set -e -u + set -eu cd packages/${{ matrix.profile }} # fix the .sha256 file format for var in $(ls | grep emqx | grep -v sha256); do diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a1de84728..483ab88ec 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -22,23 +22,31 @@ on: elixir_vsn: required: true type: string -# on: -# push: -# branches: -# - master -# - release-51 -# pull_request: -# # GitHub pull_request action is by default triggered when -# # opened reopened or synchronize, -# # we add labeled and unlabeled to the list because -# # the mac job dpends on the PR having a 'Mac' label -# types: -# - labeled -# - unlabeled -# - opened -# - reopened -# - synchronize + workflow_dispatch: + inputs: + ref: + required: false + runner: + required: false + type: string + default: 'ubuntu-22.04' + builder: + required: false + type: string + default: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04' + builder_vsn: + required: false + type: string + default: '5.1-3' + otp_vsn: + required: false + type: string + default: '25.3.2-1' + elixir_vsn: + required: false + type: string + default: '1.14.5' jobs: linux: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 8b2388c79..5ffa24946 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -14,16 +14,6 @@ on: required: true type: string -# on: -# push: -# branches: -# - master -# - 'ci/**' -# tags: -# - v* -# - e* -# pull_request: - jobs: run_conf_tests: runs-on: ${{ inputs.runner }} diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 774dd4690..4188378d0 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -17,15 +17,6 @@ on: required: true type: string -# on: -# push: -# branches: -# - master -# - 'ci/**' -# tags: -# - v* -# pull_request: - jobs: basic-tests: runs-on: ${{ inputs.runner }} diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index fbd45bc5b..85e748da4 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -14,15 +14,6 @@ on: required: true type: string -# on: -# push: -# branches: -# - master -# - 'ci/**' -# tags: -# - v* -# pull_request: - jobs: helm_test: runs-on: ubuntu-22.04 diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 7ca3363de..d45b66324 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -6,13 +6,6 @@ on: version-emqx: required: true type: string -# on: -# push: -# tags: -# - "v5.*" -# pull_request: -# branches: -# - "master" jobs: jmeter_artifact: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 166fca513..3958d5b00 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -13,13 +13,6 @@ on: builder: required: true type: string -# on: -# push: -# branches: -# - '**' -# tags: -# - e* -# pull_request: jobs: relup_test_plan: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e2d2a0abf..325a9ec9c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -23,16 +23,6 @@ on: required: true type: string -# on: -# push: -# branches: -# - master -# - 'ci/**' -# tags: -# - v* -# - e* -# pull_request: - env: IS_CI: "yes" diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml index 07308e1a6..440233705 100644 --- a/.github/workflows/spellcheck.yaml +++ b/.github/workflows/spellcheck.yaml @@ -10,23 +10,6 @@ on: runner: required: true type: string -# on: -# push: -# branches: -# - master -# - release-51 -# pull_request: -# # GitHub pull_request action is by default triggered when -# # opened reopened or synchronize, -# # we add labeled and unlabeled to the list because -# # the mac job dpends on the PR having a 'Mac' label -# types: -# - labeled -# - unlabeled -# - opened -# - reopened -# - synchronize -# workflow_dispatch: jobs: spellcheck: diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index b7fc2444b..2c36fc5fd 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -17,16 +17,6 @@ on: required: true type: string -# on: -# push: -# branches: -# - master -# - 'ci/**' -# tags: -# - v* -# - e* -# pull_request: - env: IS_CI: "yes" diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh new file mode 100755 index 000000000..9591f7b34 --- /dev/null +++ b/scripts/parse-git-ref.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# $1 is fully qualified git ref name, e.g. refs/tags/v5.1.0 or refs/heads/master + +if [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+$ ]]; then + PROFILE=emqx + EDITION=Opensource + RELEASE=true + LATEST=true +elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+$ ]]; then + PROFILE=emqx-enterprise + EDITION=Enterprise + RELEASE=true + LATEST=true +elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then + PROFILE=emqx + EDITION=Opensource + RELEASE=true + LATEST=false +elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then + PROFILE=emqx-enterprise + EDITION=Enterprise + RELEASE=true + LATEST=false +elif [[ $1 =~ ^refs/tags/.+ ]]; then + echo "Unrecognized tag: $1" + exit 1 +elif [[ $1 =~ ^refs/heads/master$ ]]; then + PROFILE=emqx + EDITION=Opensource + RELEASE=false + LATEST=false +elif [[ $1 =~ ^refs/heads/release-[5-9]+$ ]]; then + PROFILE=emqx-enterprise + EDITION=Enterprise + RELEASE=false + LATEST=false +else + echo "Unrecognized git ref: $1" + exit 1 +fi + +cat < /dev/null; then exit 1 fi -EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-1} +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-3} EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-25.3.2-1} EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} From ec75fa32cfbc471dfced7408193e8fd77c139041 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 07:19:12 +0200 Subject: [PATCH 05/22] ci: use git ref for workflow concurrency detection --- .github/workflows/_push-entrypoint.yaml | 2 +- .github/workflows/build_and_push_docker_images.yaml | 2 +- .github/workflows/build_docker_for_test.yaml | 2 +- .github/workflows/build_packages.yaml | 2 +- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/run_conf_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 2 +- .github/workflows/spellcheck.yaml | 4 ++-- .github/workflows/static_checks.yaml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index af014e74d..32232988b 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -1,7 +1,7 @@ name: Push Entrypoint concurrency: - group: push-entrypoint-${{ github.event_name }}-${{ github.sha }} + group: push-entrypoint-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 63225eec3..33128e40d 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -1,7 +1,7 @@ name: Build and push docker images concurrency: - group: docker-build-${{ github.event_name }}-${{ github.sha }} + group: docker-build-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index e43a651f7..548d5e2cd 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -1,7 +1,7 @@ name: Build docker image for test concurrency: - group: docker-test-build-${{ github.event_name }}-${{ github.sha }} + group: docker-test-build-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 20b64038d..94841edc8 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,7 +1,7 @@ name: Cross build packages concurrency: - group: build-packages-${{ github.event_name }}-${{ github.sha }} + group: build-packages-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 483ab88ec..b7ba78ef4 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,7 +1,7 @@ name: Build slim packages concurrency: - group: slim-${{ github.event_name }}-${{ github.sha }} + group: slim-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 5ffa24946..cf696e0c6 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -1,7 +1,7 @@ name: Run Configuration tests concurrency: - group: conftest-${{ github.event_name }}-${{ github.sha }} + group: conftest-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 325a9ec9c..48e551612 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -1,7 +1,7 @@ name: Run test case concurrency: - group: test-${{ github.event_name }}-${{ github.sha }} + group: test-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml index 440233705..42a464ee8 100644 --- a/.github/workflows/spellcheck.yaml +++ b/.github/workflows/spellcheck.yaml @@ -1,7 +1,7 @@ -name: Build slim packages +name: Spellcheck concurrency: - group: spellcheck-${{ github.event_name }}-${{ github.sha }} + group: spellcheck-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index 2c36fc5fd..3b32a36b4 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -1,7 +1,7 @@ name: Static checks concurrency: - group: static-checks-${{ github.event_name }}-${{ github.sha }} + group: static-checks-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true on: From 43116a0797e346eda6ec9502fc8042746fb5c795 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 07:19:31 +0200 Subject: [PATCH 06/22] ci: fix regex in parse-git-ref.sh --- scripts/parse-git-ref.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh index 9591f7b34..9ad424bc3 100755 --- a/scripts/parse-git-ref.sh +++ b/scripts/parse-git-ref.sh @@ -32,7 +32,7 @@ elif [[ $1 =~ ^refs/heads/master$ ]]; then EDITION=Opensource RELEASE=false LATEST=false -elif [[ $1 =~ ^refs/heads/release-[5-9]+$ ]]; then +elif [[ $1 =~ ^refs/heads/release-[5-9][0-9]+$ ]]; then PROFILE=emqx-enterprise EDITION=Enterprise RELEASE=false From dce2719c30f6a13b9e8e27c20a84cca61b2b00f1 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 07:19:49 +0200 Subject: [PATCH 07/22] chore: fix scripts/check-i18n-style.escript to exit with 1 when there are errors --- scripts/check-i18n-style.escript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index b8b6bdac7..f48e5a513 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -16,9 +16,9 @@ main([Files0]) -> ok = lists:foreach(fun check/1, Files), case get(errors) of 1 -> - logerr("1 error found~n", []); + die("1 error found~n", []); N when is_integer(N) andalso N > 1 -> - logerr("~p errors found~n", [N]); + die("~p errors found~n", [N]); _ -> io:format(user, "~nOK~n", []) end. From 93197137e7dc387c0b1048e7e113c2f5fecefa3b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 07:28:36 +0200 Subject: [PATCH 08/22] fix(hocon): i18n style --- rel/i18n/emqx_bridge_dynamo.hocon | 3 +-- rel/i18n/emqx_dashboard_schema.hocon | 3 +-- rel/i18n/emqx_mgmt_api_key_schema.hocon | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rel/i18n/emqx_bridge_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo.hocon index 0b49c6e2f..417b43c0c 100644 --- a/rel/i18n/emqx_bridge_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo.hocon @@ -37,8 +37,7 @@ local_topic.label: template.desc: """Template, the default value is empty. When this value is empty the whole message will be stored in the database.
The template can be any valid JSON with placeholders and make sure all keys for table are here, example:
- {"id" : "${id}", "clientid" : "${clientid}", "data" : "${payload.data}"} -""" + {"id" : "${id}", "clientid" : "${clientid}", "data" : "${payload.data}"}""" template.label: """Template""" diff --git a/rel/i18n/emqx_dashboard_schema.hocon b/rel/i18n/emqx_dashboard_schema.hocon index 0559af2c5..524e633aa 100644 --- a/rel/i18n/emqx_dashboard_schema.hocon +++ b/rel/i18n/emqx_dashboard_schema.hocon @@ -8,8 +8,7 @@ backlog.label: bind.desc: """Port without IP(18083) or port with specified IP(127.0.0.1:18083). -Disabled when setting bind to `0`. -""" +Disabled when setting bind to `0`.""" bind.label: """Bind""" diff --git a/rel/i18n/emqx_mgmt_api_key_schema.hocon b/rel/i18n/emqx_mgmt_api_key_schema.hocon index c217dc2db..811ab8a98 100644 --- a/rel/i18n/emqx_mgmt_api_key_schema.hocon +++ b/rel/i18n/emqx_mgmt_api_key_schema.hocon @@ -10,8 +10,7 @@ bootstrap_file.desc: """The bootstrap file provides API keys for EMQX. EMQX will load these keys on startup to authorize API requests. It contains key-value pairs in the format:`api_key:api_secret`. -Each line specifies an API key and its associated secret. -""" +Each line specifies an API key and its associated secret.""" bootstrap_file.label: """Initialize api_key file.""" From d100acf701feae82491decb4342e288108fd0385 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 08:04:49 +0200 Subject: [PATCH 09/22] ci: fail faster in helm tests --- .github/workflows/run_helm_tests.yaml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index 85e748da4..1106b6057 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -99,26 +99,19 @@ jobs: sleep 10; done - name: Get Token - timeout-minutes: 1 run: | kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null & - - while - [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] - do - echo "waiting emqx" - sleep 1 - done - + curl --head -X GET --retry 10 --retry-connrefused --retry-delay 6 http://localhost:18083/status echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV - name: Check cluster - timeout-minutes: 10 + timeout-minutes: 1 run: | while - [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; + nodes_length="$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" + [ $nodes_length != "3" ] do - echo "waiting ${EMQX_NAME} cluster scale" + echo "waiting ${EMQX_NAME} cluster scale. Current live nodes: $nodes_length." sleep 1 done - uses: actions/checkout@v3 From 25659d6f4b90859054ed2972925d53363d683842 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 08:08:13 +0200 Subject: [PATCH 10/22] chore(i18n): fix spelling --- rel/i18n/emqx_ldap_authn.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_ldap_authn.hocon b/rel/i18n/emqx_ldap_authn.hocon index 7c59f2039..04dc88e83 100644 --- a/rel/i18n/emqx_ldap_authn.hocon +++ b/rel/i18n/emqx_ldap_authn.hocon @@ -10,7 +10,7 @@ password_attribute.label: """Password Attribute""" is_superuser_attribute.desc: -"""Indicates which attribute is used to represent whether the user is a super user.""" +"""Indicates which attribute is used to represent whether the user is a superuser.""" is_superuser_attribute.label: """IsSuperuser Attribute""" From f31276f8d8b606312869a717281da5de0a87c94c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 08:15:05 +0200 Subject: [PATCH 11/22] build: use test profile for make cover --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index db00303d6..037d33cea 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ endif .PHONY: cover cover: $(REBAR) - @ENABLE_COVER_COMPILE=1 $(REBAR) cover + @ENABLE_COVER_COMPILE=1 $(REBAR) as test cover .PHONY: coveralls coveralls: $(REBAR) From 559cda5d743d1b9a71a422ca831374eb1c84e5f6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 08:20:14 +0200 Subject: [PATCH 12/22] ci: accept 'ci/*' branches in parse-git-ref.sh --- scripts/parse-git-ref.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh index 9ad424bc3..a8d9fdf52 100755 --- a/scripts/parse-git-ref.sh +++ b/scripts/parse-git-ref.sh @@ -37,6 +37,11 @@ elif [[ $1 =~ ^refs/heads/release-[5-9][0-9]+$ ]]; then EDITION=Enterprise RELEASE=false LATEST=false +elif [[ $1 =~ ^refs/heads/ci/.* ]]; then + PROFILE=emqx + EDITION=Opensource + RELEASE=false + LATEST=false else echo "Unrecognized git ref: $1" exit 1 From 8232d8bc8b71fe0d540aefdd5be6f4d539b34337 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 09:07:10 +0200 Subject: [PATCH 13/22] ci: download actionlint from github releases --- .github/actions/pr-sanity-checks/action.yaml | 5 ++++- scripts/pr-sanity-checks.sh | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml index 3114f8a94..d9f74b79d 100644 --- a/.github/actions/pr-sanity-checks/action.yaml +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -23,8 +23,11 @@ runs: # https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions - name: Check workflow files shell: bash + env: + ACTIONLINT_VSN: 1.6.25 run: | - bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz + tar xfz actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz # TODO: enable shellcheck when all the current issues are fixed ./actionlint -color \ -shellcheck= \ diff --git a/scripts/pr-sanity-checks.sh b/scripts/pr-sanity-checks.sh index c4d255941..113ffd324 100755 --- a/scripts/pr-sanity-checks.sh +++ b/scripts/pr-sanity-checks.sh @@ -29,4 +29,5 @@ docker run --rm -it -v "$(pwd):/emqx" -w /emqx \ -e GITHUB_BASE_REF="$BEFORE_REF" \ -e MIX_ENV=emqx-enterprise \ -e PROFILE=emqx-enterprise \ - "${EMQX_BUILDER}" /bin/bash -c "${commands}" + -e ACTIONLINT_VSN=1.6.25 \ + "${EMQX_BUILDER}" /bin/bash -c "git config --global --add safe.directory /emqx; ${commands}" From fdf02a7083eaf04754b54b0f8081d1a0cbbac22f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 09:29:11 +0200 Subject: [PATCH 14/22] ci: restore make test-compile --- .github/workflows/_pr_entrypoint.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 67cedcfb8..a1cd70a97 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -90,7 +90,9 @@ jobs: PROFILE: ${{ matrix.profile }} ENABLE_COVER_COMPILE: 1 run: | - make $PROFILE + make ensure-rebar3 + make ${PROFILE}-compile + make test-compile zip -ryq $PROFILE.zip . - uses: actions/upload-artifact@v3 with: From 75834c2d917cbb888668091605b76757282dbbb7 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 09:40:34 +0200 Subject: [PATCH 15/22] ci: don't check elixir code formatting twice also fix actionlint extraction --- .github/actions/pr-sanity-checks/action.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml index d9f74b79d..385b17755 100644 --- a/.github/actions/pr-sanity-checks/action.yaml +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -20,14 +20,13 @@ runs: run: | DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shellcheck ./scripts/shellcheck.sh - # https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions - name: Check workflow files shell: bash env: ACTIONLINT_VSN: 1.6.25 run: | wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz - tar xfz actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz + tar zxf actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz actionlint # TODO: enable shellcheck when all the current issues are fixed ./actionlint -color \ -shellcheck= \ @@ -41,14 +40,6 @@ runs: shell: bash run: | ./scripts/apps-version-check.sh - - name: Check Erlang code formatting - shell: bash - run: | - ./scripts/check-format.sh - - name: Run elvis check - shell: bash - run: | - ./scripts/elvis-check.sh $GITHUB_BASE_REF - name: Setup mix env: MIX_ENV: emqx-enterprise @@ -56,10 +47,14 @@ runs: shell: bash run: | mix local.hex --force && mix local.rebar --force - - name: Check Elixir code formatting + - name: Check formatting env: MIX_ENV: emqx-enterprise PROFILE: emqx-enterprise shell: bash run: | - mix format --check-formatted + ./scripts/check-format.sh + - name: Run elvis check + shell: bash + run: | + ./scripts/elvis-check.sh $GITHUB_BASE_REF From 6093f03cb9ddee9b626d167dd747dcdb2fe95eaf Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 09:58:20 +0200 Subject: [PATCH 16/22] ci: make $PROFILE instead of make $PROFILE-compile --- .github/workflows/_pr_entrypoint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index a1cd70a97..4fc8ee774 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -91,7 +91,7 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make ensure-rebar3 - make ${PROFILE}-compile + make ${PROFILE} make test-compile zip -ryq $PROFILE.zip . - uses: actions/upload-artifact@v3 From ff14995bf694b5878665eb3c5fd9491af7dd54c3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 13:26:35 +0200 Subject: [PATCH 17/22] ci: automatically detect if the git tag is the latest one also support tags like v5.1.1.1 --- scripts/parse-git-ref.sh | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh index a8d9fdf52..407c7b21a 100755 --- a/scripts/parse-git-ref.sh +++ b/scripts/parse-git-ref.sh @@ -4,16 +4,36 @@ set -euo pipefail # $1 is fully qualified git ref name, e.g. refs/tags/v5.1.0 or refs/heads/master +is_latest() { + ref_name=$(basename "$1") + latest_ref_name=$(git describe --tags "$(git rev-list --tags --max-count=1)") + if [[ "$ref_name" == "$latest_ref_name" ]]; then + echo true; + else + echo false; + fi +} + if [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx EDITION=Opensource RELEASE=true - LATEST=true + LATEST=$(is_latest "$1") +elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + PROFILE=emqx + EDITION=Opensource + RELEASE=true + LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+$ ]]; then PROFILE=emqx-enterprise EDITION=Enterprise RELEASE=true - LATEST=true + LATEST=$(is_latest "$1") +elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + PROFILE=emqx-enterprise + EDITION=Enterprise + RELEASE=true + LATEST=$(is_latest "$1") elif [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then PROFILE=emqx EDITION=Opensource From 308d7b872d134b109148a5d087987d8b374ab5a1 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 4 Aug 2023 14:30:39 +0200 Subject: [PATCH 18/22] ci: minor fixes --- .github/actions/pr-sanity-checks/action.yaml | 2 +- .github/actions/prepare-jmeter/action.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml index 385b17755..70efae7a3 100644 --- a/.github/actions/pr-sanity-checks/action.yaml +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -46,7 +46,7 @@ runs: PROFILE: emqx-enterprise shell: bash run: | - mix local.hex --force && mix local.rebar --force + mix local.hex --force --if-missing && mix local.rebar --force --if-missing - name: Check formatting env: MIX_ENV: emqx-enterprise diff --git a/.github/actions/prepare-jmeter/action.yaml b/.github/actions/prepare-jmeter/action.yaml index 23e868212..e3d30af1a 100644 --- a/.github/actions/prepare-jmeter/action.yaml +++ b/.github/actions/prepare-jmeter/action.yaml @@ -1,4 +1,4 @@ -name: 'Run jmeter' +name: 'Prepare jmeter' inputs: version-emqx: From 8545cac33d9cfc7a8310e8eb5e9cce37190ecd1b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 6 Aug 2023 13:45:33 +0200 Subject: [PATCH 19/22] ci(parse-git-ref): handle more than one tag on the same commit also add shelltest in CI --- .github/actions/pr-sanity-checks/action.yaml | 5 ++ scripts/parse-git-ref.sh | 18 ++-- scripts/shelltest/parse-git-ref.cleanup | 4 + scripts/shelltest/parse-git-ref.setup | 4 + scripts/shelltest/parse-git-ref.test | 94 ++++++++++++++++++++ scripts/shelltest/run_tests.sh | 17 ++++ 6 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 scripts/shelltest/parse-git-ref.cleanup create mode 100644 scripts/shelltest/parse-git-ref.setup create mode 100644 scripts/shelltest/parse-git-ref.test create mode 100755 scripts/shelltest/run_tests.sh diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml index 70efae7a3..f19fe2c6d 100644 --- a/.github/actions/pr-sanity-checks/action.yaml +++ b/.github/actions/pr-sanity-checks/action.yaml @@ -20,6 +20,11 @@ runs: run: | DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shellcheck ./scripts/shellcheck.sh + - name: Run shell tests + shell: bash + run: | + DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shelltestrunner + scripts/shelltest/run_tests.sh - name: Check workflow files shell: bash env: diff --git a/scripts/parse-git-ref.sh b/scripts/parse-git-ref.sh index 407c7b21a..a486f2589 100755 --- a/scripts/parse-git-ref.sh +++ b/scripts/parse-git-ref.sh @@ -6,12 +6,14 @@ set -euo pipefail is_latest() { ref_name=$(basename "$1") - latest_ref_name=$(git describe --tags "$(git rev-list --tags --max-count=1)") - if [[ "$ref_name" == "$latest_ref_name" ]]; then - echo true; - else - echo false; - fi + # shellcheck disable=SC2046 + for t in $(git tag --points-at $(git rev-list --tags --max-count=1)); do + if [[ "$t" == "$ref_name" ]]; then + echo true; + return; + fi + done + echo false } if [[ $1 =~ ^refs/tags/v[5-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -45,7 +47,7 @@ elif [[ $1 =~ ^refs/tags/e[5-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; th RELEASE=true LATEST=false elif [[ $1 =~ ^refs/tags/.+ ]]; then - echo "Unrecognized tag: $1" + echo "Unrecognized tag: $1" 1>&2 exit 1 elif [[ $1 =~ ^refs/heads/master$ ]]; then PROFILE=emqx @@ -63,7 +65,7 @@ elif [[ $1 =~ ^refs/heads/ci/.* ]]; then RELEASE=false LATEST=false else - echo "Unrecognized git ref: $1" + echo "Unrecognized git ref: $1" 1>&2 exit 1 fi diff --git a/scripts/shelltest/parse-git-ref.cleanup b/scripts/shelltest/parse-git-ref.cleanup new file mode 100644 index 000000000..8c6dcc027 --- /dev/null +++ b/scripts/shelltest/parse-git-ref.cleanup @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +git tag -d v5.1.99 >/dev/null +git tag -d e5.1.99 >/dev/null diff --git a/scripts/shelltest/parse-git-ref.setup b/scripts/shelltest/parse-git-ref.setup new file mode 100644 index 000000000..f8938787f --- /dev/null +++ b/scripts/shelltest/parse-git-ref.setup @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +git tag v5.1.99 +git tag e5.1.99 diff --git a/scripts/shelltest/parse-git-ref.test b/scripts/shelltest/parse-git-ref.test new file mode 100644 index 000000000..3f2ede0d1 --- /dev/null +++ b/scripts/shelltest/parse-git-ref.test @@ -0,0 +1,94 @@ +./parse-git-ref.sh refs/tags/v5.2.0-foobar.1 +>>>2 +Unrecognized tag: refs/tags/v5.2.0-foobar.1 +>>>= 1 + +./parse-git-ref.sh v5.2.0 +>>>2 +Unrecognized git ref: v5.2.0 +>>>= 1 + +./parse-git-ref.sh refs/tags/v5.1.0 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/v5.1.5.1 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/v5.2.0-alpha.1 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/v5.2.0-alpha-1 +>>>2 +Unrecognized tag: refs/tags/v5.2.0-alpha-1 +>>>= 1 + +./parse-git-ref.sh refs/tags/v5.2.0-beta.1 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/v5.2.0-rc.1 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.1.0 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.1.5.1 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.2.0-alpha.1 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.2.0-beta.1 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.2.0-rc.1 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/tags/e5.1.99 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": true, "latest": true} +>>>= 0 + +./parse-git-ref.sh refs/tags/v5.1.99 +>>> +{"profile": "emqx", "edition": "Opensource", "release": true, "latest": true} +>>>= 0 + +./parse-git-ref.sh refs/heads/master +>>> +{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/heads/release-51 +>>> +{"profile": "emqx-enterprise", "edition": "Enterprise", "release": false, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/heads/ci/foobar +>>> +{"profile": "emqx", "edition": "Opensource", "release": false, "latest": false} +>>>= 0 + +./parse-git-ref.sh refs/heads/release-44 +>>>2 +Unrecognized git ref: refs/heads/release-44 +>>>= 1 diff --git a/scripts/shelltest/run_tests.sh b/scripts/shelltest/run_tests.sh new file mode 100755 index 000000000..28f48eaee --- /dev/null +++ b/scripts/shelltest/run_tests.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +cd -P -- "$(dirname -- "$0")/.." + +exit_code=0 + +for test in shelltest/*.test; do + echo "Running $test" + /bin/sh "${test%.test}.setup" + shelltest -c --diff --all --precise -- "$test" + if [ $? -ne 0 ]; then + exit_code=1 + fi + /bin/sh "${test%.test}.cleanup" +done + +exit $exit_code From 4767b952dd2d0773562048047ab36a6a7032ed96 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 6 Aug 2023 13:57:47 +0200 Subject: [PATCH 20/22] ci: move sanity checks to the main workflow --- .github/actions/pr-sanity-checks/action.yaml | 65 -------------------- .github/workflows/_pr_entrypoint.yaml | 50 ++++++++++++++- scripts/pr-sanity-checks.sh | 2 +- scripts/shelltest/run_tests.sh | 2 + 4 files changed, 51 insertions(+), 68 deletions(-) delete mode 100644 .github/actions/pr-sanity-checks/action.yaml diff --git a/.github/actions/pr-sanity-checks/action.yaml b/.github/actions/pr-sanity-checks/action.yaml deleted file mode 100644 index f19fe2c6d..000000000 --- a/.github/actions/pr-sanity-checks/action.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: 'Pull request sanity checks' - -defaults: - run: - shell: 'bash -Eeuo pipefail {0}' - -runs: - using: composite - steps: - - name: Run gitlint - shell: bash - env: - BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} - AFTER_REF: ${{ github.sha }} - run: | - pip install gitlint - gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint - - name: Run shellcheck - shell: bash - run: | - DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shellcheck - ./scripts/shellcheck.sh - - name: Run shell tests - shell: bash - run: | - DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shelltestrunner - scripts/shelltest/run_tests.sh - - name: Check workflow files - shell: bash - env: - ACTIONLINT_VSN: 1.6.25 - run: | - wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz - tar zxf actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz actionlint - # TODO: enable shellcheck when all the current issues are fixed - ./actionlint -color \ - -shellcheck= \ - -ignore 'label ".+" is unknown' \ - -ignore 'value "emqx-enterprise" in "exclude"' - - name: Check line-break at EOF - shell: bash - run: | - ./scripts/check-nl-at-eof.sh - - name: Check apps version - shell: bash - run: | - ./scripts/apps-version-check.sh - - name: Setup mix - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - shell: bash - run: | - mix local.hex --force --if-missing && mix local.rebar --force --if-missing - - name: Check formatting - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - shell: bash - run: | - ./scripts/check-format.sh - - name: Run elvis check - shell: bash - run: | - ./scripts/elvis-check.sh $GITHUB_BASE_REF diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 4fc8ee774..ec2bbf2e1 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -38,8 +38,54 @@ jobs: - name: Work around https://github.com/actions/checkout/issues/766 run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: ./.github/actions/pr-sanity-checks - - name: Build matrix + - name: Run gitlint + env: + BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + AFTER_REF: ${{ github.sha }} + run: | + pip install gitlint + gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint + - name: Run shellcheck + run: | + DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shellcheck + ./scripts/shellcheck.sh + - name: Run shell tests + run: | + DEBIAN_FRONTEND=noninteractive apt-get update -qy && apt-get install -qy shelltestrunner + scripts/shelltest/run_tests.sh + - name: Check workflow files + env: + ACTIONLINT_VSN: 1.6.25 + run: | + wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz + tar zxf actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz actionlint + # TODO: enable shellcheck when all the current issues are fixed + ./actionlint -color \ + -shellcheck= \ + -ignore 'label ".+" is unknown' \ + -ignore 'value "emqx-enterprise" in "exclude"' + - name: Check line-break at EOF + run: | + ./scripts/check-nl-at-eof.sh + - name: Check apps version + run: | + ./scripts/apps-version-check.sh + - name: Setup mix + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + run: | + mix local.hex --force --if-missing && mix local.rebar --force --if-missing + - name: Check formatting + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise + run: | + ./scripts/check-format.sh + - name: Run elvis check + run: | + ./scripts/elvis-check.sh $GITHUB_BASE_REF + - name: Generate CT Matrix id: matrix run: | APPS="$(./scripts/find-apps.sh --ci)" diff --git a/scripts/pr-sanity-checks.sh b/scripts/pr-sanity-checks.sh index 113ffd324..6b193b74e 100755 --- a/scripts/pr-sanity-checks.sh +++ b/scripts/pr-sanity-checks.sh @@ -18,7 +18,7 @@ EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}} -commands=$(yq ".runs.steps[].run" .github/actions/pr-sanity-checks/action.yaml | grep -v null) +commands=$(yq ".jobs.sanity-checks.steps[].run" .github/workflows/_pr_entrypoint.yaml | grep -v null) BEFORE_REF=${BEFORE_REF:-$(git rev-parse master)} AFTER_REF=${AFTER_REF:-$(git rev-parse HEAD)} diff --git a/scripts/shelltest/run_tests.sh b/scripts/shelltest/run_tests.sh index 28f48eaee..11caa6cac 100755 --- a/scripts/shelltest/run_tests.sh +++ b/scripts/shelltest/run_tests.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +# shellcheck disable=SC2164 cd -P -- "$(dirname -- "$0")/.." exit_code=0 @@ -8,6 +9,7 @@ for test in shelltest/*.test; do echo "Running $test" /bin/sh "${test%.test}.setup" shelltest -c --diff --all --precise -- "$test" + # shellcheck disable=SC2181 if [ $? -ne 0 ]; then exit_code=1 fi From 0c17b31fffc53b98a4f9cc53761c9b81cfe7ae8f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 6 Aug 2023 14:18:08 +0200 Subject: [PATCH 21/22] fix(i18n): i18n file styles --- rel/i18n/emqx_conf_schema.hocon | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 46d2817ca..442df1fa8 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -779,8 +779,7 @@ db_shard_transports.label: node_broker_pool_size.desc: """The number of workers in emqx_broker pool. Increasing this value may improve performance by enhancing parallelism, especially when EMQX cluster interconnect network latency is high. -Defaults to the number of Erlang schedulers (CPU cores) * 2. -""" +Defaults to the number of Erlang schedulers (CPU cores) * 2.""" node_broker_pool_size.label: """Node Broker Pool Size""" @@ -788,8 +787,7 @@ node_broker_pool_size.label: node_generic_pool_size.desc: """The number of workers in emqx_pool. Increasing this value may improve performance by enhancing parallelism, especially when EMQX cluster interconnect network latency is high. -Defaults to the number of Erlang schedulers (CPU cores). -""" +Defaults to the number of Erlang schedulers (CPU cores).""" node_generic_pool_size.label: """Node Generic Pool Size""" @@ -798,8 +796,7 @@ node_channel_cleanup_batch_size.desc: """The size of the channel cleanup batch. if EMQX cluster interconnect network latency is high, reducing this value together with increasing node.generic_pool_size may improve performance during an abrupt disconnect of a large numbers of clients. -Defaults to 100000. -""" +Defaults to 100000.""" node_channel_cleanup_batch_size.label: """Node Channel Cleanup Batch Size""" From 0e54db90390885d468e167367d11a18a4e4bde77 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 7 Aug 2023 09:42:51 +0200 Subject: [PATCH 22/22] ci: build slim packages on push to branches --- .github/workflows/_push-entrypoint.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 32232988b..9faceebbc 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -85,6 +85,18 @@ jobs: echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT + build_slim_packages: + if: ${{ needs.prepare.outputs.release != 'true' }} + needs: + - prepare + uses: ./.github/workflows/build_slim_packages.yaml + with: + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} + builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} + elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} + build_packages: if: ${{ needs.prepare.outputs.release == 'true' }} needs: