From cab063926ce7c4b68bd0bc6441cd0057baa3ede6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 14:06:24 +0300 Subject: [PATCH 01/19] chore: bump `redbug` to 2.0.8 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index e9678b1cd..22c6a23ed 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ {deps, [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} - , {redbug, "2.0.7"} + , {redbug, "2.0.8"} , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} From 202d2b07ff07ee7ce71cf144cf6b9d0769bed0d2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 14:12:21 +0300 Subject: [PATCH 02/19] chore: switch to `emqx/rebar3` 3.19.0-emqx-1 Which should be safe to run on OTP-25. --- scripts/ensure-rebar3.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index e31dba207..1d6794bda 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,7 +2,7 @@ set -euo pipefail -## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. +## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image. ## we have to use an otp24-compiled rebar3 because the defination of record #application{} ## in systools.hrl is changed in otp24. OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" @@ -13,6 +13,9 @@ case ${OTP_VSN} in 24*) VERSION="3.18.0-emqx-1" ;; + 25*) + VERSION="3.19.0-emqx-1" + ;; *) echo "Unsupporetd Erlang/OTP version $OTP_VSN" exit 1 From 2e1cdd039e443b3b6662775f0efa1bb01bdf5d5a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 14:21:52 +0300 Subject: [PATCH 03/19] ci: include OTP-25 in some workflows * Use OTP-25 exclusively for Elixir builds * Include OTP-25 in slim package builds * Include OTP-25 in test workflow matrices --- .ci/docker-compose-file/Makefile.local | 2 +- .../docker-compose-kafka.yaml | 2 +- .ci/docker-compose-file/docker-compose.yaml | 6 +- .github/actions/package-macos/action.yaml | 2 +- .../build_and_push_docker_images.yaml | 18 +-- .github/workflows/build_packages.yaml | 13 ++- .github/workflows/build_slim_packages.yaml | 11 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 3 +- .github/workflows/run_emqx_app_tests.yaml | 5 +- .github/workflows/run_fvt_tests.yaml | 14 ++- .github/workflows/run_jmeter_tests.yaml | 14 +-- .github/workflows/run_relup_tests.yaml | 8 +- .github/workflows/run_test_cases.yaml | 105 +++++++++--------- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- deploy/docker/Dockerfile.alpine | 2 +- rebar.config.erl | 4 +- scripts/buildx.sh | 6 +- scripts/ct/run.sh | 4 +- scripts/relup-test/run-relup-lux.sh | 4 +- .../relup-test/start-relup-test-cluster.sh | 2 +- 25 files changed, 127 insertions(+), 110 deletions(-) diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local index d11ab64a6..ff4f348b0 100644 --- a/.ci/docker-compose-file/Makefile.local +++ b/.ci/docker-compose-file/Makefile.local @@ -2,7 +2,7 @@ define usage make -f .ci/docker-compose-file/Makefile.local up -make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang24 SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl make -f .ci/docker-compose-file/Makefile.local down endef export usage diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index ba0161293..716c4c709 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -19,7 +19,7 @@ services: command: /bin/generate-certs.sh kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 container_name: kdc.emqx.net networks: emqx_bridge: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 4a5ef7070..b55b3196f 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -1,9 +1,9 @@ version: '3.9' services: - erlang24: - container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + erlang: + container_name: erlang + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04} env_file: - conf.env environment: diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 37284e8b6..acd4975e6 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -3,7 +3,7 @@ inputs: profile: # emqx, emqx-enterprise required: true type: string - otp: # 24.2.1-1, 23.3.4.9-3 + otp: # 25.1.2-2, 24.3.4.2-1 required: true type: string os: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 2893489e4..76238c75f 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -23,7 +23,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} @@ -119,8 +119,10 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix + builder: + - 5.0-26 # update to latest otp: - - 24.3.4.2-1 # update to latest + - 24.3.4.2-1 # switch to 25 once ready to release 5.1 elixir: - 1.13.4 # update to latest exclude: # TODO: publish enterprise to ecr too? @@ -174,7 +176,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -208,8 +210,10 @@ jobs: - 'docker.io' os: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] + builder: + - 5.0-26 # update to latest otp: - - 24.3.4.2-1 # update to latest + - 25.1.2-2 # update to latest elixir: - 1.13.4 # update to latest @@ -252,7 +256,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -277,7 +281,7 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: only support latest otp version, not a matrix otp: - - 24.3.4.2-1 # update to latest + - 24.3.4.2-1 # switch to 25 once ready to release 5.1 registry: - 'docker.io' - 'public.ecr.aws' @@ -342,7 +346,7 @@ jobs: - ${{ needs.prepare.outputs.BUILD_PROFILE }} # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.3.4.2-1 # update to latest + - 25.1.2-2 # update to latest elixir: - 1.13.4 # update to latest registry: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 80a8f1468..2bb4fb42e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -48,7 +48,7 @@ jobs: echo "WARN: This is NOT an exact git tag, will not publish release" is_exact_tag='false' fi - echo "::set-output name=IS_EXACT_TAG::${is_exact_tag}" + echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT case $tag in refs/tags/v*) PROFILE='emqx' @@ -103,7 +103,7 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1 with: - otp-version: 24.2.1 + otp-version: 24.3.4 - name: build env: PYTHON: python @@ -173,13 +173,15 @@ jobs: needs: prepare runs-on: ${{ matrix.build_machine }} container: - image: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false matrix: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -244,6 +246,7 @@ jobs: - name: build emqx packages working-directory: source env: + BUILDER: ${{ matrix.builder }} OTP: ${{ matrix.otp }} ELIXIR: ${{ matrix.elixir }} PROFILE: ${{ matrix.profile }} @@ -272,7 +275,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IsElixir}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-24:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 12b17535a..ebaa17abf 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -31,12 +31,15 @@ jobs: profile: - ["emqx", "el7"] - ["emqx-enterprise", "ubuntu20.04"] + builder: + - 5.0-26 otp: - 24.3.4.2-1 + - 25.1.2-2 elixir: - 1.13.4 - container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" steps: - uses: AutoModality/action-clean@v1 @@ -89,7 +92,8 @@ jobs: profile: - emqx otp: - - 24.2.1 + - 24.3.4 + - 25.1.2 steps: - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1.12.0 @@ -119,7 +123,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: windows - path: _packages/${{ matrix.profile}}/* + path: _packages/${{ matrix.profile }}/* mac: strategy: @@ -130,6 +134,7 @@ jobs: - emqx-enterprise otp: - 24.3.4.2-1 + - 25.1.2-2 os: - macos-11 - macos-12-arm64 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2578e078f..59fc01d74 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 910bb8835..6ab8ea8b6 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index e4b93cf80..62000f421 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -8,7 +8,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 9f1d6f3e4..5e64d69c4 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 0ac1e9155..f53051d1d 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,8 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 - + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 71b016761..ddbb14609 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -11,8 +11,11 @@ jobs: run_emqx_app_tests: strategy: matrix: + builder: + - 5.0-26 otp: - 24.3.4.2-1 + - 25.1.2-2 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify # the base image. @@ -34,7 +37,7 @@ jobs: use-self-hosted: false runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8870b8257..9e51b7142 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-alpine3.15.1 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1 steps: - uses: actions/checkout@v3 @@ -48,6 +48,8 @@ jobs: - rlog os: - ["alpine3.15.1", "alpine:3.15.1"] + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -57,7 +59,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3.4 - uses: actions/download-artifact@v3 with: name: source @@ -68,7 +70,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker @@ -119,6 +121,8 @@ jobs: - emqx-enterprise os: - ["debian11", "debian:11-slim"] + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -130,7 +134,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3.4 - uses: actions/download-artifact@v3 with: name: source @@ -141,7 +145,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index f67730b30..283d7c014 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -16,7 +16,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - name: download jmeter timeout-minutes: 3 env: @@ -36,7 +36,7 @@ jobs: PROFILE='emqx' make "${PROFILE}-docker" VSN="$(./pkg-vsn.sh $PROFILE)" - echo "::set-output name=version::${VSN}" + echo "version=${VSN}" >> $GITHUB_OUTPUT docker save -o emqx.tar emqx/emqx:${VSN} - uses: actions/upload-artifact@v3 with: @@ -59,7 +59,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -155,7 +155,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -261,7 +261,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -363,7 +363,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -462,7 +462,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index ac2da6f85..cb1540fdd 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -16,7 +16,7 @@ on: jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} @@ -37,8 +37,8 @@ jobs: ee_vsn="$(./pkg-vsn.sh enterprise)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') - echo "::set-output name=CUR_EE_VSN::$ee_vsn" - echo "::set-output name=OLD_VERSIONS::$old_vsns" + echo "CUR_EE_VSN=$ee_vsn" >> $GITHUB_OUTPUT + echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT - name: build emqx run: | set -x @@ -74,7 +74,7 @@ jobs: # setup Erlang to run lux - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 with: repository: hawk/lux diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index a21f130d4..23adbc5bb 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,8 +16,20 @@ on: jobs: prepare: runs-on: aws-amd64 - # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + builder: + - 5.0-26 + otp: + - 24.3.4.2-1 + - 25.1.2-2 + elixir: + - 1.13.4 + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" outputs: fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }} @@ -32,57 +44,27 @@ jobs: run: | fast_ct_apps="$(./scripts/find-apps.sh --ci fast)" docker_ct_apps="$(./scripts/find-apps.sh --ci docker)" - echo "fast: $fast_ct_apps" - echo "docker: $docker_ct_apps" - echo "::set-output name=fast_ct_apps::$fast_ct_apps" - echo "::set-output name=docker_ct_apps::$docker_ct_apps" + echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT + echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT - name: get_all_deps working-directory: source env: - PROFILE: emqx + PROFILE: ${{ matrix.profile }} #DIAGNOSTIC: 1 run: | make ensure-rebar3 # fetch all deps and compile - make emqx + make ${{ matrix.profile }} make test-compile cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: - name: source-emqx - path: source.zip - - prepare_ee: - runs-on: aws-amd64 - # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 - with: - path: source - - name: get_all_deps - working-directory: source - env: - PROFILE: emqx-enterprise - #DIAGNOSTIC: 1 - run: | - make ensure-rebar3 - # fetch all deps and compile - make emqx-enterprise - make test-compile - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source-emqx-enterprise + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: source.zip eunit_and_proper: - needs: - - prepare - - prepare_ee + needs: [prepare] runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false @@ -90,6 +72,13 @@ jobs: profile: - emqx - emqx-enterprise + builder: + - 5.0-26 + otp: + - 24.3.4.2-1 + - 25.1.2-2 + elixir: + - 1.13.4 runs-on: - aws-amd64 - ubuntu-20.04 @@ -104,17 +93,15 @@ jobs: defaults: run: shell: bash - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code - env: - PROFILE: ${{ matrix.profile }} run: unzip -o -q source.zip # produces eunit.coverdata - name: eunit @@ -136,9 +123,7 @@ jobs: path: source/_build/test/cover ct_docker: - needs: - - prepare - - prepare_ee + needs: [prepare] strategy: fail-fast: false matrix: @@ -148,11 +133,19 @@ jobs: - ubuntu-20.04 use-self-hosted: - ${{ github.repository_owner == 'emqx' }} + builder: + - 5.0-26 + otp: + - 24.3.4.2-1 + - 25.1.2-2 + elixir: + - 1.13.4 exclude: - runs-on: ubuntu-20.04 use-self-hosted: true - runs-on: aws-amd64 use-self-hosted: false + runs-on: ${{ matrix.runs-on }} defaults: run: @@ -162,13 +155,14 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }} + name: source-${{ matrix.app[1] }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip - name: run tests working-directory: source env: + DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" MONGO_TAG: 5 MYSQL_TAG: 8 PGSQL_TAG: 13 @@ -191,13 +185,18 @@ jobs: path: source/_build/test/logs ct: - needs: - - prepare - - prepare_ee + needs: [prepare] strategy: fail-fast: false matrix: app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} + builder: + - 5.0-26 + otp: + - 24.3.4.2-1 + - 25.1.2-2 + elixir: + - 1.13.4 runs-on: - aws-amd64 - ubuntu-20.04 @@ -210,7 +209,7 @@ jobs: use-self-hosted: false runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" defaults: run: shell: bash @@ -219,7 +218,7 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }} + name: source-${{ matrix.app[1] }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip @@ -249,12 +248,12 @@ jobs: - ct - ct_docker runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise + name: source-emqx-enterprise-24.3.4.2-1 path: . - name: unzip source code run: unzip -q source.zip diff --git a/Makefile b/Makefile index d6d913a11..503889acb 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 51b4dbd0c..4a00c68fb 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 ARG RUN_FROM=debian:11-slim FROM ${BUILD_FROM} AS builder diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index dc16afc54..0f72be9ab 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-15:1.13.3-24.2.1-1-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/rebar.config.erl b/rebar.config.erl index 8c8f87fa8..0b1ca97eb 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -15,8 +15,8 @@ do(Dir, CONFIG) -> end. assert_otp() -> - Oldest = 23, - Latest = 24, + Oldest = 24, + Latest = 25, OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < Oldest orelse OtpRelease > Latest of true -> diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 96e48d9eb..ca7f812f6 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10 +## --builder ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 set -euo pipefail @@ -22,9 +22,9 @@ help() { echo "--elixir: Specify if the release should be built with Elixir, " echo " defaults to 'no'." echo "--arch amd64|arm64: Target arch to build the EMQX package for" - echo "--src_dir : EMQX source ode in this dir, default to PWD" + echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11" } while [ "$#" -gt 0 ]; do diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index ae779c572..d86ee4f6a 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -66,7 +66,7 @@ if [ "${WHICH_APP}" = 'novalue' ]; then exit 1 fi -ERLANG_CONTAINER='erlang24' +ERLANG_CONTAINER='erlang' DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" case "${WHICH_APP}" in @@ -89,7 +89,7 @@ FILES=( ) for dep in ${CT_DEPS}; do case "${dep}" in - erlang24) + erlang) FILES+=( '.ci/docker-compose-file/docker-compose.yaml' ) ;; toxiproxy) diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh index d446ee29f..cf30db850 100755 --- a/scripts/relup-test/run-relup-lux.sh +++ b/scripts/relup-test/run-relup-lux.sh @@ -45,8 +45,8 @@ fi # From now on, no need for the v|e prefix OLD_VSN="${old_vsn#[e|v]}" -OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" -CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" +OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" +CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" if [ ! -f "$OLD_PKG" ]; then echo "$OLD_PKG not found" diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index b66894c95..c9e3edbd9 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE" From a614e3065a002151b46b7984a294feb69ea219e7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 14:14:50 +0300 Subject: [PATCH 04/19] chore(tls): drop OTP-22-related workaround --- apps/emqx/src/emqx_tls_lib.erl | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 85c4396ab..fc3ed9353 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -55,18 +55,6 @@ %% non-empty list of strings -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))). -%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl) -%% should return when running on otp 23. -%% But we still have to hard-code them because tlsv1.3 on otp 22 is -%% not trustworthy. --define(TLSV13_EXCLUSIVE_CIPHERS, [ - "TLS_AES_256_GCM_SHA384", - "TLS_AES_128_GCM_SHA256", - "TLS_CHACHA20_POLY1305_SHA256", - "TLS_AES_128_CCM_SHA256", - "TLS_AES_128_CCM_8_SHA256" -]). - -define(SELECTED_CIPHERS, [ "ECDHE-ECDSA-AES256-GCM-SHA384", "ECDHE-RSA-AES256-GCM-SHA384", @@ -162,7 +150,7 @@ all_ciphers(['tlsv1.3']) -> %% because 'all' returns legacy cipher suites too, %% which does not make sense since tlsv1.3 can not use %% legacy cipher suites. - ?TLSV13_EXCLUSIVE_CIPHERS; + ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); all_ciphers(Versions) -> %% assert non-empty List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]), @@ -197,7 +185,7 @@ selected_ciphers(Vsns) -> do_selected_ciphers('tlsv1.3') -> case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of - true -> ?TLSV13_EXCLUSIVE_CIPHERS; + true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); false -> [] end ++ do_selected_ciphers('tlsv1.2'); do_selected_ciphers(_) -> From c301c8e992b7548012654b3f0e2b2f8c3dfd8b54 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 15:24:45 +0300 Subject: [PATCH 05/19] chore: drop few unused macros / includes --- apps/emqx/test/emqx_common_test_helpers.erl | 8 -------- apps/emqx/test/emqx_router_helper_SUITE.erl | 1 - apps/emqx/test/emqx_shared_sub_SUITE.erl | 2 -- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 3 --- 4 files changed, 14 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 1d0d793bd..ac8659735 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,8 +16,6 @@ -module(emqx_common_test_helpers). --include_lib("common_test/include/ct.hrl"). - -type special_config_handler() :: fun(). -type apps() :: list(atom()). @@ -78,12 +76,6 @@ -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])). --define(MQTT_SSL_TWOWAY, [ - {cacertfile, ?CERTS_PATH("cacert.pem")}, - {verify, verify_peer}, - {fail_if_no_peer_cert, true} -]). - -define(MQTT_SSL_CLIENT_CERTS, [ {keyfile, ?CERTS_PATH("client-key.pem")}, {cacertfile, ?CERTS_PATH("cacert.pem")}, diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 875ec8b6c..20c94087a 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 7c38fdf17..a428a6fa9 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -21,8 +21,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(SUITE, ?MODULE). diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index a3787fe6e..fcf65046d 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -19,10 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx_conf.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). From 9f49b8fd4a480ee41cc2520e24dd0f9acd6d83b7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 15:42:41 +0300 Subject: [PATCH 06/19] fix: also pin `redbug` to 2.0.8 in mix.exs --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 11ef14eb1..e58bbf28c 100644 --- a/mix.exs +++ b/mix.exs @@ -45,7 +45,7 @@ defmodule EMQXUmbrella.MixProject do # other exact versions, and not ranges. [ {:lc, github: "emqx/lc", tag: "0.3.2", override: true}, - {:redbug, "2.0.7"}, + {:redbug, "2.0.8"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.2", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, From 51d849f267aa75b94cbb6c75d1aebc6d2c412655 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 8 Dec 2022 19:40:20 +0300 Subject: [PATCH 07/19] fix(release): pass overlay vars with correct option Even though relx 4.7.0 seems to support passing overlay vars through `overlay_vars` option, it was designed to hold list of filenames where vars are defined and works unreliable if used as a proplist. --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index 0b1ca97eb..fe0992240 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -270,7 +270,7 @@ relx(Vsn, RelType, PkgType, Edition) -> {vm_args, false}, {release, {emqx, Vsn}, relx_apps(RelType, Edition)}, {overlay, relx_overlay(RelType, Edition)}, - {overlay_vars, + {overlay_vars_values, build_info() ++ [ {emqx_description, emqx_description(RelType, Edition)} From 86d3710f179ae3a65e192cbf44baaf35bc157670 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 12 Dec 2022 10:32:16 +0300 Subject: [PATCH 08/19] fix: use proper indentation in Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 503889acb..cb26a2529 100644 --- a/Makefile +++ b/Makefile @@ -238,8 +238,8 @@ define gen-elixir-pkg-target # the Elixir places the tar in a different path than Rebar3 $1-elixir-pkg: $(COMMON_DEPS) @env TAR_PKG_DIR=_build/$1-pkg \ - IS_ELIXIR=yes \ - $(BUILD) $1-pkg pkg + IS_ELIXIR=yes \ + $(BUILD) $1-pkg pkg endef $(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt)))) From 8a0ca38a772d9f30a8db623d1c09288bf912bb88 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 12 Dec 2022 10:33:01 +0300 Subject: [PATCH 09/19] fix: drop no longer supported dialyzer option --- apps/emqx/rebar.config | 2 +- apps/emqx_resource/rebar.config | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index ff27566ec..4c00d7f23 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -50,7 +50,7 @@ ]}. {dialyzer, [ - {warnings, [unmatched_returns, error_handling, race_conditions]}, + {warnings, [unmatched_returns, error_handling]}, {plt_location, "."}, {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index e6857829f..929514ddd 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -10,10 +10,6 @@ {extra_src_dirs, ["examples"]}. -%% try to override the dialyzer 'race_conditions' defined in the top-level dir, -%% but it doesn't work -{dialyzer, [{warnings, [unmatched_returns, error_handling]}]}. - {deps, [ {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}}, {emqx, {path, "../emqx"}} From 30aa7d7a7ee80899e214d4034da48f02a7ed8f1e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 12 Dec 2022 13:26:44 +0300 Subject: [PATCH 10/19] ci: bump elvis version to `1.1.0-emqx-2` --- scripts/elvis-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index bf5cf9b8e..fcc0be03b 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -5,7 +5,7 @@ set -euo pipefail -elvis_version='1.0.0-emqx-2' +elvis_version='1.1.0-emqx-2' base="${1:-}" repo="${2:-emqx/emqx}" From fed7c80a9a8d69184134b371904c0f4fdf536af0 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 12 Dec 2022 14:48:00 +0300 Subject: [PATCH 11/19] fix(ci): spec function with explicit exception --- apps/emqx/src/emqx_connection.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index d892f5acc..d604d4784 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -641,6 +641,7 @@ close_socket_ok(State) -> ok. %% tell truth about the original exception +-spec maybe_raise_exception(any()) -> no_return(). maybe_raise_exception(#{ exception := Exception, context := Context, From 4f00f0968d8e9403691a7b92be6bcd9814181f17 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 12 Dec 2022 18:20:01 +0300 Subject: [PATCH 12/19] ci: update to emqx-plugin-template 5.0.0 --- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 39eea5fc2..e933a36d4 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0-rc.3"). +-define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0"). -define(EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN, "0.1.0"). -define(PACKAGE_SUFFIX, ".tar.gz"). From 41a4e41d0c221b6b617e44ba5692583e7d3ba547 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 14 Dec 2022 12:39:24 +0300 Subject: [PATCH 13/19] fix(ct): prevent testcase flapping Testcase `t_log_file` was flapping when run happened to pass through boundary of a second. Archive files' `ctime`s could differ between consecutive archive downloads. --- .../test/emqx_mgmt_api_trace_SUITE.erl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index d73cb79fd..d0f22f134 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -209,20 +209,27 @@ t_log_file(_Config) -> ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)), {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header), {ok, [ - _Comment, + Comment, #zip_file{ name = ZipName, info = #file_info{size = Size, type = regular, access = read_write} } - ]} = - ZipTab = - zip:table(Binary), + ]} = zip:table(Binary), ?assert(Size > 0), ZipNamePrefix = lists:flatten(io_lib:format("~s-trace_~s", [node(), Name])), ?assertNotEqual(nomatch, re:run(ZipName, [ZipNamePrefix])), Path = api_path("trace/test_client_id/download?node=" ++ atom_to_list(node())), {ok, Binary2} = request_api(get, Path, Header), - ?assertEqual(ZipTab, zip:table(Binary2)), + ?assertMatch( + {ok, [ + Comment, + #zip_file{ + name = ZipName, + info = #file_info{size = Size, type = regular, access = read_write} + } + ]}, + zip:table(Binary2) + ), {error, {_, 400, _}, _} = request_api( get, From aece9fb96f27b5f1ac0576e82cd0c93cbeb78713 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 14 Dec 2022 16:55:29 +0300 Subject: [PATCH 14/19] ci: unset compile time vsn in dev profiles Before this commit source had been compiling with inconsistent `erl_opts` under test profile with `PROFILE` env set to `emqx`. ``` {compile_info,[{emqx_vsn,"5.0.0-beta.5-g948d4170"}]}, {d,'EMQX_RELEASE_EDITION',ce}, ``` Instead stop passing `emqx_vsn` through compile options as is expected by `emqx_release:version/0`. --- rebar.config.erl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index fe0992240..f40c86a48 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -119,7 +119,7 @@ is_win32() -> win32 =:= element(1, os:type()). project_app_dirs() -> - project_app_dirs(get_edition_from_profille_env()). + project_app_dirs(get_edition_from_profile_env()). project_app_dirs(Edition) -> ["apps/*"] ++ @@ -154,14 +154,14 @@ test_deps() -> {erl_csv, "0.2.0"} ]. -common_compile_opts(Vsn) -> - common_compile_opts(get_edition_from_profille_env(), Vsn). +common_compile_opts() -> + common_compile_opts(get_edition_from_profile_env(), undefined). common_compile_opts(Edition, Vsn) -> % always include debug_info [ debug_info, - {compile_info, [{emqx_vsn, Vsn}]}, + {compile_info, [{emqx_vsn, Vsn} || Vsn /= undefined]}, {d, 'EMQX_RELEASE_EDITION', Edition} ] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ @@ -180,7 +180,7 @@ warn_profile_env() -> end. %% this function is only used for test/check profiles -get_edition_from_profille_env() -> +get_edition_from_profile_env() -> case os:getenv("PROFILE") of "emqx-enterprise" ++ _ -> ee; @@ -243,15 +243,14 @@ profiles_ee() -> %% EE has more files than CE, always test/check with EE options. profiles_dev() -> - Vsn = get_vsn('emqx-enterprise'), [ {check, [ - {erl_opts, common_compile_opts(Vsn)}, + {erl_opts, common_compile_opts()}, {project_app_dirs, project_app_dirs()} ]}, {test, [ {deps, test_deps()}, - {erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()}, + {erl_opts, common_compile_opts() ++ erl_opts_i()}, {extra_src_dirs, [{"test", [{recursive, true}]}]}, {project_app_dirs, project_app_dirs()} ]} From ed2d5aa48a40526aae443cc7b23f869e54f40fff Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 14 Dec 2022 19:31:52 +0300 Subject: [PATCH 15/19] fix: handle status in `format_status/1` callback Which is expected argument type for this callback. Also try to make sure that random maps won't pass through this callback unnoticed. --- .../src/emqx_connector_jwt_worker.erl | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index c925aca20..cb975ca63 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -154,8 +154,8 @@ handle_info({timeout, TRef, ?refresh_jwt}, State0 = #{refresh_timer := TRef}) -> handle_info(_Msg, State) -> {noreply, State}. -format_status(State) -> - censor_secrets(State). +format_status(Status = #{state := State}) -> + Status#{state => censor_secrets(State)}. format_status(_Opt, [_PDict, State0]) -> State = censor_secrets(State0), @@ -222,16 +222,13 @@ ensure_timer(State) -> State. -spec censor_secrets(state()) -> map(). -censor_secrets(State) -> - maps:map( - fun - (Key, _Value) when - Key =:= jwt; - Key =:= jwk - -> - "******"; - (_Key, Value) -> - Value - end, - State - ). +censor_secrets(State = #{jwt := JWT, jwk := JWK}) -> + State#{ + jwt := censor_secret(JWT), + jwk := censor_secret(JWK) + }. + +censor_secret(undefined) -> + undefined; +censor_secret(_Secret) -> + "******". From 9be8c4e076f2772573c0bb92a4322e6d3a9f604d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 15 Dec 2022 12:08:03 +0300 Subject: [PATCH 16/19] ci: pin emqx/erlang-rocksdb to 1.7.2-emqx-7 Which has binaries prebuilt under OTP-25. --- mix.exs | 1 + rebar.config | 1 + 2 files changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index e58bbf28c..86bc540ca 100644 --- a/mix.exs +++ b/mix.exs @@ -52,6 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, + {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-7", override: true}, {:ekka, github: "emqx/ekka", tag: "0.13.7", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, diff --git a/rebar.config b/rebar.config index 22c6a23ed..40614a443 100644 --- a/rebar.config +++ b/rebar.config @@ -54,6 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} + , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-7"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} From f17ebb87965baeeb1f49b4a753023b7b152f763d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 16 Dec 2022 17:48:49 +0300 Subject: [PATCH 17/19] ci: cut down slim build matrix --- .github/workflows/build_slim_packages.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index ebaa17abf..c48833453 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -29,17 +29,15 @@ jobs: fail-fast: false matrix: profile: - - ["emqx", "el7"] - - ["emqx-enterprise", "ubuntu20.04"] + - ["emqx", "24.3.4.2-1", "el7"] + - ["emqx", "25.1.2-2", "ubuntu20.04"] + - ["emqx-enterprise", "24.3.4.2-1", "ubuntu20.04"] builder: - 5.0-26 - otp: - - 24.3.4.2-1 - - 25.1.2-2 elixir: - 1.13.4 - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" steps: - uses: AutoModality/action-clean@v1 @@ -75,7 +73,7 @@ jobs: ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - uses: actions/upload-artifact@v3 with: - name: "${{ matrix.profile[0] }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" + name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" path: _packages/${{ matrix.profile[0] }}/* - uses: actions/upload-artifact@v3 with: @@ -92,7 +90,6 @@ jobs: profile: - emqx otp: - - 24.3.4 - 25.1.2 steps: - uses: actions/checkout@v3 @@ -134,7 +131,6 @@ jobs: - emqx-enterprise otp: - 24.3.4.2-1 - - 25.1.2-2 os: - macos-11 - macos-12-arm64 From 501f0fc8d5b84ac043ff232c3434dbbebc8c576a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sun, 18 Dec 2022 21:37:39 +0300 Subject: [PATCH 18/19] ci: reduce test matrix size Test emqx profile under OTP-25 only. Build test matrix programmatically for that purpose. --- .github/workflows/run_test_cases.yaml | 162 +++++++++++--------------- scripts/find-apps.sh | 52 ++++----- 2 files changed, 91 insertions(+), 123 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 23adbc5bb..8f3a84c36 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -14,38 +14,60 @@ on: pull_request: jobs: + build-matrix: + runs-on: ubuntu-latest + outputs: + prepare: ${{ steps.matrix.outputs.prepare }} + host: ${{ steps.matrix.outputs.host }} + docker: ${{ steps.matrix.outputs.docker }} + runs-on: ${{ steps.runner.outputs.runs-on }} + steps: + - uses: actions/checkout@v3 + - name: Build matrix + id: matrix + run: | + APPS="$(./scripts/find-apps.sh --ci)" + MATRIX="$(echo "${APPS}" | jq -c ' + [ + (.[] | select(.profile == "emqx") | . + { + builder: "5.0-26", + otp: "25.1.2-2", + elixir: "1.13.4" + }), + (.[] | select(.profile == "emqx-enterprise") | . + { + builder: "5.0-26", + otp: ["24.3.4.2-1", "25.1.2-2"][], + elixir: "1.13.4" + }) + ] + ')" + echo "${MATRIX}" | jq + MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT + echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT + echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT + - name: Choose runner host + id: runner + run: | + RUNS_ON="ubuntu-20.04" + ${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64" + echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT + prepare: runs-on: aws-amd64 + needs: [build-matrix] strategy: fail-fast: false matrix: - profile: - - emqx - - emqx-enterprise - builder: - - 5.0-26 - otp: - - 24.3.4.2-1 - - 25.1.2-2 - elixir: - - 1.13.4 + include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" - outputs: - fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }} - docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }} steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 with: path: source - - name: Find CT Apps - working-directory: source - id: find_ct_apps - run: | - fast_ct_apps="$(./scripts/find-apps.sh --ci fast)" - docker_ct_apps="$(./scripts/find-apps.sh --ci docker)" - echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT - echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT - name: get_all_deps working-directory: source env: @@ -64,31 +86,14 @@ jobs: path: source.zip eunit_and_proper: - needs: [prepare] - runs-on: ${{ matrix.runs-on }} + needs: + - build-matrix + - prepare + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - profile: - - emqx - - emqx-enterprise - builder: - - 5.0-26 - otp: - - 24.3.4.2-1 - - 25.1.2-2 - elixir: - - 1.13.4 - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false + include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} defaults: run: @@ -123,30 +128,15 @@ jobs: path: source/_build/test/cover ct_docker: - needs: [prepare] + needs: + - build-matrix + - prepare + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - builder: - - 5.0-26 - otp: - - 24.3.4.2-1 - - 25.1.2-2 - elixir: - - 1.13.4 - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false + include: ${{ fromJson(needs.build-matrix.outputs.docker) }} - runs-on: ${{ matrix.runs-on }} defaults: run: shell: bash @@ -155,7 +145,7 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }}-${{ matrix.otp }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip @@ -168,12 +158,8 @@ jobs: PGSQL_TAG: 13 REDIS_TAG: 6 INFLUXDB_TAG: 2.5.0 - WHICH_APP: ${{ matrix.app[0] }} - PROFILE: ${{ matrix.app[1] }} - run: | - echo $PROFILE - rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt - ./scripts/ct/run.sh --app $WHICH_APP + PROFILE: ${{ matrix.profile }} + run: ./scripts/ct/run.sh --app ${{ matrix.app }} - uses: actions/upload-artifact@v3 with: name: coverdata @@ -181,34 +167,19 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }} path: source/_build/test/logs ct: - needs: [prepare] + needs: + - build-matrix + - prepare + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} - builder: - - 5.0-26 - otp: - - 24.3.4.2-1 - - 25.1.2-2 - elixir: - - 1.13.4 - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false + include: ${{ fromJson(needs.build-matrix.outputs.host) }} - runs-on: ${{ matrix.runs-on }} container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" defaults: run: @@ -218,7 +189,7 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }}-${{ matrix.otp }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip @@ -227,10 +198,9 @@ jobs: - name: run common test working-directory: source env: - WHICH_APP: ${{ matrix.app[0] }} - PROFILE: ${{ matrix.app[1] }} + PROFILE: ${{ matrix.profile }} run: | - make "${WHICH_APP}-ct" + make "${{ matrix.app }}-ct" - uses: actions/upload-artifact@v3 with: name: coverdata @@ -239,7 +209,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }} path: source/_build/test/logs make_cover: diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index a92c56748..f07cd2f7d 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -8,10 +8,10 @@ cd -P -- "$(dirname -- "$0")/.." help() { echo echo "-h|--help: To display this usage info" - echo "--ci fast|docker: Print apps in json format for github ci mtrix" + echo "--ci: Print apps in json format for github ci matrix" } -CI='novalue' +MODE='list' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -19,8 +19,8 @@ while [ "$#" -gt 0 ]; do exit 0 ;; --ci) - CI="$2" - shift 2 + MODE='ci' + shift 1 ;; *) echo "unknown option $1" @@ -45,7 +45,7 @@ CE="$(find_app 'apps')" EE="$(find_app 'lib-ee')" APPS_ALL="$(echo -e "${CE}\n${EE}")" -if [ "$CI" = 'novalue' ]; then +if [ "$MODE" = 'list' ]; then echo "${APPS_ALL}" exit 0 fi @@ -54,16 +54,22 @@ fi ###### now deal with the github action's matrix. ################################################## -dimensions() { +format_app_description() { + ## prefix is for github actions (they don't like slash in variables) + local prefix=${1//\//_} + echo -n -e "$( +cat <, , apps_ - ## third one is for github actions (they don't like slash in variables) - echo -n -e "[\"$app\", \"$profile\", \"${app//\//_}\"]" + format_app_description "$app" "$profile" "$runner" } matrix() { - first_row='yes' + local sep='[' for app in ${APPS_ALL}; do - row="$(dimensions "$app")" + row="$(describe_app "$app")" if [ -z "$row" ]; then continue fi - if [ "$first_row" = 'yes' ]; then - first_row='no' - echo -n "$row" - else - echo -n ",${row}" - fi + echo -n "${sep}${row}" + sep=', ' done + echo ']' } -echo -n '[' matrix -echo ']' From e267b3c89e1e926e77652c939f645fbe1fea09b9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sun, 18 Dec 2022 21:42:15 +0300 Subject: [PATCH 19/19] ci: avoid coverdata filename collisions --- .github/workflows/run_test_cases.yaml | 4 ++++ Makefile | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8f3a84c36..4ee8efa2e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -112,6 +112,7 @@ jobs: - name: eunit env: PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} working-directory: source run: make eunit @@ -119,6 +120,7 @@ jobs: - name: proper env: PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} working-directory: source run: make proper @@ -159,6 +161,7 @@ jobs: REDIS_TAG: 6 INFLUXDB_TAG: 2.5.0 PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --app ${{ matrix.app }} - uses: actions/upload-artifact@v3 with: @@ -199,6 +202,7 @@ jobs: working-directory: source env: PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: | make "${{ matrix.app }}-ct" - uses: actions/upload-artifact@v3 diff --git a/Makefile b/Makefile index cb26a2529..9a02fcd08 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default CT_NODE_NAME ?= 'test@127.0.0.1' CT_READABLE ?= true +CT_COVER_EXPORT_PREFIX ?= $(PROFILE) export REBAR_GIT_CLONE_OPTIONS += --depth=1 @@ -62,7 +63,7 @@ mix-deps-get: $(ELIXIR_COMMON_DEPS) .PHONY: eunit eunit: $(REBAR) merge-config - @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(PROFILE)-eunit + @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(CT_COVER_EXPORT_PREFIX)-eunit .PHONY: proper proper: $(REBAR) @@ -74,7 +75,7 @@ test-compile: $(REBAR) merge-config .PHONY: ct ct: $(REBAR) merge-config - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-ct + @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(CT_COVER_EXPORT_PREFIX)-ct .PHONY: static_checks static_checks: @@ -86,7 +87,10 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh) define gen-app-ct-target $1-ct: $(REBAR) @$(SCRIPTS)/pre-compile.sh $(PROFILE) - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ + --name $(CT_NODE_NAME) \ + --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ + --suite $(shell $(SCRIPTS)/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))