From 419b80d11e7f449a9168fd1490bac7670d8efc2f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 12 Oct 2023 01:26:35 +0700 Subject: [PATCH 1/7] chore(ci): try to reduce CI artifacts footprint To speed up uploads and downloads in dependent jobs. --- .github/workflows/.zipignore | 3 +++ .github/workflows/_pr_entrypoint.yaml | 5 ++--- .github/workflows/_push-entrypoint.yaml | 2 +- .github/workflows/run_conf_tests.yaml | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/.zipignore diff --git a/.github/workflows/.zipignore b/.github/workflows/.zipignore new file mode 100644 index 000000000..d72c7ba23 --- /dev/null +++ b/.github/workflows/.zipignore @@ -0,0 +1,3 @@ +.git/* +*/.git/* +*/.github/* diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index f2688d8d1..8fc19c1aa 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -138,9 +138,8 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make ensure-rebar3 - make ${PROFILE} - make test-compile - zip -ryq $PROFILE.zip . + make ${PROFILE}-compile test-compile + zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index a8a01fb45..19c25fa2d 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -149,7 +149,7 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make $PROFILE - zip -ryq $PROFILE.zip . + zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index a69746d76..813aa663b 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -34,6 +34,7 @@ jobs: run: | unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: make ${{ matrix.profile }} - run: ./scripts/test/check-example-configs.sh - run: ./scripts/conf-test/run.sh - name: print erlang log From fdd9d77d41866a55b510a7f60c8b875be4e7f36b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 12 Oct 2023 17:37:57 +0700 Subject: [PATCH 2/7] chore(ci): simplify `check-i18n-style` + stop relying on git --- scripts/check-i18n-style.escript | 3 +-- scripts/check-i18n-style.sh | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index f48e5a513..e7e0ea42e 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -8,10 +8,9 @@ -define(RED, "\e[31m"). -define(RESET, "\e[39m"). -main([Files0]) -> +main(Files) -> io:format(user, "checking i18n file styles~n", []), _ = put(errors, 0), - Files = string:tokens(Files0, "\n"), ok = load_hocon(), ok = lists:foreach(fun check/1, Files), case get(errors) of diff --git a/scripts/check-i18n-style.sh b/scripts/check-i18n-style.sh index d21f43a72..b7d4d2113 100755 --- a/scripts/check-i18n-style.sh +++ b/scripts/check-i18n-style.sh @@ -3,6 +3,4 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")/.." -all_files="$(git ls-files 'rel/i18n/*.hocon')" - -./scripts/check-i18n-style.escript "$all_files" +./scripts/check-i18n-style.escript rel/i18n/*.hocon From 397686fd183dc892a4163c7f51e1efcb4f426532 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 12 Oct 2023 17:54:56 +0700 Subject: [PATCH 3/7] fix(ci-conf): stop requiring git index in conf tests --- .github/workflows/_pr_entrypoint.yaml | 2 ++ .github/workflows/run_conf_tests.yaml | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 8fc19c1aa..dd2edfaa9 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -217,6 +217,8 @@ jobs: with: runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} + version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} + version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} check_deps_integrity: needs: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 813aa663b..c91291836 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -13,6 +13,12 @@ on: builder: required: true type: string + version-emqx: + required: false + type: string + version-emqx-enterprise: + required: false + type: string jobs: run_conf_tests: @@ -20,6 +26,7 @@ jobs: container: ${{ inputs.builder }} env: PROFILE: ${{ matrix.profile }} + PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} strategy: fail-fast: false matrix: From d0dac25644b4f52353b592254292566807a2e50c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 13 Oct 2023 00:10:46 +0700 Subject: [PATCH 4/7] chore(ci): stop depending on git in `check-example-configs.sh` --- scripts/test/check-example-configs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/test/check-example-configs.sh b/scripts/test/check-example-configs.sh index f71fb15eb..cffea24ce 100755 --- a/scripts/test/check-example-configs.sh +++ b/scripts/test/check-example-configs.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -PROJ_DIR="$(git rev-parse --show-toplevel)" + +cd -P -- "$(dirname -- "$0")/../.." PROFILE="${PROFILE:-emqx}" DIR_NAME='examples' @@ -11,7 +12,7 @@ if [ "${PROFILE}" = 'emqx-enterprise' ]; then SCHEMA_MOD='emqx_enterprise_schema' fi -IFS=$'\n' read -r -d '' -a FILES < <(find "${PROJ_DIR}/rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') +IFS=$'\n' read -r -d '' -a FILES < <(find "rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') prepare_erl_libs() { local libs_dir="$1" @@ -30,7 +31,7 @@ prepare_erl_libs() { } # This is needed when checking schema -export EMQX_ETC_DIR="${PROJ_DIR}/apps/emqx/etc" +export EMQX_ETC_DIR="apps/emqx/etc" prepare_erl_libs "_build/$PROFILE/lib" From 08b2b36b87f2bcf8bb570080d6bbb712d906254b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 13 Oct 2023 00:40:25 +0700 Subject: [PATCH 5/7] feat(ci): push some envs through artifact files in workflows So that we could optimize away the need for the full git history. --- .github/workflows/_pr_entrypoint.yaml | 4 ++-- .github/workflows/_push-entrypoint.yaml | 2 ++ .github/workflows/run_conf_tests.yaml | 10 +--------- .github/workflows/static_checks.yaml | 3 +-- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index dd2edfaa9..930182c7e 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -139,6 +139,8 @@ jobs: run: | make ensure-rebar3 make ${PROFILE}-compile test-compile + echo "PROFILE=${PROFILE}" | tee -a .env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: @@ -217,8 +219,6 @@ jobs: with: runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} - version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }} - version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }} check_deps_integrity: needs: diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 19c25fa2d..b2d7e2c4a 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -149,6 +149,8 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make $PROFILE + echo "PROFILE=${PROFILE}" | tee -a .env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index c91291836..788c9c9fa 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -13,20 +13,11 @@ on: builder: required: true type: string - version-emqx: - required: false - type: string - version-emqx-enterprise: - required: false - type: string jobs: run_conf_tests: runs-on: ${{ fromJSON(inputs.runner_labels) }} container: ${{ inputs.builder }} - env: - PROFILE: ${{ matrix.profile }} - PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} strategy: fail-fast: false matrix: @@ -41,6 +32,7 @@ jobs: run: | unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: cat .env | tee -a $GITHUB_ENV - run: make ${{ matrix.profile }} - run: ./scripts/test/check-example-configs.sh - run: ./scripts/conf-test/run.sh diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index 758c3712b..6cf89275c 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -43,7 +43,6 @@ jobs: key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }} restore-keys: | rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- + - run: cat .env | tee -a $GITHUB_ENV - name: run static checks - env: - PROFILE: ${{ matrix.profile }} run: make static_checks From 0b9ac24c1e69ebe74ab6483ea431a4d60f151a2b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 13 Oct 2023 15:40:49 +0700 Subject: [PATCH 6/7] fix(bpapicheck): stop depending on git index --- apps/emqx/test/emqx_bpapi_static_checks.erl | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 6766912c0..0be5895f6 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -411,10 +411,19 @@ setnok() -> put(bpapi_ok, false). dumps_dir() -> - filename:join(project_root_dir(), "apps/emqx/test/emqx_static_checks_data"). - -project_root_dir() -> - string:trim(os:cmd("git rev-parse --show-toplevel")). + filename:join(emqx_app_dir(), "test/emqx_static_checks_data"). versions_file() -> - filename:join(project_root_dir(), "apps/emqx/priv/bpapi.versions"). + filename:join(emqx_app_dir(), "priv/bpapi.versions"). + +emqx_app_dir() -> + Info = ?MODULE:module_info(compile), + case proplists:get_value(source, Info) of + Source when is_list(Source) -> + filename:dirname(filename:dirname(Source)); + undefined -> + "apps/emqx" + end. + +project_root_dir() -> + filename:dirname(filename:dirname(emqx_app_dir())). From 083e2da3478f1dd4549a851a33ed7f4d5088ab7f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 13 Oct 2023 15:42:08 +0700 Subject: [PATCH 7/7] chore(bpapicheck): make some failures more user-friendly --- apps/emqx/test/emqx_bpapi_static_checks.erl | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 0be5895f6..657776317 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -244,19 +244,28 @@ get_param_types(Signatures, {M, F, A}) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% dump() -> - case - { - filelib:wildcard(project_root_dir() ++ "/*_plt"), - filelib:wildcard(project_root_dir() ++ "/_build/check/lib") - } - of + RootDir = project_root_dir(), + TryRelDir = RootDir ++ "/_build/check/lib", + case {filelib:wildcard(RootDir ++ "/*_plt"), filelib:wildcard(TryRelDir)} of {[PLT | _], [RelDir | _]} -> dump(#{ plt => PLT, reldir => RelDir }); - _ -> - error("failed to guess run options") + {[], _} -> + logger:error( + "No usable PLT files found in \"~s\", abort ~n" + "Try running `rebar3 as check dialyzer` at least once first", + [RootDir] + ), + error(run_failed); + {_, []} -> + logger:error( + "No built applications found in \"~s\", abort ~n" + "Try running `rebar3 as check compile` at least once first", + [TryRelDir] + ), + error(run_failed) end. %% Collect the local BPAPI modules to a dump file