From 68d28fdcdd6fa44ea7f782afd3b52b02ababa599 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 17 Sep 2022 14:50:46 +0200 Subject: [PATCH 1/5] build(update-appup.sh): fix usage info --- scripts/update-appup.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 788c25ab0..9294dbb3f 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -7,12 +7,20 @@ set -euo pipefail -usage() { - echo "$0 PROFILE" -} # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +usage() { + echo "$0 PROFILE [options]" + echo "options:" + echo "--skip-build: Skip building the profile only to re-generate the appup files." + echo "--skip-build-base: This script by default forces a git clean before rebuilding on the base version " + echo " this option is useful when you are sure the past builds can be trusted," + echo " that is, there were no re-tags or anything." + echo "--check: Exit with non-zero code if there is git diff after the execution." + echo " Mostly used in CI." +} + PROFILE="${1:-}" case "$PROFILE" in emqx-ee) @@ -48,7 +56,7 @@ ESCRIPT_ARGS=( '' ) while [ "$#" -gt 0 ]; do case $1 in -h|--help) - help + usage exit 0 ;; --skip-build) @@ -100,7 +108,7 @@ else pushd "${PREV_DIR_BASE}/${PREV_TAG}" if [ "$NEW_COPY" = 'no' ]; then REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')" - git fetch "$REMOTE" + git fetch "$REMOTE" --tags -f fi git reset --hard git clean -ffdx From 7c4842e6e59d86ad879bf6b1dc1d23ad90a4b46e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Sep 2022 08:57:53 -0300 Subject: [PATCH 2/5] test: attempt to fix inter-suite flakiness Ex: https://github.com/emqx/emqx-enterprise/actions/runs/3124750818/jobs/5068407612#step:7:769 ``` %%% undefined ==> end_per_suite: FAILED %%% undefined ==> {{badmatch,{error,enoent}}, [{emqx_auth_mongo_SUITE,end_per_suite,1, [{file,"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl"}, {line,62}]}, {test_server,ts_tc,3,[{file,"test_server.erl"},{line,1784}]}, {test_server,run_test_case_eval1,6,[{file,"test_server.erl"},{line,1381}]}, {test_server,run_test_case_eval,9,[{file,"test_server.erl"},{line,1225}]}]} Testing lib.emqx_auth_mongo: TEST COMPLETE, 3 ok, 0 failed of 3 test cases ``` --- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 1765b3821..77a11e213 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -115,7 +115,7 @@ init_per_suite(Config) -> end_per_suite(_Cfg) -> deinit_mongo_data(), %% avoid inter-suite flakiness - ok = emqx_mod_acl_internal:load([]), + emqx_mod_acl_internal:load([]), emqx_ct_helpers:stop_apps([emqx_auth_mongo]). set_special_confs(emqx) -> From a61c97ed9bfb647bebed42a4722b8081c1744fdc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Sep 2022 09:13:07 -0300 Subject: [PATCH 3/5] test: attempt to reduce inter-testcase flakiness in CI Ex: ``` === ERROR! init_per_testcase crashed! Location: [{emqx_auth_mongo_SUITE,'-init_mongo_data/0-fun-0-',207}, {emqx_auth_mongo_SUITE,init_mongo_data,207}, {emqx_auth_mongo_SUITE,init_per_testcase,177}, {test_server,do_init_per_testcase,1554}, {test_server,run_test_case_eval1,1255}, {test_server,run_test_case_eval,1225}] Reason: {{assertMatch,[{module,emqx_auth_mongo_SUITE}, {line,207}, {expression,"mongo_api : insert ( Connection , ? MONGO_CL_USER , ? INIT_AUTH )"}, {pattern,"{ { true , _ } , _ }"}, {value,{error,timeout}}]}, [{emqx_auth_mongo_SUITE,'-init_mongo_data/0-fun-0-',1, [{file,"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl"}, {line,207}]}, {emqx_auth_mongo_SUITE,init_mongo_data,0, [{file,"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl"}, {line,207}]}, {emqx_auth_mongo_SUITE,init_per_testcase,2, [{file,"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl"}, {line,177}]}, {test_server,do_init_per_testcase,2,[{file,"test_server.erl"},{line,1554}]}, {test_server,run_test_case_eval1,6,[{file,"test_server.erl"},{line,1255}]}, {test_server,run_test_case_eval,9,[{file,"test_server.erl"},{line,1225}]}]} ``` --- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 77a11e213..0031bc8c4 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -186,6 +186,8 @@ end_per_testcase(TestCase, Config) when TestCase =:= t_available_acl_query_timeout; TestCase =:= t_acl_superuser_timeout; TestCase =:= t_authn_no_connection; + TestCase =:= t_available_authn_query_timeout; + TestCase =:= t_authn_timeout; TestCase =:= t_available_acl_query_no_connection -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), From 5c2edaec73df986b1a921f6bcb15732f3931827f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 28 Sep 2022 18:20:01 +0200 Subject: [PATCH 4/5] build: fix profile detection when building packages --- .github/actions/package-macos/action.yaml | 13 +++++-------- .github/workflows/build_packages.yaml | 12 ++++++------ .github/workflows/build_slim_packages.yaml | 9 +++------ 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 5c0649b17..c8ebfd795 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -1,8 +1,5 @@ name: 'Create MacOS package' inputs: - profile: # emqx, emqx-enterprise - required: true - type: string otp: # 24.2.1-1, 23.3.4.9-3 required: true type: string @@ -49,7 +46,7 @@ runs: kerl update releases kerl build ${{ inputs.otp }} kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }} - - name: build ${{ inputs.profile }} + - name: build env: AUTO_INSTALL_BUILD_DEPS: 1 APPLE_SIGN_BINARIES: 1 @@ -64,12 +61,12 @@ runs: . $HOME/.kerl/${{ inputs.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 - make ${{ inputs.profile }}-zip - - name: test ${{ inputs.profile }} + make ${EMQX_NAME}-zip + - name: test shell: bash run: | - pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip) - unzip -q _packages/${{ inputs.profile }}/$pkg_name + pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip) + unzip -q _packages/${EMQX_NAME}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f2024fb5b..c64af4f93 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -120,11 +120,8 @@ jobs: strategy: matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 23.3.4.9-3 - exclude: - - profile: emqx-edge os: - macos-11 runs-on: ${{ matrix.os }} @@ -139,9 +136,12 @@ jobs: ln -s . source unzip -q source.zip rm source source.zip + - id: detect-profiles + uses: ./.github/actions/detect-profiles + with: + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - uses: ./.github/actions/package-macos with: - profile: ${{ matrix.profile }} otp: ${{ matrix.otp }} os: ${{ matrix.os }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} @@ -150,8 +150,8 @@ jobs: apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v1 with: - name: ${{ matrix.profile }}-${{ matrix.otp }} - path: _packages/${{ matrix.profile }}/. + name: ${EMQX_NAME}-${{ matrix.otp }} + path: _packages/${EMQX_NAME}/. linux: runs-on: ubuntu-20.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index c0bba8c96..a13c11f11 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -11,10 +11,10 @@ jobs: strategy: matrix: otp: - - erl23.3.4.9-3 + - erl23.3.4.9-3 os: - - ubuntu20.04 - - centos7 + - ubuntu20.04 + - centos7 runs-on: - aws-amd64 - ubuntu-20.04 @@ -65,8 +65,6 @@ jobs: mac: strategy: matrix: - profile: - - emqx otp: - 23.3.4.9-3 os: @@ -79,7 +77,6 @@ jobs: ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - uses: ./.github/actions/package-macos with: - profile: ${{ matrix.profile }} otp: ${{ matrix.otp }} os: ${{ matrix.os }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} From 747c48c3ce8e9b32d99a0ce78bc2338d0e9485bc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 21 Sep 2022 15:15:24 +0200 Subject: [PATCH 5/5] build: merge release- branch back to main- branch asap --- scripts/rel/sync-remotes.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index 1e5e21fc4..c79e7b957 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -170,27 +170,29 @@ remote_refs() { upstream_branches() { local base="$1" case "$base" in - release-v43|main-v4.3) - ## no upstream for 4.3 opensource + release-v43) remote_ref "$base" ;; release-v44) remote_refs "$base" 'release-v43' ;; - main-v4.4) - remote_refs "$base" 'main-v4.3' - ;; release-e43) remote_refs "$base" 'release-v43' ;; - main-v4.3-enterprise) - remote_refs "$base" 'main-v4.3' - ;; release-e44) remote_refs "$base" 'release-v44' 'release-e43' 'release-v43' ;; + main-v4.3) + remote_refs "$base" 'release-v43' + ;; + main-v4.4) + remote_refs "$base" 'release-v44' 'main-v4.3' + ;; + main-v4.3-enterprise) + remote_refs "$base" 'release-e43' 'main-v4.3' + ;; main-v4.4-enterprise) - remote_refs "$base" 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3' + remote_refs "$base" 'release-e44' 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3' ;; esac }