From 5242a6d3932e9e74759d644e484f91593602775e Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 28 Oct 2022 16:38:59 +0800 Subject: [PATCH 1/8] chore: bump otp version --- .github/workflows/apps_version_check.yaml | 4 ++-- .github/workflows/build_packages.yaml | 18 +++++++++--------- .github/workflows/build_slim_packages.yaml | 8 ++++---- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 14 +++++++------- .github/workflows/run_test_cases.yaml | 2 +- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 37f03b332..7875d3001 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,11 +9,11 @@ jobs: strategy: matrix: erl_otp: - - 24.1.5-3 + - 24.3.4.2-1 os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 9954d1ee6..04d7074b3 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -21,7 +21,7 @@ jobs: if: endsWith(github.repository, 'emqx') runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 outputs: profiles: ${{ steps.detect-profiles.outputs.profiles}} @@ -60,7 +60,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.2.1 + - 24.3.4.2 exclude: - profile: emqx-edge steps: @@ -108,7 +108,7 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 os: - macos-11 runs-on: ${{ matrix.os }} @@ -153,7 +153,7 @@ jobs: - zip - pkg otp: - - 24.1.5-3 + - 24.3.4.2-1 arch: - amd64 - arm64 @@ -210,7 +210,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: name: ${{ matrix.profile }}-${{ matrix.otp }} @@ -225,7 +225,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.1.5-3 + - 24.3.4.2-1 registry: - 'docker.io' - 'public.ecr.aws' @@ -286,7 +286,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -302,7 +302,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise @@ -320,7 +320,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 24.1.5-3 + - 24.3.4.2-1 include: - profile: emqx otp: windows # otp version on windows is rather fixed diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 9ef69ff70..815b2e60e 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 os: - ubuntu20.04 - el8 @@ -32,7 +32,7 @@ jobs: - runs-on: aws-amd64 use-self-hosted: false - container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -78,7 +78,7 @@ jobs: profile: - emqx otp: - - 24.2.1 + - 24.3.4.2 steps: - uses: actions/checkout@v2 - uses: ilammy/msvc-dev-cmd@v1 @@ -111,7 +111,7 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 os: - macos-11 runs-on: ${{ matrix.os }} diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 433b00d71..7db081000 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6890f9567..6f3c829a9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 outputs: profiles: ${{ steps.detect-profiles.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 6ebe37f98..65078ca65 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 7c43f38c4..be9f0ea23 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -200,7 +200,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -249,9 +249,9 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.3.4.2-1 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 defaults: run: shell: bash @@ -286,13 +286,13 @@ jobs: - relup_test_plan - relup_test_build runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 strategy: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} otp: - - 24.1.5-3 + - 24.3.4.2-1 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -331,8 +331,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="24.1.5-3" \ - --var TO_OTP_VSN="24.1.5-3" \ + --var FROM_OTP_VSN="24.3.4.2-1" \ + --var TO_OTP_VSN="24.3.4.2-1" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 5f98f6940..8bff3a33c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -12,7 +12,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v2 From 03cf2da2568133d97eecea6486190c2f39949387 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 28 Oct 2022 15:34:58 +0200 Subject: [PATCH 2/8] chore: enable OTP upgrade for 4.4 --- build | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build b/build index f5bf13a58..7577d2219 100755 --- a/build +++ b/build @@ -77,8 +77,13 @@ make_relup() { local zip_file mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() + local OTP_CHANGED='no' if [ -d "$releases_dir" ]; then for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do + OTP_BASE=$(relup_db otp-vsn-for "$BASE_VSN") + if [[ "$OTP_BASE" != "$OTP_VSN" ]]; then + OTP_CHANGED='yes' + fi OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip" if [ ! -d "$releases_dir/$BASE_VSN" ]; then @@ -99,8 +104,8 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS - if [[ ${PKG_VSN} == 4.3* ]]; then - echo "EMQX 4.3 specific, overwrite OTP app versions" + if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then + echo "EMQX 4.[3,4] specific, overwrite OTP app versions" local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" if [ ! -f "${emqx_rel_file}" ]; then ./rebar3 as "${PROFILE}" release @@ -118,7 +123,7 @@ make_relup() { # rollback rel file per releases # - if [[ ${PKG_VSN} == 4.3* ]]; then + if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then echo "restore upgrade base rel files... " for rel in ${releases[*]}; do From 8bda9dd05506b23d050eec598bfe10d69bbcd19e Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 28 Oct 2022 17:22:19 +0200 Subject: [PATCH 3/8] docs: upgrade OTP to 24.3.4.2-1 --- changes/v4.4.11-en.md | 3 +++ changes/v4.4.11-zh.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/changes/v4.4.11-en.md b/changes/v4.4.11-en.md index bc0ee9bb9..671c8a5a2 100644 --- a/changes/v4.4.11-en.md +++ b/changes/v4.4.11-en.md @@ -1,5 +1,8 @@ ### Enhancements +- OTP upgrade from 24.1.5-3 to 24.3.4.2-1 +[#9265](https://github.com/emqx/emqx/pull/9265) + ### Bug fixes - Fix get trace list crash when trace not initialize. [#9156](https://github.com/emqx/emqx/pull/9156) diff --git a/changes/v4.4.11-zh.md b/changes/v4.4.11-zh.md index f27d12d59..fb9fda782 100644 --- a/changes/v4.4.11-zh.md +++ b/changes/v4.4.11-zh.md @@ -1,5 +1,7 @@ ### 增强 +- OTP 升级: 从 24.1.5-3 至 24.3.4.2-1 +[#9265](https://github.com/emqx/emqx/pull/9265) ### 修复 From 7b14ea92c38747dc7125fc5f154420b9a8feadd7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 30 Oct 2022 21:53:25 +0100 Subject: [PATCH 4/8] chore: update relup-paths.eterm --- data/relup-paths.eterm | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/data/relup-paths.eterm b/data/relup-paths.eterm index 39294e3ad..3bbfbcb79 100644 --- a/data/relup-paths.eterm +++ b/data/relup-paths.eterm @@ -39,6 +39,13 @@ [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>], otp => <<"24.1.5-3">>}}. -{<<"4.5.0">>, - #{from_versions => [<<"4.4.10">>,<<"4.4.8">>,<<"4.4.9">>], +{<<"4.4.11">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>, + <<"4.4.10">>], + otp => <<"24.3.4.2-1">>}}. +{<<"4.5.0">>, + #{from_versions => [<<"4.4.8">>,<<"4.4.9">>,<<"4.4.10">>, + <<"4.4.11">>], otp => <<"24.3.4.2-1">>}}. From d33b00bfc7926d7cf06b112513e904b5f3fee432 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 31 Oct 2022 10:54:41 +0100 Subject: [PATCH 5/8] ci(relup_test): Get OTP base vsn from relup-paths.eterm --- .github/workflows/run_fvt_tests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index be9f0ea23..2add87ac0 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -280,6 +280,8 @@ jobs: path: | emqx/_packages/*/*.zip emqx/.ci/fvt_tests + emqx/data/relup-paths.eterm + emqx/scripts/relup-base-vsns.escript relup_test_run: needs: @@ -291,8 +293,6 @@ jobs: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} - otp: - - 24.3.4.2-1 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -318,8 +318,10 @@ jobs: set -e -x -u mkdir -p packages cp emqx_built/_packages/*/*.zip packages + old_emqx_vsn=${OLD_VSN#[e|v]} + old_otp_vsn=$(escript emqx_built/scripts/relup-base-vsns.escript otp-vsn-for "${old_emqx_vsn}" emqx_built/data/relup-paths.eterm) cd packages - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${old_emqx_vsn}-otp${old_otp_vsn}-ubuntu20.04-amd64.zip - name: Run relup test scenario timeout-minutes: 5 run: | From 23569198cf87a63bff46845992d06e2ddc4dabb0 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 31 Oct 2022 14:55:20 +0100 Subject: [PATCH 6/8] ci(relup-test): merge two jobs --- .github/workflows/run_fvt_tests.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 2add87ac0..2750be7f4 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -313,18 +313,14 @@ jobs: repository: terry-xiaoyu/one_more_emqx ref: master path: one_more_emqx - - name: Prepare packages + - name: Run relup test scenario + timeout-minutes: 6 run: | set -e -x -u - mkdir -p packages - cp emqx_built/_packages/*/*.zip packages old_emqx_vsn=${OLD_VSN#[e|v]} old_otp_vsn=$(escript emqx_built/scripts/relup-base-vsns.escript otp-vsn-for "${old_emqx_vsn}" emqx_built/data/relup-paths.eterm) - cd packages - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${old_emqx_vsn}-otp${old_otp_vsn}-ubuntu20.04-amd64.zip - - name: Run relup test scenario - timeout-minutes: 5 - run: | + wget --no-verbose -P packages https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${old_emqx_vsn}-otp${old_otp_vsn}-ubuntu20.04-amd64.zip + cp emqx_built/_packages/*/*.zip packages lux \ --progress verbose \ --case_timeout infinity \ @@ -333,7 +329,7 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="24.3.4.2-1" \ + --var FROM_OTP_VSN="${old_otp_vsn}" \ --var TO_OTP_VSN="24.3.4.2-1" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 From dde4a28f3130be34edb6ec6fb66111a91f7c7f72 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 31 Oct 2022 17:14:35 +0100 Subject: [PATCH 7/8] docs: Update changes/v4.4.11-en.md Co-authored-by: Zaiming (Stone) Shi --- changes/v4.4.11-en.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/changes/v4.4.11-en.md b/changes/v4.4.11-en.md index 671c8a5a2..55507c981 100644 --- a/changes/v4.4.11-en.md +++ b/changes/v4.4.11-en.md @@ -1,7 +1,10 @@ ### Enhancements -- OTP upgrade from 24.1.5-3 to 24.3.4.2-1 -[#9265](https://github.com/emqx/emqx/pull/9265) +- OTP upgrade from 24.1.5-3 to 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265). + Change highlights: + - Erlang/OTP [SSL library vulnerability fix](https://nvd.nist.gov/vuln/detail/CVE-2022-37026) + - Added support for OCSP (Online Certificate Status Protocol) Stapling + - Added CRL (Certificate Revocation List) cache auto refresh ### Bug fixes From f36fd9f3dbb1f0838d0e9648027aed2fff61a00f Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 1 Nov 2022 09:57:11 +0100 Subject: [PATCH 8/8] docs: update changes/v4.4.11-zh.md Co-authored-by: Zaiming (Stone) Shi --- changes/v4.4.11-zh.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/changes/v4.4.11-zh.md b/changes/v4.4.11-zh.md index fb9fda782..82c4558e1 100644 --- a/changes/v4.4.11-zh.md +++ b/changes/v4.4.11-zh.md @@ -1,8 +1,11 @@ ### 增强 -- OTP 升级: 从 24.1.5-3 至 24.3.4.2-1 -[#9265](https://github.com/emqx/emqx/pull/9265) - +- OTP 升级: 从 24.1.5-3 至 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265)。 + 重要更新: + - Erlang/OTP [SSL库漏洞修复](https://nvd.nist.gov/vuln/detail/CVE-2022-37026) + - 增加了对 OCSP (Online Certificate Status Protocol) Stapling 的支持 + - 增加了 CRL（证书吊销列表）缓存的自动刷新功能 + ### 修复 - 修复日志追踪模块没开启时，GET Trace 列表接口报错的问题。[#9156](https://github.com/emqx/emqx/pull/9156)