Merge pull request #9265 from qzhuyan/upgrade/william/vsn-4.4-otp-upgrade

Upgrade/william/vsn 4.4 otp upgrade

.github/workflows/apps_version_check.yaml

@@ -9,11 +9,11 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 24.1.5-3
+        - 24.3.4.2-1
         os:
         - ubuntu20.04
 
-    container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/build_packages.yaml

@@ -21,7 +21,7 @@ jobs:
     if: endsWith(github.repository, 'emqx')
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
     outputs:
       profiles: ${{ steps.detect-profiles.outputs.profiles}}
 
@@ -60,7 +60,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.2.1
+          - 24.3.4.2
         exclude:
           - profile: emqx-edge
     steps:
@@ -108,7 +108,7 @@ jobs:
       fail-fast: false
       matrix:
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         os:
           - macos-11
     runs-on: ${{ matrix.os }}
@@ -153,7 +153,7 @@ jobs:
           - zip
           - pkg
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         arch:
           - amd64
           - arm64
@@ -210,7 +210,7 @@ jobs:
           --profile "${PROFILE}" \
           --pkgtype "${PACKAGE}" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}"
+          --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}"
     - uses: actions/upload-artifact@v1
       with:
         name: ${{ matrix.profile }}-${{ matrix.otp }}
@@ -225,7 +225,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         registry:
           - 'docker.io'
           - 'public.ecr.aws'
@@ -286,7 +286,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
@@ -302,7 +302,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise
@@ -320,7 +320,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         include:
           - profile: emqx
             otp: windows # otp version on windows is rather fixed

.github/workflows/build_slim_packages.yaml

@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         os:
           - ubuntu20.04
           - el8
@@ -32,7 +32,7 @@ jobs:
           - runs-on: aws-amd64
             use-self-hosted: false
 
-    container: ghcr.io/emqx/emqx-builder/4.4-19:${{ matrix.otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v1
@@ -78,7 +78,7 @@ jobs:
         profile:
           - emqx
         otp:
-          - 24.2.1
+          - 24.3.4.2
     steps:
     - uses: actions/checkout@v2
     - uses: ilammy/msvc-dev-cmd@v1
@@ -111,7 +111,7 @@ jobs:
       fail-fast: false
       matrix:
         otp:
-          - 24.1.5-3
+          - 24.3.4.2-1
         os:
           - macos-11
     runs-on: ${{ matrix.os }}

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/release.yaml

@@ -7,7 +7,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.detect-profiles.outputs.profiles}}

.github/workflows/run_acl_migration_tests.yaml

@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
         strategy:
             fail-fast: true
         env:

.github/workflows/run_fvt_tests.yaml

@@ -200,7 +200,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -249,9 +249,9 @@ jobs:
            fail-fast: false
            matrix:
              otp:
-             - 24.1.5-3
+             - 24.3.4.2-1
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
         defaults:
           run:
             shell: bash
@@ -280,19 +280,19 @@ jobs:
             path: |
               emqx/_packages/*/*.zip
               emqx/.ci/fvt_tests
+              emqx/data/relup-paths.eterm
+              emqx/scripts/relup-base-vsns.escript
 
     relup_test_run:
         needs:
           - relup_test_plan
           - relup_test_build
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
         strategy:
           fail-fast: false
           matrix:
             old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
-            otp:
-            - 24.1.5-3
         env:
           OLD_VSN: "${{ matrix.old_vsn }}"
           PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
@@ -313,16 +313,14 @@ jobs:
             repository: terry-xiaoyu/one_more_emqx
             ref: master
             path: one_more_emqx
-        - name: Prepare packages
+        - name: Run relup test scenario
+          timeout-minutes: 6
           run: |
             set -e -x -u
-            mkdir -p packages
+            old_emqx_vsn=${OLD_VSN#[e|v]}
+            old_otp_vsn=$(escript emqx_built/scripts/relup-base-vsns.escript otp-vsn-for "${old_emqx_vsn}" emqx_built/data/relup-paths.eterm)
+            wget --no-verbose -P packages https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${old_emqx_vsn}-otp${old_otp_vsn}-ubuntu20.04-amd64.zip
             cp emqx_built/_packages/*/*.zip packages
-            cd packages
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip
-        - name: Run relup test scenario
-          timeout-minutes: 5
-          run: |
             lux \
             --progress verbose \
             --case_timeout infinity \
@@ -331,8 +329,8 @@ jobs:
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
             --var OLD_VSN="$OLD_VSN" \
-            --var FROM_OTP_VSN="24.1.5-3" \
+            --var FROM_OTP_VSN="${old_otp_vsn}" \
-            --var TO_OTP_VSN="24.1.5-3" \
+            --var TO_OTP_VSN="24.3.4.2-1" \
             emqx_built/.ci/fvt_tests/relup.lux
         - uses: actions/upload-artifact@v2
           name: Save debug data

.github/workflows/run_test_cases.yaml

@@ -12,7 +12,7 @@ on:
 jobs:
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2

build

@@ -77,8 +77,13 @@ make_relup() {
     local zip_file
     mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
     local releases=()
+    local OTP_CHANGED='no'
     if [ -d "$releases_dir" ]; then
         for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do
+            OTP_BASE=$(relup_db otp-vsn-for "$BASE_VSN")
+            if [[ "$OTP_BASE" != "$OTP_VSN" ]]; then
+              OTP_CHANGED='yes'
+            fi
             OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN")
             zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip"
             if [ ! -d "$releases_dir/$BASE_VSN" ]; then
@@ -99,8 +104,8 @@ make_relup() {
     fi
     RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
     export RELX_BASE_VERSIONS
-    if [[ ${PKG_VSN} == 4.3* ]]; then
+    if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
-        echo "EMQX 4.3 specific, overwrite OTP app versions"
+        echo "EMQX 4.[3,4] specific, overwrite OTP app versions"
         local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
         if [ ! -f "${emqx_rel_file}" ]; then
             ./rebar3 as "${PROFILE}" release
@@ -118,7 +123,7 @@ make_relup() {
 
     # rollback rel file per releases
     #
-    if [[ ${PKG_VSN} == 4.3* ]]; then
+    if [[ ${PKG_VSN} == 4.[3,4]* && ${OTP_CHANGED} == 'yes' ]]; then
         echo "restore upgrade base rel files... "
         for rel in ${releases[*]};
         do

changes/v4.4.11-en.md

@@ -1,5 +1,11 @@
 ### Enhancements
 
+- OTP upgrade from 24.1.5-3 to 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265).
+  Change highlights:
+    - Erlang/OTP [SSL library vulnerability fix](https://nvd.nist.gov/vuln/detail/CVE-2022-37026)
+    - Added support for OCSP (Online Certificate Status Protocol) Stapling
+    - Added CRL (Certificate Revocation List) cache auto refresh
+
 ### Bug fixes
 
 - Fix get trace list crash when trace not initialize. [#9156](https://github.com/emqx/emqx/pull/9156)

changes/v4.4.11-zh.md

@@ -1,5 +1,10 @@
 ### 增强
 
+- OTP 升级: 从 24.1.5-3 至 24.3.4.2-1 [#9265](https://github.com/emqx/emqx/pull/9265)。
+  重要更新:
+    - Erlang/OTP [SSL库漏洞修复](https://nvd.nist.gov/vuln/detail/CVE-2022-37026)
+    - 增加了对 OCSP (Online Certificate Status Protocol) Stapling 的支持
+    - 增加了 CRL（证书吊销列表）缓存的自动刷新功能
   
 ### 修复
 

data/relup-paths.eterm

@@ -39,6 +39,13 @@
        [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
         <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>],
    otp => <<"24.1.5-3">>}}.
-{<<"4.5.0">>,
+{<<"4.4.11">>,
- #{from_versions => [<<"4.4.10">>,<<"4.4.8">>,<<"4.4.9">>],
+ #{from_versions =>
+       [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
+        <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>,
+        <<"4.4.10">>],
+   otp => <<"24.3.4.2-1">>}}.
+{<<"4.5.0">>,
+ #{from_versions => [<<"4.4.8">>,<<"4.4.9">>,<<"4.4.10">>,
+                     <<"4.4.11">>],
    otp => <<"24.3.4.2-1">>}}.