From 80727065ba79821986b21c1d8713b67412827ce0 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 19 Jul 2022 17:07:22 +0200
Subject: [PATCH 01/10] ci: [4.3 only] release with OTP 23.3

rpm/deb: build packages with two tracks: 23.2 and 23.3
    - Track 1: Add 23.3, bring in fix of https://github.com/erlang/otp/issues/5346
    - Track 2: Keep 23.2 for hot upgrading

docker:
    Default to 23.3
---
 .github/workflows/build_packages.yaml | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 8735e0d04..51f63971f 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -227,6 +227,9 @@ jobs:
           - centos6
           # - raspbian10 #armv6l is too slow to emulate
           # - raspbian9
+        otp_version:
+          - 23.2.7.2-emqx-3
+          - 23.3.4.9-3
         exclude:
         - os: centos6
           arch: arm64
@@ -265,7 +268,7 @@ jobs:
       run: unzip -q source.zip
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-3
+        ERL_OTP: erl${{ matrix.otp_version }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -289,11 +292,21 @@ jobs:
         docker volume prune -f
     - name: create sha256
       env:
-        PROFILE: ${{ matrix.profile}}
+        PROFILE: ${{ matrix.profile }}
+        ERL_OTP: erl${{ matrix.otp_version }}
+        ARCH: ${{ matrix.arch }}
       run: |
         if [ -d /tmp/packages/$PROFILE ]; then
           cd /tmp/packages/$PROFILE
             for var in $(ls emqx-* ); do
+              if [[ $ERL_OTP == erl23.2* ]]; then
+                # Keep package with new OTP as default
+                # But move package with old otp to track 2
+                echo "rename track 2 package"
+                oldfile="$var"
+                var="${var/${ARCH}/2-${ARCH}}"
+                mv "$oldfile" "$var"
+              fi
               bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
             done
           cd -
@@ -372,7 +385,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine
           RUN_FROM=alpine:3.12
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
@@ -388,7 +401,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine
           RUN_FROM=alpine:3.12
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise

From abbb2e2ec7fbf0b2868b471a4991c5597d24b528 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 20 Jul 2022 13:31:51 +0200
Subject: [PATCH 02/10] build: otp23.2 is marked in package name

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 51f63971f..be76d5525 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -304,7 +304,7 @@ jobs:
                 # But move package with old otp to track 2
                 echo "rename track 2 package"
                 oldfile="$var"
-                var="${var/${ARCH}/2-${ARCH}}"
+                var="${var/${ARCH}/otp23.2-${ARCH}}"
                 mv "$oldfile" "$var"
               fi
               bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"

From 3655ea050c167e5daeff15c58e9d0520b3cc1882 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 21 Jul 2022 10:14:51 +0200
Subject: [PATCH 03/10] build(relup): overwrite OTP vers of upgrade bases

---
 build                                      |  8 +++
 scripts/emqx_rel_otp_app_overwrite.escript | 57 ++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100755 scripts/emqx_rel_otp_app_overwrite.escript

diff --git a/build b/build
index 44287bff5..b028aae75 100755
--- a/build
+++ b/build
@@ -94,6 +94,14 @@ make_relup() {
     fi
     RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
     export RELX_BASE_VERSIONS
+    if [[ ${PKG_VSN} == 4.3* ]]; then
+        echo "EMQX 4.3 specific, overwrite OTP app versions"
+        local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
+        if [ ! -f "${emqx_rel_file}" ]; then
+            ./rebar3 as "${PROFILE}" release
+        fi
+        scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}"
+    fi
     ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
 }
 
diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript
new file mode 100755
index 000000000..843628e24
--- /dev/null
+++ b/scripts/emqx_rel_otp_app_overwrite.escript
@@ -0,0 +1,57 @@
+#!/usr/bin/env escript
+%% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base
+%% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator.
+%%
+%% It simply read OTP app version (incl. ERTS) from the rel file of *NEW* Release ($RelVsn) and write back to the ones
+%% in *OLD* versions ($BASE_VERSIONS)
+%%
+%% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup'
+%%
+main([Dir, Profile, RelVsn, BASE_VERSIONS]) ->
+    {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Profile, Dir, RelVsn), RelVsn),
+    lists:foreach(fun(BaseVer) ->
+                          base_rel_overwrites(BaseVer, Profile, Dir, ErtsVsn, Overwrites)
+                  end, string:tokens(BASE_VERSIONS, ",")).
+
+get_otp_apps(RelFile, RelVsn) ->
+    {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} = file:consult(RelFile),
+    Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList),
+    {ErtsVsn, Apps}.
+
+base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) ->
+    RelFile = rel_file(Profile, Dir, RelVsn),
+    file:copy(RelFile, RelFile++".bak"),
+    {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile),
+    NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn},
+                 lists:map(fun(X) ->
+                                   Name = element(1, X),
+                                   case lists:keyfind(Name, 1, Overwrites) of
+                                       false -> X;
+                                       Y when is_tuple(Y) -> Y
+                                   end
+                           end, BaseAppList)
+                }
+              ],
+    ok = file:write_file(RelFile, io_lib:format("~p.", NewData)).
+
+rel_file(Profile, Dir, RelVsn)->
+    filename:join([Dir, RelVsn, Profile++".rel"]).
+
+
+%% Couldn't find a good way to get this list dynamicly.
+otp_apps() ->
+    [ kernel
+    , stdlib
+    , sasl
+    , crypto
+    , public_key
+    , asn1
+    , syntax_tools
+    , ssl
+    , os_mon
+    , inets
+    , compiler
+    , runtime_tools
+    , mnesia
+    , xmerl
+    ].

From 08033a82e80511b41e1802708a87fa8862cf470d Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 22 Jul 2022 17:46:12 +0200
Subject: [PATCH 04/10] build(relup): restore emqx.rel backups after rebar
 relup

---
 build | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/build b/build
index b028aae75..6672685be 100755
--- a/build
+++ b/build
@@ -103,6 +103,21 @@ make_relup() {
         scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}"
     fi
     ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
+
+    # rollback relup
+    #
+    if [[ ${PKG_VSN} == 4.3* ]]; then
+        echo "restore upgrade base rel files... "
+        for rel in ${releases[*]};
+        do
+            bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak"
+            echo "restore $bakfile ..."
+            if [ -f "$bakfile" ]; then
+                echo "restore from $bakfile"
+                mv "${bakfile}" "${bakfile%.bak}"
+            fi
+        done
+    fi
 }
 
 cp_dyn_libs() {

From c9c1dfb8236e302680e72f00bc99a561c5c0d709 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 25 Jul 2022 23:09:17 +0200
Subject: [PATCH 05/10] fix(upgrade): ensure files in BINDIR executable

During upgrade procedure, emqx zip file is repacked to .tar.gz but does not
preserve permission of files.

This commit ensure files under BINDIR is executable
---
 bin/emqx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/emqx b/bin/emqx
index d6ce86e10..af3751ad7 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -42,6 +42,8 @@ assert_node_alive() {
 }
 
 check_erlang_start() {
+    # Fix bin permission
+    find "$BINDIR" ! -executable -exec chmod a+x {} \;
     "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
 }
 

From b17078b93f0fd7edac25c6defa5b86d474463efb Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 26 Jul 2022 10:41:01 +0200
Subject: [PATCH 06/10] fix(otp23.3): OTP app overwrites support emqx-edge

---
 scripts/emqx_rel_otp_app_overwrite.escript | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript
index 843628e24..52c6bb70e 100755
--- a/scripts/emqx_rel_otp_app_overwrite.escript
+++ b/scripts/emqx_rel_otp_app_overwrite.escript
@@ -34,6 +34,8 @@ base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) ->
               ],
     ok = file:write_file(RelFile, io_lib:format("~p.", NewData)).
 
+rel_file("emqx-edge", Dir, RelVsn)->
+    rel_file("emqx", Dir, RelVsn);
 rel_file(Profile, Dir, RelVsn)->
     filename:join([Dir, RelVsn, Profile++".rel"]).
 

From 8835296c7afed5c4abff86f00c71fe3363f8ca6d Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 1 Aug 2022 15:09:41 +0200
Subject: [PATCH 07/10] ci(rel): intro. rel_otp_apps.eterm

---
 rebar.config.erl                           | 21 +++++++--------------
 scripts/emqx_rel_otp_app_overwrite.escript | 22 +++++-----------------
 scripts/rel_otp_apps.eterm                 | 16 ++++++++++++++++
 3 files changed, 28 insertions(+), 31 deletions(-)
 create mode 100644 scripts/rel_otp_apps.eterm

diff --git a/rebar.config.erl b/rebar.config.erl
index c94f80c18..e0ee9b891 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -246,21 +246,10 @@ overlay_vars_pkg(pkg) ->
     ].
 
 relx_apps(ReleaseType) ->
-    [ kernel
-    , sasl
-    , crypto
-    , public_key
-    , asn1
-    , syntax_tools
-    , ssl
-    , os_mon
-    , inets
-    , compiler
-    , runtime_tools
-    , redbug
+    relx_otp_apps() ++
+    [ redbug
     , cuttlefish
     , emqx
-    , {mnesia, load}
     , {ekka, load}
     , {emqx_plugin_libs, load}
     , observer_cli
@@ -271,9 +260,13 @@ relx_apps(ReleaseType) ->
     ++ relx_apps_per_rel(ReleaseType)
     ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)].
 
+relx_otp_apps() ->
+    {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"),
+    true = is_list(Apps),
+    Apps.
+
 relx_apps_per_rel(cloud) ->
     [ luerl
-    , xmerl
     | [{observer, load} || is_app(observer)]
     ];
 relx_apps_per_rel(edge) ->
diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript
index 52c6bb70e..5eace01ff 100755
--- a/scripts/emqx_rel_otp_app_overwrite.escript
+++ b/scripts/emqx_rel_otp_app_overwrite.escript
@@ -39,21 +39,9 @@ rel_file("emqx-edge", Dir, RelVsn)->
 rel_file(Profile, Dir, RelVsn)->
     filename:join([Dir, RelVsn, Profile++".rel"]).
 
-
-%% Couldn't find a good way to get this list dynamicly.
 otp_apps() ->
-    [ kernel
-    , stdlib
-    , sasl
-    , crypto
-    , public_key
-    , asn1
-    , syntax_tools
-    , ssl
-    , os_mon
-    , inets
-    , compiler
-    , runtime_tools
-    , mnesia
-    , xmerl
-    ].
+    {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"),
+    true = is_list(Apps),
+    lists:map(fun(App) when is_atom(App) -> App;
+                 ({App, _}) -> App %% handle like {mnesia, load}
+              end, Apps).
diff --git a/scripts/rel_otp_apps.eterm b/scripts/rel_otp_apps.eterm
new file mode 100644
index 000000000..4999bbffa
--- /dev/null
+++ b/scripts/rel_otp_apps.eterm
@@ -0,0 +1,16 @@
+%% Single source of truth of list otp apps that we use
+[ kernel
+, stdlib
+, sasl
+, crypto
+, public_key
+, asn1
+, syntax_tools
+, ssl
+, os_mon
+, inets
+, compiler
+, runtime_tools
+, {mnesia, load}
+, xmerl
+].

From 19d36fbf144517faacd752518f37684cd73d7f00 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 1 Aug 2022 15:27:51 +0200
Subject: [PATCH 08/10] ci: assert no restart_emulator in relup

---
 build | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/build b/build
index 6672685be..2f773d295 100755
--- a/build
+++ b/build
@@ -104,7 +104,14 @@ make_relup() {
     fi
     ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
 
-    # rollback relup
+    # assert that there is no 'restart_emulator' in relup
+    # EMQX wants hot upgrade without VM restart
+    if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then
+        echo "Error: restart_emulator instruction found in relup";
+        exit 1
+    fi
+
+    # rollback rel file per releases
     #
     if [[ ${PKG_VSN} == 4.3* ]]; then
         echo "restore upgrade base rel files... "

From 874d30833422acff365b5d8e67279efc1cdbf8cb Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 2 Aug 2022 09:59:59 +0200
Subject: [PATCH 09/10] build: docker build from otp-23.3

---
 deploy/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index f24b4a348..346d33630 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine
 ARG RUN_FROM=alpine:3.12
 FROM ${BUILD_FROM} AS builder
 

From 0d1b26cabfc652c98a83ab8e5661fbdf0465dfab Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 2 Aug 2022 10:01:54 +0200
Subject: [PATCH 10/10] ci(linux): disable 23.2.7.2-emqx-3

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index be76d5525..84e02bf89 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -228,7 +228,7 @@ jobs:
           # - raspbian10 #armv6l is too slow to emulate
           # - raspbian9
         otp_version:
-          - 23.2.7.2-emqx-3
+          #- 23.2.7.2-emqx-3
           - 23.3.4.9-3
         exclude:
         - os: centos6