Merge remote-tracking branch 'origin/release-v43' into release-v44

.github/workflows/run_test_cases.yaml

@@ -6,7 +6,7 @@ on:
       - v*
       - e*
     branches:
-      - 'main-v4.?'
+      - 'main-v4.[0-9]?'
   pull_request:
 
 jobs:

CHANGES-4.3.md

@@ -9,22 +9,33 @@ File format:
 - Use weight-2 heading for releases
 - One list item per change topic
   Change log ends with a list of GitHub PRs
+
 ## v4.3.21
 
 ### Enhancements
 
-- TLS listener memory usage optimization
+- TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005).
-  new option 'hibernate_after' to hibernate TLS process after idling
+  New config `listener.ssl.$NAME.hibernate_after` to hibernate TLS connection process after idling.
-- TLS listener default buffer size to 4KB
+  Hibernation can reduce RAM usage significantly, but may cost more CPU.
-  Eliminate uncertainty that the buffer size is set by OS default
+  This configuration is by default disabled.
-  
+  Our preliminary test shows a 50% of RAM usage decline when configured to '5s'.
+
+- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
+  Eliminate uncertainty that the buffer size is set by OS default.
+
+- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908)
+
+- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
+
+- Added a test to prevent a last will testament message to be
+  published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)
+
 ## v4.3.20
 
 ### Bug fixes
 
 - Fix rule-engine update behaviour which may initialize actions for disabled rules. [#8849](https://github.com/emqx/emqx/pull/8849)
 - Fix JWT plugin don't support non-integer timestamp claims. [#8862](https://github.com/emqx/emqx/pull/8862)
-- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908)
 - Fix a possible dead loop caused by shared subscriptions with `shared_dispatch_ack_enabled=true`. [#8918](https://github.com/emqx/emqx/pull/8918)
 - Fix dashboard binding IP address not working. [#8916](https://github.com/emqx/emqx/pull/8916)
 - Fix rule SQL topic matching to null values failed. [#8927](https://github.com/emqx/emqx/pull/8927)
@@ -32,9 +43,6 @@ File format:
   `SELECT topic =~ 't' as r FROM "$events/client_connected"`.
   The topic is a null value as there's no such field in event `$events/client_connected`, so it
   should return false if match it to a topic.
-- Added a test to prevent a last will testament message to be
-  published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)
-- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
 
 ## v4.3.19
 

scripts/rel/cut4x.sh

@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+
+## cut a new 4.x release for EMQX (opensource or enterprise).
+
+set -euo pipefail
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
+
+usage() {
+    cat <<EOF
+$0 RELEASE_GIT_TAG [option]
+RELEASE_GIT_TAG is a 'v*' or 'e*' tag for example:
+  v4.3.21
+  e4.4.10-alpha.2
+
+options:
+  -h|--help: Print this usage.
+  -b|--base: Specify the current release base branch, can be one of
+             release-v43, release-v44, release-e43, or release-e44.
+             NOTE: this option should be used when --dryrun.
+  --dryrun:  Do not actually create the git tag.
+  --skip-appup: Skip checking appup
+                Useful when you are sure that appup is already updated'
+
+NOTE: When cutting a 'e*' tag release, both the opensource and enterprise
+      repos should be found as a fetch-remote in the current git repo.
+
+NOTE: For 4.3 series the current working branch must be 'release-v43' for opensource edition
+      and 'release-e43' for enterprise edition.
+      --.--[main-v4.3]------------------.-----------.---
+         \\                             /             \\
+          \`---[release-v43]----(v4.3.X)               \\
+                                       \\               \\
+          .---[release-e43]-------------'--(e4.3.Y)     \\
+         /                                         \\     V
+      --'------[main-v4.3-enterprise]---------------'----'---
+      The same applies to 4.4 series, however, a 4.3 branch is also the upstream branch
+      for the corresponding 4.4 branch. e.g. release-e44 has two upstreams: release-v44 and release-e43
+EOF
+}
+
+logerr() {
+    echo -e "\e[31mERROR: $1\e[39m"
+}
+logmsg() {
+    echo -e "\e[33mINFO: $1\e[39m"
+}
+
+REL_BRANCH_CE="${REL_BRANCH_CE:-release-v43}"
+REL_BRANCH_EE="${REL_BRANCH_EE:-release-e43}"
+
+TAG="${1:-}"
+
+case "$TAG" in
+    v*)
+        if [ -f EMQX_ENTERPRISE ]; then
+            logerr 'Cannot create v-tag on enterprise branch'
+            exit 1
+        fi
+        TAG_PREFIX='v'
+        APPUP_CHECK_PROFILE='emqx'
+        ;;
+    e*)
+        if [ ! -f EMQX_ENTERPRISE ]; then
+            logerr 'Cannot create e-tag on opensource branch'
+            exit 1
+        fi
+        TAG_PREFIX='e'
+        APPUP_CHECK_PROFILE='emqx-ee'
+        ;;
+    -h|--help)
+        usage
+        exit 0
+        ;;
+    *)
+        logerr "Unknown version tag $TAG"
+        usage
+        exit 1
+        ;;
+esac
+
+shift 1
+
+SKIP_APPUP='no'
+DRYRUN='no'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --skip-appup)
+            shift
+            SKIP_APPUP='yes'
+            ;;
+        --dryrun)
+            shift
+            DRYRUN='yes'
+            ;;
+        -b|--base)
+            BASE_BR="${2:-}"
+            if [ -z "${BASE_BR}" ]; then
+                logerr "Must specify which base branch"
+                exit 1
+            fi
+            shift 2
+            ;;
+        *)
+            logerr "Unknown option $1"
+            exit 1
+            ;;
+    esac
+done
+
+rel_branch() {
+    local tag="$1"
+    case "$tag" in
+        v4.3.*)
+            echo 'release-v43'
+            ;;
+        e4.3.*)
+            echo 'release-e43'
+            ;;
+        v4.4.*)
+            echo 'release-v44'
+            ;;
+        e4.4.*)
+            echo 'release-e44'
+            ;;
+        *)
+            logerr "Unsupported version tag $TAG"
+            exit 1
+            ;;
+    esac
+}
+
+## Ensure the current work branch
+assert_work_branch() {
+    local tag="$1"
+    local release_branch
+    release_branch="$(rel_branch "$tag")"
+    local base_branch
+    base_branch="${BASE_BR:-$(git branch --show-current)}"
+    if [ "$base_branch" != "$release_branch" ]; then
+        logerr "Base branch: $base_branch"
+        logerr "Relase tag must be on the release branch: $release_branch"
+        logerr "or must use -b|--base option to specify which release branch is current branch based on"
+        exit 1
+    fi
+}
+assert_work_branch "$TAG"
+
+## Ensure no dirty changes
+assert_not_dirty() {
+    local diff
+    diff="$(git diff --name-only)"
+    if [ -n "$diff" ]; then
+        logerr "Git status is not clean? Changed files:"
+        logerr "$diff"
+        exit 1
+    fi
+}
+assert_not_dirty
+
+## Assert that the tag is not already created
+assert_tag_absent() {
+    local tag="$1"
+    ## Fail if the tag already exists
+    EXISTING="$(git tag --list "$tag")"
+    if [ -n "$EXISTING" ]; then
+        logerr "$tag already released?"
+        logerr 'This script refuse to force re-tag.'
+        logerr 'If re-tag is intended, you must first delete the tag from both local and remote'
+        exit 1
+    fi
+}
+assert_tag_absent "$TAG"
+
+PKG_VSN=$(./pkg-vsn.sh)
+
+## Assert package version is updated to the tag which is being created
+assert_release_version() {
+    local tag="$1"
+    # shellcheck disable=SC2001
+    pkg_vsn="$(echo "$PKG_VSN" | sed 's/-[0-9a-f]\{8\}$//g')"
+    if [ "${TAG_PREFIX}${pkg_vsn}" != "${tag}" ]; then
+        logerr "The release version ($pkg_vsn) is different from the desired git tag."
+        logerr "Update the release version in emqx_release.hrl"
+        exit 1
+    fi
+}
+assert_release_version "$TAG"
+
+## Check if all upstream branches are merged
+if [ -z "${BASE_BR:-}" ]; then
+    ./scripts/rel/sync-remotes.sh
+else
+    ./scripts/rel/sync-remotes.sh --base "$BASE_BR"
+fi
+
+## Check if the Chart versions are in sync
+./scripts/check-chart-vsn.sh
+
+## Check if app versions are bumped
+./scripts/check-apps-vsn.sh
+
+## Ensure appup files are updated
+if [ "$SKIP_APPUP" = 'no' ]; then
+    logmsg "Checking appups"
+    ./scripts/update-appup.sh "$APPUP_CHECK_PROFILE" --check
+else
+    logmsg "Skipped checking appup updates"
+fi
+
+## Ensure relup paths are updated
+case "${PKG_VSN}" in
+    4.3.*)
+        HAS_RELUP_DB='no'
+        ;;
+    *)
+        HAS_RELUP_DB='yes'
+        ;;
+esac
+if [ "$HAS_RELUP_DB" = 'yes' ]; then
+    if [ -f EMQX_ENTERPRISE ]; then
+        RELUP_PATHS='./data/relup-paths-ee.eterm'
+    else
+        RELUP_PATHS='./data/relup-paths.eterm'
+    fi
+    ./scripts/relup-base-vsns.escript check-vsn-db "$PKG_VSN" "$RELUP_PATHS"
+fi
+
+if [ "$DRYRUN" = 'yes' ]; then
+    logmsg "Release tag is ready to be created with command: git tag $TAG"
+else
+    git tag "$TAG"
+    logmsg "$TAG is created OK."
+fi

scripts/rel/sync-remotes.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
+
+CE_BASES=( 'release-v43' 'release-v44' 'main-v4.3' 'main-v4.4' )
+EE_BASES=( 'release-e43' 'release-e44' 'main-v4.3-enterprise' 'main-v4.4-enterprise' )
+
+usage() {
+    cat <<EOF
+$0 [option]
+
+options:
+
+  -h|--help:
+    This script works on one of the branches listed in the -b|--base option below.
+    It tries to merge (by default with --ff-only option)
+    upstreams branches for the current working branch.
+    The uppstream branch of the current branch are as below:
+    * v43: []              # no upstream for 4.3 opensource edition
+    * e43: [v43]           # 4.3 enterprise has 4.3 opensource as upstream
+    * v44: [v43]           # 4.4 opensource has 4.3 opensource as upstream
+    * e44: [e43, v44, v43] # 4.4 enterprise has all the above 3 as upstream
+    For e44:
+      v43 is an indirect upstream.
+      Ideally the merge of v43 should be done through v44 or e43,
+      but it does not hurt to apply a direct merge.
+
+  -b|--base:
+    The base branch of current working branch if currently is not
+    on one of the following branches.
+    ${CE_BASES[@]}
+    ${EE_BASES[@]}
+
+  -i|--interactive:
+    With this option, the script will try to merge upstream
+    branches to local working branch interactively.
+    That is, there will be git prompts to edit commit messages etc.
+    Without this option, the script executes 'git merge' command
+    with '--ff-only' option which conveniently pulls remote
+    updates if there is any, and fails when fast-forward is not possible
+EOF
+}
+
+logerr() {
+    echo -e "\e[31mERROR: $1\e[39m"
+}
+
+logwarn() {
+    echo -e "\e[33mINFO: $1\e[39m"
+}
+
+logmsg() {
+    echo "INFO: $1"
+}
+
+INTERACTIVE='no'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        -i|--interactive)
+            shift
+            INTERACTIVE='yes'
+            ;;
+        -b|--base)
+            shift
+            BASE_BRANCH="$1"
+            shift
+            ;;
+        *)
+            logerr "Unknown option $1"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -f EMQX_ENTERPRISE ]; then
+    IS_ENTERPRISE='yes'
+    BASE_BRANCHES=( "${EE_BASES[@]}" )
+else
+    IS_ENTERPRISE='no'
+    BASE_BRANCHES=( "${CE_BASES[@]}" )
+fi
+
+CURRENT_BRANCH="$(git branch --show-current)"
+BASE_BRANCH="${BASE_BRANCH:-${CURRENT_BRANCH}}"
+
+## check if arg1 is one of the elements in arg2-N
+is_element() {
+    local e match="$1"
+    shift
+    for e in "${@}"; do
+        if [ "$e" = "$match" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+if ! is_element "$BASE_BRANCH" "${BASE_BRANCHES[@]}"; then
+    logerr "Cannot work with branch $BASE_BRANCH"
+    logerr "The base branch must be one of: ${BASE_BRANCHES[*]}"
+    logerr "Change work branch to one of the above."
+    logerr "OR: use -b|--base to specify from which base branch is current working branch created"
+    exit 1
+fi
+
+## Find git remotes to fetch from.
+##
+## NOTE: For enterprise, the opensource repo must be added as a remote.
+##       Because not all changes in opensource repo are synced to enterprise repo immediately.
+##
+## NOTE: grep -v enterprise here, but why not to match on full repo name 'emqx/emqx.git'?
+##       It's because the git remote does not always end with .git
+GIT_REMOTE_CE="$(git remote -v | grep 'emqx/emqx' | grep -v enterprise | grep fetch | head -1 | awk '{print $1}' || true)"
+if [ -z "$GIT_REMOTE_CE" ]; then
+	logerr "Cannot find git remote for emqx/emqx"
+    exit 1
+fi
+REMOTES=( "${GIT_REMOTE_CE}" )
+if [ "$IS_ENTERPRISE" = 'yes' ]; then
+    GIT_REMOTE_EE="$(git remote -v | grep 'emqx/emqx-enterprise' | grep fetch | head -1 | awk '{print $1}' || true)"
+    if [ -z "$GIT_REMOTE_EE" ]; then
+        logerr "Cannot find git remote for emqx/emqx-enterprise"
+        exit 1
+    fi
+    REMOTES+=( "$GIT_REMOTE_EE" )
+fi
+
+## Fetch the remotes
+for remote in "${REMOTES[@]}"; do
+    logwarn "Fetching from remote=${remote} (force tag sync)."
+    git fetch "$remote" --tags --force
+done
+
+logmsg 'Fetched all remotes'
+
+if [ "$INTERACTIVE" = 'yes' ]; then
+    MERGE_OPTS=''
+else
+    ## Using --ff-only to *check* if the remote is already merged
+    ## Also conveniently merged it in case it's *not* merged but can be fast-forwarded
+    ## Alternative is to check with 'git merge-base'
+    MERGE_OPTS='--ff-only'
+fi
+
+## Get the git remote reference of the given 'release-' or 'main-' branch
+remote_ref() {
+    local branch="$1"
+    if is_element "$branch" "${EE_BASES[@]}"; then
+        echo -n "${GIT_REMOTE_EE}/${branch} "
+    else
+        echo -n "${GIT_REMOTE_CE}/${branch} "
+    fi
+}
+
+remote_refs() {
+    local br
+    for br in "${@}"; do
+        remote_ref "$br"
+    done
+}
+
+## Get upstream branches of the given branch
+upstream_branches() {
+    local base="$1"
+    case "$base" in
+        release-v43|main-v4.3)
+            ## no upstream for 4.3 opensource
+            remote_ref "$base"
+            ;;
+        release-v44)
+            remote_refs "$base" 'release-v43'
+            ;;
+        main-v4.4)
+            remote_refs "$base" 'main-v4.3'
+            ;;
+        release-e43)
+            remote_refs "$base" 'release-v43'
+            ;;
+        main-v4.3-enterprise)
+            remote_refs "$base" 'main-v4.3'
+            ;;
+        release-e44)
+            remote_refs "$base" 'release-v44' 'release-e43' 'release-v43'
+            ;;
+        main-v4.4-enterprise)
+            remote_refs "$base" 'main-v4.4' 'main-v4.3-enterprise' 'main-v4.3'
+            ;;
+    esac
+}
+
+for remote_ref in $(upstream_branches "$BASE_BRANCH"); do
+    logmsg "Merging $remote_ref"
+    git merge $MERGE_OPTS "$remote_ref"
+done

scripts/update_appup.escript

@@ -3,6 +3,9 @@
 
 -mode(compile).
 
+-define(RED, "\e[31m").
+-define(RESET, "\e[39m").
+
 usage() ->
 "A script that fills in boilerplate for appup files.
 
@@ -109,7 +112,7 @@ changes, supervisor changes, process restarts and so on. Also the load order of
 the beam files might need updating.~n"),
     halt(0);
 warn_and_exit(false) ->
-    log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"),
+    logerr("Incomplete appups found. Please inspect the output for more details.~n", []),
     halt(1).
 
 prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) ->
@@ -474,8 +477,8 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
             ok;
         {diffs, Diffs} ->
             set_invalid(),
-            log("ERROR: Appup file for '~p' is not complete.~n"
+            logerr("Appup file for '~p' is not complete.~n"
-                "Missing:~100p~n", [App, Diffs]),
+                   "Missing:~100p~n", [App, Diffs]),
             notok
     end.
 
@@ -494,7 +497,7 @@ render_appup(App, File, Up, Down) ->
             do_render_appup(File, Up, Down);
         {error, enoent} when IsCheck ->
             %% failed to read old file, exit
-            log("ERROR: ~s is missing", [File]),
+            logerr("~s is missing", [File]),
             set_invalid()
     end.
 
@@ -580,8 +583,8 @@ diff_app(UpOrDown, App,
             case UpOrDown =:= up of
                 true ->
                     %% only log for the upgrade case because it would be the same result
-                    log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
+                    logerr("Application '~p' contains changes, but its version is not updated. ~s",
-                        [App, format_changes(Changes)]);
+                           [App, format_changes(Changes)]);
                 false ->
                     ok
             end;
@@ -723,5 +726,8 @@ log(Msg) ->
 log(Msg, Args) ->
     io:format(standard_error, Msg, Args).
 
+logerr(Msg, Args) ->
+    io:format(standard_error,  ?RED ++ "ERROR: "++ Msg ++ ?RESET, Args).
+
 otp_standard_apps() ->
     [ssl, mnesia, kernel, asn1, stdlib].