Merge remote-tracking branch 'origin/master' into resolve-conflict-master-to-v5.0

.github/workflows/build_packages.yaml

@@ -255,8 +255,8 @@ jobs:
             old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//"))
         fi
 
-        mkdir -p tmp/relup_packages/$PROFILE
+        mkdir -p _upgrade_base
-        cd tmp/relup_packages/$PROFILE
+        cd _upgrade_base
         for tag in ${old_vsns[@]};do
           if [ ! -z "$(echo  $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
             wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip

.gitignore

@@ -46,3 +46,4 @@ emqx_dialyzer_*_plt
 dist.zip
 scripts/git-token
 etc/*.seg
+_upgrade_base/

Makefile

@@ -1,11 +1,11 @@
 $(shell $(CURDIR)/scripts/git-hooks-init.sh)
-REBAR_VERSION = 3.14.3-emqx-6
+REBAR_VERSION = 3.14.3-emqx-7
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.0
+export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 endif
@@ -111,30 +111,35 @@ xref: $(REBAR)
 dialyzer: $(REBAR)
 	@$(REBAR) as check dialyzer
 
-.PHONY: $(REL_PROFILES:%=relup-%)
+COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
-$(REL_PROFILES:%=relup-%): $(REBAR)
-ifneq ($(OS),Windows_NT)
-	@$(BUILD) $(@:relup-%=%) relup
-endif
 
-.PHONY: $(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar)
+## rel target is to create release package without relup
-$(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR) get-dashboard $(CONF_SEGS)
+.PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
-	@$(BUILD) $(subst -tar,,$(@)) tar
+$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
+	@$(BUILD) $(subst -rel,,$(@)) rel
 
-## zip targets depend on the corresponding relup and tar artifacts
+## relup target is to create relup instructions
+.PHONY: $(REL_PROFILES:%=%-relup)
+define gen-relup-target
+$1-relup: $(COMMON_DEPS)
+	@$(BUILD) $1 relup
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt))))
+
+## zip target is to create a release package .zip with relup
 .PHONY: $(REL_PROFILES:%=%-zip)
 define gen-zip-target
-$1-zip: relup-$1 $1-tar
+$1-zip: $1-relup
 	@$(BUILD) $1 zip
 endef
-ALL_ZIPS = $(REL_PROFILES) $(PKG_PROFILES)
+ALL_ZIPS = $(REL_PROFILES)
 $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt))))
 
-## A pkg target depend on a regular release profile zip to include relup,
+## A pkg target depend on a regular release
-## and also a -pkg suffixed profile tar (without relup) for making deb/rpm package
 .PHONY: $(PKG_PROFILES)
 define gen-pkg-target
-$1: $(subst -pkg,,$1)-zip $1-tar
+$1: $1-rel
 	@$(BUILD) $1 pkg
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))

build

@@ -2,7 +2,7 @@
 
 # This script helps to build release artifacts.
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
-# arg2: artifact, e.g. tar | relup | zip | pkg
+# arg2: artifact, e.g. rel | relup | zip | pkg
 
 set -euo pipefail
 
@@ -62,42 +62,48 @@ log() {
     echo "===< $msg"
 }
 
-make_tar() {
+make_rel() {
-    ./rebar3 as "$PROFILE" tar
+    # shellcheck disable=SC1010
+    ./rebar3 as "$PROFILE" do release,tar
 }
 
 ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
 make_relup() {
+    local lib_dir="_build/$PROFILE/rel/emqx/lib"
     local releases_dir="_build/$PROFILE/rel/emqx/releases"
+    mkdir -p "$lib_dir" "$releases_dir"
     local releases=()
     if [ -d "$releases_dir" ]; then
-        while read -r dir; do
+        while read -r zip; do
-            local version
+            local base_vsn
-            version="$(basename "$dir")"
+            base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-e]{8})?")"
-            # skip current version
+            if [ ! -d "$releases_dir/$base_vsn" ]; then
-            if [ "$version" != "$PKG_VSN" ]; then
+                local tmp_dir
-                releases+=( "$version" )
+                tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
+                unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
+                unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
+                cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir"
+                cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir"
+                rm -rf "$tmp_dir"
             fi
-        done < <(find "_build/$PROFILE/rel/emqx/releases" -maxdepth 1 -name '*.*.*' -type d)
+            releases+=( "$base_vsn" )
+        done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f)
     fi
     if [ ${#releases[@]} -eq 0 ]; then
-        log "No previous release found, relup ignored"
+        log "No upgrade base found, relup ignored"
         return 0
     fi
-    if [ ${#releases[@]} -gt 1 ]; then
+    RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
-        log "Found more than one previous versions in $releases_dir:"
+    export RELX_BASE_VERSIONS
-        log "${releases[@]}"
+    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
-        log "ERROR: So far we can not support multi base-version relup creation"
-        return 1
-    fi
-    local base_version="${releases[0]}"
-    # TODO: comma separate base-versions when supported
-    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" --upfrom "$base_version"
 }
 
 ## make_zip turns .tar.gz into a .zip with a slightly different name.
 ## It assumes the .tar.gz has been built -- relies on Makefile dependency
 make_zip() {
+    # build the tarball again to ensure relup is included
+    make_rel
+
     tard="/tmp/emqx_untar_${PKG_VSN}"
     rm -rf "${tard}"
     mkdir -p "${tard}/emqx"
@@ -117,8 +123,8 @@ make_zip() {
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
-    tar)
+    rel)
-        make_tar
+        make_rel
         ;;
     relup)
         make_relup
@@ -132,8 +138,6 @@ case "$ARTIFACT" in
             log "Skipped making deb/rpm package for $SYSTEM"
             exit 0
         fi
-        # build the tar which is going to be used as the base of deb and rpm packages
-        make_tar
         make -C "deploy/packages/${PKGERDIR}" clean
         EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
         ;;

rebar.config

@@ -28,8 +28,7 @@
 {cover_export_enabled, true}.
 {cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}.
 
-{provider_hooks,[{pre,[{release,{relup_helper,gen_appups}}]}
+{provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}.
-                ]}.
 
 {post_hooks,[]}.
 

src/emqx_logger_jsonfmt.erl

@@ -207,17 +207,16 @@ json_kv(K0, V, Data, Config) ->
         false -> maps:put(json(K, Config), json(V, Config), Data)
     end.
 
-json_key('' = K) -> throw({badkey, K});
+json_key(A) when is_atom(A) -> json_key(atom_to_binary(A, utf8));
-json_key("" = K) -> throw({badkey, K});
-json_key("\"\"" = K) -> throw({badkey, K});
-json_key(<<>> = K) -> throw({badkey, K});
-json_key(<<"\"\"">> = K) -> throw({badkey, K});
-json_key(A) when is_atom(A) -> atom_to_binary(A, utf8);
 json_key(Term) ->
     try unicode:characters_to_binary(Term, utf8) of
-        OK when is_binary(OK) -> OK;
+        OK when is_binary(OK) andalso OK =/= <<>> ->
-        _ -> throw({badkey, Term})
+            OK;
-    catch _:_ -> throw({badkey, Term})
+        _ ->
+            throw({badkey, Term})
+    catch
+        _:_ ->
+            throw({badkey, Term})
     end.
 
 -ifdef(TEST).
@@ -262,7 +261,7 @@ p_report_cb() ->
                        , fun ?MODULE:report_cb_crash/2
                        , fun logger:format_otp_report/1
                        , fun logger:format_report/1
-                       , format_reportundefined
+                       , format_report_undefined
                        ]).
 
 report_cb_1(Input) -> {"~p", [Input]}.