From 396cae26fed91934fc21da8242e0b56803fbce55 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 09:39:28 +0200 Subject: [PATCH 1/7] build: fix build order of release and relup --- Makefile | 31 +++++++++++++++++-------------- build | 10 ++++------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index f0c5d0833..766123072 100644 --- a/Makefile +++ b/Makefile @@ -111,30 +111,33 @@ xref: $(REBAR) dialyzer: $(REBAR) @$(REBAR) as check dialyzer -.PHONY: $(REL_PROFILES:%=relup-%) -$(REL_PROFILES:%=relup-%): $(REBAR) -ifneq ($(OS),Windows_NT) - @$(BUILD) $(@:relup-%=%) relup -endif +## rel target is to create release package without relup +.PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) +$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(REBAR) get-dashboard $(CONF_SEGS) + @$(BUILD) $(subst -rel,,$(@)) rel -.PHONY: $(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar) -$(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR) get-dashboard $(CONF_SEGS) - @$(BUILD) $(subst -tar,,$(@)) tar +## relup target is to create relup instructions +.PHONY: $(REL_PROFILES:%=%-relup) +define gen-relup-target +$1-relup: $1-rel + @$(BUILD) $1 relup +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt)))) -## zip targets depend on the corresponding relup and tar artifacts +## zip target is to create a release package .zip with relup .PHONY: $(REL_PROFILES:%=%-zip) define gen-zip-target -$1-zip: relup-$1 $1-tar +$1-zip: $1-relup @$(BUILD) $1 zip endef -ALL_ZIPS = $(REL_PROFILES) $(PKG_PROFILES) +ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt)))) -## A pkg target depend on a regular release profile zip to include relup, -## and also a -pkg suffixed profile tar (without relup) for making deb/rpm package +## A pkg target depend on a regular release .PHONY: $(PKG_PROFILES) define gen-pkg-target -$1: $(subst -pkg,,$1)-zip $1-tar +$1: $1-rel @$(BUILD) $1 pkg endef $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt)))) diff --git a/build b/build index c4f57d4ea..a794c90a4 100755 --- a/build +++ b/build @@ -2,7 +2,7 @@ # This script helps to build release artifacts. # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg -# arg2: artifact, e.g. tar | relup | zip | pkg +# arg2: artifact, e.g. rel | relup | zip | pkg set -euo pipefail @@ -62,7 +62,7 @@ log() { echo "===< $msg" } -make_tar() { +make_rel() { ./rebar3 as "$PROFILE" tar } @@ -117,8 +117,8 @@ make_zip() { log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in - tar) - make_tar + rel) + make_rel ;; relup) make_relup @@ -132,8 +132,6 @@ case "$ARTIFACT" in log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi - # build the tar which is going to be used as the base of deb and rpm packages - make_tar make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; From 3c4e95a246aaec1c21f1f04fa62c1dedabb5dbbe Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 7 May 2021 18:56:14 +0900 Subject: [PATCH 2/7] chore(rebar.config): cuttlefish 3.3.3 -> 3.3.4 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index f15c31351..04a23b14f 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.3"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.4"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} From 30c214d64b2c5550d68d7407eeed6b8c7628cd2f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 12:46:24 +0200 Subject: [PATCH 3/7] build: fix relup hook point name --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 04a23b14f..5a1971b7e 100644 --- a/rebar.config +++ b/rebar.config @@ -28,7 +28,7 @@ {cover_export_enabled, true}. {cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}. -{provider_hooks,[{pre,[{release,{relup_helper,gen_appups}}]} +{provider_hooks,[{pre, [{tar, {relup_helper,gen_appups}}]} ]}. {post_hooks,[]}. From 958143f7a9be5a60666e943ba0dfa537437a0998 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 11:24:46 +0200 Subject: [PATCH 4/7] fix(frontend): dashboard 4.3.1 for a security patch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 766123072..80e35d099 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_CE_DASHBOARD_VERSION ?= v4.3.0 +export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif From 6595d53962de8749bfccfa0c2b17205a7949b5fe Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 7 May 2021 18:03:05 +0800 Subject: [PATCH 5/7] build(relup): update build script for support relup --- .gitignore | 1 + Makefile | 8 +++++--- build | 42 ++++++++++++++++++++++++------------------ rebar.config | 3 +-- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index c702d3955..218b22261 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ emqx_dialyzer_*_plt dist.zip scripts/git-token etc/*.seg +_upgrade_base/ diff --git a/Makefile b/Makefile index 80e35d099..01fc40435 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.14.3-emqx-6 +REBAR_VERSION = 3.14.3-emqx-7 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -111,15 +111,17 @@ xref: $(REBAR) dialyzer: $(REBAR) @$(REBAR) as check dialyzer +COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS) + ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) -$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(REBAR) get-dashboard $(CONF_SEGS) +$(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS) @$(BUILD) $(subst -rel,,$(@)) rel ## relup target is to create relup instructions .PHONY: $(REL_PROFILES:%=%-relup) define gen-relup-target -$1-relup: $1-rel +$1-relup: $(COMMON_DEPS) @$(BUILD) $1 relup endef ALL_ZIPS = $(REL_PROFILES) diff --git a/build b/build index a794c90a4..af7ff9865 100755 --- a/build +++ b/build @@ -63,41 +63,47 @@ log() { } make_rel() { - ./rebar3 as "$PROFILE" tar + # shellcheck disable=SC1010 + ./rebar3 as "$PROFILE" do release,tar } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup make_relup() { + local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" + mkdir -p "$lib_dir" "$releases_dir" local releases=() if [ -d "$releases_dir" ]; then - while read -r dir; do - local version - version="$(basename "$dir")" - # skip current version - if [ "$version" != "$PKG_VSN" ]; then - releases+=( "$version" ) + while read -r zip; do + local base_vsn + base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-e]{8})?")" + if [ ! -d "$releases_dir/$base_vsn" ]; then + local tmp_dir + tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" + unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" + unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" + cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" + cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" + rm -rf "$tmp_dir" fi - done < <(find "_build/$PROFILE/rel/emqx/releases" -maxdepth 1 -name '*.*.*' -type d) + releases+=( "$base_vsn" ) + done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then - log "No previous release found, relup ignored" + log "No upgrade base found, relup ignored" return 0 fi - if [ ${#releases[@]} -gt 1 ]; then - log "Found more than one previous versions in $releases_dir:" - log "${releases[@]}" - log "ERROR: So far we can not support multi base-version relup creation" - return 1 - fi - local base_version="${releases[0]}" - # TODO: comma separate base-versions when supported - ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" --upfrom "$base_version" + RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" + export RELX_BASE_VERSIONS + ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } ## make_zip turns .tar.gz into a .zip with a slightly different name. ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_zip() { + # build the tarball again to ensure relup is included + make_rel + tard="/tmp/emqx_untar_${PKG_VSN}" rm -rf "${tard}" mkdir -p "${tard}/emqx" diff --git a/rebar.config b/rebar.config index 5a1971b7e..90841fbfa 100644 --- a/rebar.config +++ b/rebar.config @@ -28,8 +28,7 @@ {cover_export_enabled, true}. {cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}. -{provider_hooks,[{pre, [{tar, {relup_helper,gen_appups}}]} - ]}. +{provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. {post_hooks,[]}. From 05324dfd7cf7270ea960d739b3a55f6ee2463b6f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 18:40:00 +0200 Subject: [PATCH 6/7] build(ci): download old versions --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 9f4a56da7..697820299 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -255,8 +255,8 @@ jobs: old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//")) fi - mkdir -p tmp/relup_packages/$PROFILE - cd tmp/relup_packages/$PROFILE + mkdir -p _upgrade_base + cd _upgrade_base for tag in ${old_vsns[@]};do if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip From 18cd7e7af5e146e50ecc10fbe5c14826af334573 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 19:05:58 +0200 Subject: [PATCH 7/7] fix(emqx_logger_jsonfmt): corner test case --- src/emqx_logger_jsonfmt.erl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/emqx_logger_jsonfmt.erl b/src/emqx_logger_jsonfmt.erl index e3169e1aa..7d0a1a328 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/src/emqx_logger_jsonfmt.erl @@ -207,17 +207,16 @@ json_kv(K0, V, Data, Config) -> false -> maps:put(json(K, Config), json(V, Config), Data) end. -json_key('' = K) -> throw({badkey, K}); -json_key("" = K) -> throw({badkey, K}); -json_key("\"\"" = K) -> throw({badkey, K}); -json_key(<<>> = K) -> throw({badkey, K}); -json_key(<<"\"\"">> = K) -> throw({badkey, K}); -json_key(A) when is_atom(A) -> atom_to_binary(A, utf8); +json_key(A) when is_atom(A) -> json_key(atom_to_binary(A, utf8)); json_key(Term) -> try unicode:characters_to_binary(Term, utf8) of - OK when is_binary(OK) -> OK; - _ -> throw({badkey, Term}) - catch _:_ -> throw({badkey, Term}) + OK when is_binary(OK) andalso OK =/= <<>> -> + OK; + _ -> + throw({badkey, Term}) + catch + _:_ -> + throw({badkey, Term}) end. -ifdef(TEST). @@ -262,7 +261,7 @@ p_report_cb() -> , fun ?MODULE:report_cb_crash/2 , fun logger:format_otp_report/1 , fun logger:format_report/1 - , format_reportundefined + , format_report_undefined ]). report_cb_1(Input) -> {"~p", [Input]}.