diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index d27c2b3d8..a86b06967 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -21,6 +21,10 @@ jobs: fetch-depth: 0 # need full history - name: fix-git-unsafe-repository run: git config --global --add safe.directory /__w/emqx/emqx + - name: Check relup version DB + run: | + PKG_VSN=$(./pkg-vsn.sh) + ./scripts/relup-base-vsns.escript check-vsn-db $PKG_VSN ./data/relup-paths.eterm - name: Check relup (ce) if: endsWith(github.repository, 'emqx') run: ./scripts/update-appup.sh emqx --check diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 970fc981f..d940760db 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -17,7 +17,6 @@ jobs: fail-fast: false matrix: erl_otp: - - 23.3.4.9-3 - 24.1.5-3 os: - ubuntu20.04 diff --git a/build b/build index 979af4510..732240194 100755 --- a/build +++ b/build @@ -66,29 +66,32 @@ make_rel() { ./rebar3 as "$PROFILE" tar } +relup_db() { + ./scripts/relup-base-vsns.escript "$@" ./data/relup-paths.eterm +} + ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" - local name_pattern - name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" + local zip_file mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then - while read -r zip; do - local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" - if [ ! -d "$releases_dir/$base_vsn" ]; then + for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do + OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") + zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip" + if [ ! -d "$releases_dir/$BASE_VSN" ]; then local tmp_dir tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" - unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" - unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" + unzip -q "$zip_file" "emqx/releases/*" -d "$tmp_dir" + unzip -q "$zip_file" "emqx/lib/*" -d "$tmp_dir" cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true rm -rf "$tmp_dir" fi - releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) + releases+=( "$BASE_VSN" ) + done fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -181,6 +184,11 @@ make_zip() { esac ;; esac + # shellcheck disable=SC2207 + bases=($(relup_db base-vsns "$PKG_VSN")) + if [[ "${#bases[@]}" -eq 0 ]]; then + has_relup='no' + fi if [ "$has_relup" = 'yes' ]; then ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" fi diff --git a/data/relup-paths.eterm b/data/relup-paths.eterm new file mode 100644 index 000000000..02d171e8f --- /dev/null +++ b/data/relup-paths.eterm @@ -0,0 +1,32 @@ +%% -*- mode: erlang; -*- + +{<<"4.4.0">>,#{from_versions => [],otp => <<"24.1.5-3">>}}. +{<<"4.4.1">>,#{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}}. +{<<"4.4.2">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}}. +{<<"4.4.3">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.4">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.5">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.6">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.7">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.8">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], + otp => <<"24.1.5-3">>}}. +{<<"4.5.0">>,#{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}. diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 9505a68f0..60785adc8 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -63,8 +63,12 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." mkdir -p _upgrade_base pushd _upgrade_base +otp_vsn_for() { + ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup-paths.eterm +} + for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" + filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then echo "downloading base package from ${url} ..." @@ -77,6 +81,8 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do ## https://askubuntu.com/questions/1202208/checking-sha256-checksum echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi + else + echo "file $filename already downloaded or doesn't exist in the archives; skipping it" fi done diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript new file mode 100755 index 000000000..94cc4bd4f --- /dev/null +++ b/scripts/relup-base-vsns.escript @@ -0,0 +1,285 @@ +#!/usr/bin/env escript +%% -*- mode: erlang; -*- + +-mode(compile). + +-define(RED, "\e[31m"). +-define(RESET, "\e[39m"). + +usage() -> +"A script to manage the released versions of EMQX for relup and hot +upgrade/downgrade. + +We store a \"database\" of released versions as an `eterm' file, which +is a mapping from a given version `Vsn' to its OTP version and a list +of previous versions from which one can upgrade to `Vsn' (the +\"from_versions\" list). That allow us to more easily/explicitly keep +track of allowed version upgrades/downgrades, as well as OTP changes +between releases + +In the examples below, `VERSION_DB_PATH' represents the path to the +`eterm' file containing the version database to be used. + +Usage: + + * List the previous base versions from which `TO_VSN' may be + upgraded to. Used to list versions for which relup files are to + be made. + + relup-base-vsns.escript base-vsns TO_VSN VERSION_DB_PATH + + + * Show the OTP version with which `Vsn' was built. + + relup-base-vsns.escript otp-vsn-for VSN VERSION_DB_PATH + + + * Automatically inserts a new version into the database. Previous + versions with the same Major and Minor numbers as `Vsn' are + considered to be upgradeable from, and versions with higher Major + and Minor numbers will automatically include `Vsn' in their + \"from_versions\" list. + + For example, if inserting 4.4.8 when 4.5.0 and 4.5.1 exists, + versions `BASE_FROM_VSN'...4.4.7 will be considered 4.4.8's + \"from_versions\", and 4.4.8 will be included into 4.5.0 and + 4.5.1's from versions. + + relup-base-vsns.escript insert-new-vsn NEW_VSN BASE_FROM_VSN OTP_VSN VERSION_DB_PATH + + * Check if the version database is consistent considering `VSN'. + + relup-base-vsns.escript check-vsn-db VSN VERSION_DB_PATH +". + +main(["base-vsns", To0, VsnDB]) -> + VsnMap = read_db(VsnDB), + To = strip_pre_release(To0), + #{from_versions := Froms} = fetch_version(To, VsnMap), + AvailableVersionsIndex = available_versions_index(), + lists:foreach( + fun(From) -> + io:format(user, "~s~n", [From]) + end, + filter_froms(Froms, AvailableVersionsIndex)), + halt(0); +main(["otp-vsn-for", Vsn0, VsnDB]) -> + VsnMap = read_db(VsnDB), + Vsn = strip_pre_release(Vsn0), + #{otp := OtpVsn} = fetch_version(Vsn, VsnMap), + io:format(user, "~s~n", [OtpVsn]), + halt(0); +main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> + VsnMap = read_db(VsnDB), + NewVsn = strip_pre_release(NewVsn0), + validate_version(NewVsn), + BaseFromVsn = strip_pre_release(BaseFromVsn0), + validate_version(BaseFromVsn), + OtpVsn = list_to_binary(OtpVsn0), + case VsnMap of + #{NewVsn := _} -> + print_warning("Version ~s already in DB!~n", [NewVsn]), + halt(1); + #{BaseFromVsn := _} -> + ok; + _ -> + print_warning("Version ~s not found in DB!~n", [BaseFromVsn]), + halt(1) + end, + NewVsnMap = insert_new_vsn(VsnMap, NewVsn, OtpVsn, BaseFromVsn), + NewVsnList = + lists:sort( + fun({Vsn1, _}, {Vsn2, _}) -> + parse_vsn(Vsn1) < parse_vsn(Vsn2) + end, maps:to_list(NewVsnMap)), + {ok, FD} = file:open(VsnDB, [write]), + io:format(FD, "%% -*- mode: erlang; -*-\n\n", []), + lists:foreach( + fun(Entry) -> + io:format(FD, "~p.~n", [Entry]) + end, + NewVsnList), + file:close(FD), + halt(0); +main(["check-vsn-db", NewVsn0, VsnDB]) -> + VsnMap = read_db(VsnDB), + NewVsn = strip_pre_release(NewVsn0), + case check_all_vsns_schema(VsnMap) of + [] -> ok; + Problems -> + print_warning("Invalid Version DB ~s!~n", [VsnDB]), + print_warning("Problems found:~n"), + lists:foreach( + fun(Problem) -> + print_warning(" ~p~n", [Problem]) + end, Problems), + halt(1) + end, + case VsnMap of + #{NewVsn := _} -> + io:format(user, "ok~n", []), + halt(0); + _ -> + Candidates = find_insertion_candidates(NewVsn, VsnMap), + print_warning("Version ~s not found in the version DB!~n", [NewVsn]), + [] =/= Candidates + andalso print_warning("Candidates for to insert this version into:~n"), + lists:foreach( + fun(Vsn) -> + io:format(user, " ~s~n", [Vsn]) + end, Candidates), + print_warning( + "To insert this version automatically, run:~n" + "./scripts/relup-base-vsns insert-new-vsn NEW-VSN BASE-FROM-VSN NEW-OTP-VSN ~s~n" + "And commit the results. Be sure to revise the changes.~n" + "Otherwise, edit the file manually~n", + [VsnDB]), + halt(1) + end; +main(_) -> + io:format(user, usage(), []), + halt(1). + +strip_pre_release(Vsn0) -> + case re:run(Vsn0, "[0-9]+\\.[0-9]+\\.[0-9]+", [{capture, all, binary}]) of + {match, [Vsn]} -> + Vsn; + _ -> + print_warning("Invalid Version: ~s ~n", [Vsn0]), + halt(1) + end. + +fetch_version(Vsn, VsnMap) -> + case VsnMap of + #{Vsn := VsnData} -> + VsnData; + _ -> + print_warning("Version not found in releases: ~s ~n", [Vsn]), + halt(1) + end. + +filter_froms(Froms0, AvailableVersionsIndex) -> + Froms1 = + case os:getenv("SYSTEM") of + %% debian11 is introduced since v4.4.2 and e4.4.2 + %% exclude tags before them + "debian11" -> + lists:filter( + fun(Vsn) -> + not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>]) + end, Froms0); + _ -> + Froms0 + end, + lists:filter( + fun(V) -> maps:get(V, AvailableVersionsIndex, false) end, + Froms1). + +%% assumes that's X.Y.Z, without pre-releases +parse_vsn(VsnBin) -> + {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)", + [{capture, all_but_first, binary}]), + [Major, Minor, Patch] = lists:map(fun binary_to_integer/1, [Major0, Minor0, Patch0]), + {Major, Minor, Patch}. + +parsed_vsn_to_bin({Major, Minor, Patch}) -> + iolist_to_binary(io_lib:format("~b.~b.~b", [Major, Minor, Patch])). + +find_insertion_candidates(NewVsn, VsnMap) -> + ParsedNewVsn = parse_vsn(NewVsn), + [Vsn + || Vsn <- maps:keys(VsnMap), + ParsedVsn <- [parse_vsn(Vsn)], + ParsedVsn > ParsedNewVsn]. + +check_all_vsns_schema(VsnMap) -> + maps:fold( + fun(Vsn, Val, Acc) -> + Problems = + [{Vsn, should_be_binary} || not is_binary(Vsn)] ++ + [{Vsn, must_have_map_value} || not is_map(Val)] ++ + [{Vsn, {must_contain_keys, [otp, from_versions]}} + || case Val of + #{otp := _, from_versions := _} -> + false; + _ -> + true + end] ++ + [{Vsn, otp_version_must_be_binary} + || case Val of + #{otp := Otp} when is_binary(Otp) -> + false; + _ -> + true + end] ++ + [{Vsn, versions_must_be_list_of_binaries} + || case Val of + #{from_versions := Froms} when is_list(Froms) -> + not lists:all(fun is_binary/1, Froms); + _ -> + true + end], + Problems ++ Acc + end, + [], + VsnMap). + +insert_new_vsn(VsnMap0, NewVsn, OtpVsn, BaseFromVsn) -> + ParsedNewVsn = parse_vsn(NewVsn), + ParsedBaseFromVsn = parse_vsn(BaseFromVsn), + %% candidates to insert this version into (they are "future" versions) + Candidates = find_insertion_candidates(NewVsn, VsnMap0), + %% Past versions we can upgrade from + Froms = [Vsn || Vsn <- maps:keys(VsnMap0), + ParsedVsn <- [parse_vsn(Vsn)], + ParsedVsn >= ParsedBaseFromVsn, + ParsedVsn < ParsedNewVsn], + VsnMap1 = + lists:foldl( + fun(FutureVsn, Acc) -> + FutureData0 = #{from_versions := Froms0} = maps:get(FutureVsn, Acc), + FutureData = FutureData0#{from_versions => lists:usort(Froms0 ++ [NewVsn])}, + Acc#{FutureVsn => FutureData} + end, + VsnMap0, + Candidates), + VsnMap1#{NewVsn => #{otp => OtpVsn, from_versions => Froms}}. + +validate_version(Vsn) -> + ParsedVsn = parse_vsn(Vsn), + VsnBack = parsed_vsn_to_bin(ParsedVsn), + case VsnBack =:= Vsn of + true -> ok; + false -> + print_warning("Invalid version ~p !~n", [Vsn]), + print_warning("Versions MUST be of the form X.Y.Z " + "and not prefixed by `e` or `v`~n"), + halt(1) + end. + +available_versions_index() -> + Output = os:cmd("git tag -l"), + AllVersions = + lists:filtermap( + fun(Line) -> + case re:run(Line, "^[ve]([0-9]+)\\.([0-9]+)\\.([0-9]+)$", + [{capture, all_but_first, binary}]) of + {match, [Major, Minor, Patch]} -> + Vsn = iolist_to_binary(io_lib:format("~s.~s.~s", [Major, Minor, Patch])), + {true, Vsn}; + _ -> false + end + end, string:split(Output, "\n", all)), + %% FIXME: `maps:from_keys' is available only in OTP 24, but we + %% still build with 23. Switch to that once we drop OTP 23. + maps:from_list([{Vsn, true} || Vsn <- AllVersions]). + +read_db(VsnDB) -> + {ok, VsnList} = file:consult(VsnDB), + maps:from_list(VsnList). + +print_warning(Msg) -> + print_warning(Msg, []). + +print_warning(Msg, Args) -> + io:format(user, ?RED ++ Msg ++ ?RESET, Args). diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 91c26823e..15b97921c 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -41,14 +41,6 @@ if [ "${#CUR_SEMVER[@]}" -lt 3 ]; then usage fi -## when the current version has no suffix such as -abcdef00 -## it is a formal release -if [ "${#CUR_SEMVER[@]}" -eq 3 ]; then - IS_RELEASE=true -else - IS_RELEASE=false -fi - case "${EDITION}" in *enterprise*) GIT_TAG_PREFIX="e" @@ -61,28 +53,11 @@ esac # must not be empty for MacOS (bash 3.x) TAGS=( 'dummy' ) TAGS_EXCLUDE=( 'dummy' ) -while read -r git_tag; do - # shellcheck disable=SC2207 - semver=($(parse_semver "$git_tag")) - if [ "${#semver[@]}" -eq 3 ] && [ "${semver[2]}" -le "${CUR_SEMVER[2]}" ]; then - if [ ${IS_RELEASE} = true ] && [ "${semver[2]}" -eq "${CUR_SEMVER[2]}" ] ; then - # do nothing - # exact match, do not print current version - # because current version is not an upgrade base - true - else - TAGS+=( "$git_tag" ) - fi - fi -done < <(git tag -l "${GIT_TAG_PREFIX}${CUR_SEMVER[0]}.${CUR_SEMVER[1]}.*") -# debian11 is introduced since v4.4.2 and e4.4.2 -# exclude tags before them -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" -if [ "$SYSTEM" = 'debian11' ]; then - TAGS_EXCLUDE+=( 'v4.4.0' 'v4.4.1' ) - TAGS_EXCLUDE+=( 'e4.4.0' 'e4.4.1' ) -fi +while read -r vsn; do + # shellcheck disable=SC2207 + TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}")) +done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup-paths.eterm) for tag_to_del in "${TAGS_EXCLUDE[@]}"; do TAGS=( "${TAGS[@]/$tag_to_del}" )