Merge pull request #8776 from thalesmg/relup-base-db-44

ci: add relup version db (4.4)
This commit is contained in:
Thales Macedo Garitezi 2022-08-24 09:01:48 -03:00 committed by GitHub
commit 4fd62a7ace
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 350 additions and 41 deletions

View File

@ -21,6 +21,10 @@ jobs:
fetch-depth: 0 # need full history
- name: fix-git-unsafe-repository
run: git config --global --add safe.directory /__w/emqx/emqx
- name: Check relup version DB
run: |
PKG_VSN=$(./pkg-vsn.sh)
./scripts/relup-base-vsns.escript check-vsn-db $PKG_VSN ./data/relup-paths.eterm
- name: Check relup (ce)
if: endsWith(github.repository, 'emqx')
run: ./scripts/update-appup.sh emqx --check

View File

@ -17,7 +17,6 @@ jobs:
fail-fast: false
matrix:
erl_otp:
- 23.3.4.9-3
- 24.1.5-3
os:
- ubuntu20.04

28
build
View File

@ -66,29 +66,32 @@ make_rel() {
./rebar3 as "$PROFILE" tar
}
relup_db() {
./scripts/relup-base-vsns.escript "$@" ./data/relup-paths.eterm
}
## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib"
local releases_dir="_build/$PROFILE/rel/emqx/releases"
local name_pattern
name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')"
local zip_file
mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
local releases=()
if [ -d "$releases_dir" ]; then
while read -r zip; do
local base_vsn
base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)"
if [ ! -d "$releases_dir/$base_vsn" ]; then
for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do
OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN")
zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip"
if [ ! -d "$releases_dir/$BASE_VSN" ]; then
local tmp_dir
tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
unzip -q "$zip_file" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip_file" "emqx/lib/*" -d "$tmp_dir"
cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true
cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true
rm -rf "$tmp_dir"
fi
releases+=( "$base_vsn" )
done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f)
releases+=( "$BASE_VSN" )
done
fi
if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored"
@ -181,6 +184,11 @@ make_zip() {
esac
;;
esac
# shellcheck disable=SC2207
bases=($(relup_db base-vsns "$PKG_VSN"))
if [[ "${#bases[@]}" -eq 0 ]]; then
has_relup='no'
fi
if [ "$has_relup" = 'yes' ]; then
./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
fi

32
data/relup-paths.eterm Normal file
View File

@ -0,0 +1,32 @@
%% -*- mode: erlang; -*-
{<<"4.4.0">>,#{from_versions => [],otp => <<"24.1.5-3">>}}.
{<<"4.4.1">>,#{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}}.
{<<"4.4.2">>,
#{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}}.
{<<"4.4.3">>,
#{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>],
otp => <<"24.1.5-3">>}}.
{<<"4.4.4">>,
#{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>],
otp => <<"24.1.5-3">>}}.
{<<"4.4.5">>,
#{from_versions =>
[<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>],
otp => <<"24.1.5-3">>}}.
{<<"4.4.6">>,
#{from_versions =>
[<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
<<"4.4.5">>],
otp => <<"24.1.5-3">>}}.
{<<"4.4.7">>,
#{from_versions =>
[<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
<<"4.4.5">>,<<"4.4.6">>],
otp => <<"24.1.5-3">>}}.
{<<"4.4.8">>,
#{from_versions =>
[<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
<<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>],
otp => <<"24.1.5-3">>}}.
{<<"4.5.0">>,#{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}.

View File

@ -63,8 +63,12 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
mkdir -p _upgrade_base
pushd _upgrade_base
otp_vsn_for() {
../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup-paths.eterm
}
for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip"
filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip"
url="https://packages.emqx.io/$DIR/$tag/$filename"
if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
echo "downloading base package from ${url} ..."
@ -77,6 +81,8 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
## https://askubuntu.com/questions/1202208/checking-sha256-checksum
echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1
fi
else
echo "file $filename already downloaded or doesn't exist in the archives; skipping it"
fi
done

285
scripts/relup-base-vsns.escript Executable file
View File

@ -0,0 +1,285 @@
#!/usr/bin/env escript
%% -*- mode: erlang; -*-
-mode(compile).
-define(RED, "\e[31m").
-define(RESET, "\e[39m").
usage() ->
"A script to manage the released versions of EMQX for relup and hot
upgrade/downgrade.
We store a \"database\" of released versions as an `eterm' file, which
is a mapping from a given version `Vsn' to its OTP version and a list
of previous versions from which one can upgrade to `Vsn' (the
\"from_versions\" list). That allow us to more easily/explicitly keep
track of allowed version upgrades/downgrades, as well as OTP changes
between releases
In the examples below, `VERSION_DB_PATH' represents the path to the
`eterm' file containing the version database to be used.
Usage:
* List the previous base versions from which `TO_VSN' may be
upgraded to. Used to list versions for which relup files are to
be made.
relup-base-vsns.escript base-vsns TO_VSN VERSION_DB_PATH
* Show the OTP version with which `Vsn' was built.
relup-base-vsns.escript otp-vsn-for VSN VERSION_DB_PATH
* Automatically inserts a new version into the database. Previous
versions with the same Major and Minor numbers as `Vsn' are
considered to be upgradeable from, and versions with higher Major
and Minor numbers will automatically include `Vsn' in their
\"from_versions\" list.
For example, if inserting 4.4.8 when 4.5.0 and 4.5.1 exists,
versions `BASE_FROM_VSN'...4.4.7 will be considered 4.4.8's
\"from_versions\", and 4.4.8 will be included into 4.5.0 and
4.5.1's from versions.
relup-base-vsns.escript insert-new-vsn NEW_VSN BASE_FROM_VSN OTP_VSN VERSION_DB_PATH
* Check if the version database is consistent considering `VSN'.
relup-base-vsns.escript check-vsn-db VSN VERSION_DB_PATH
".
main(["base-vsns", To0, VsnDB]) ->
VsnMap = read_db(VsnDB),
To = strip_pre_release(To0),
#{from_versions := Froms} = fetch_version(To, VsnMap),
AvailableVersionsIndex = available_versions_index(),
lists:foreach(
fun(From) ->
io:format(user, "~s~n", [From])
end,
filter_froms(Froms, AvailableVersionsIndex)),
halt(0);
main(["otp-vsn-for", Vsn0, VsnDB]) ->
VsnMap = read_db(VsnDB),
Vsn = strip_pre_release(Vsn0),
#{otp := OtpVsn} = fetch_version(Vsn, VsnMap),
io:format(user, "~s~n", [OtpVsn]),
halt(0);
main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) ->
VsnMap = read_db(VsnDB),
NewVsn = strip_pre_release(NewVsn0),
validate_version(NewVsn),
BaseFromVsn = strip_pre_release(BaseFromVsn0),
validate_version(BaseFromVsn),
OtpVsn = list_to_binary(OtpVsn0),
case VsnMap of
#{NewVsn := _} ->
print_warning("Version ~s already in DB!~n", [NewVsn]),
halt(1);
#{BaseFromVsn := _} ->
ok;
_ ->
print_warning("Version ~s not found in DB!~n", [BaseFromVsn]),
halt(1)
end,
NewVsnMap = insert_new_vsn(VsnMap, NewVsn, OtpVsn, BaseFromVsn),
NewVsnList =
lists:sort(
fun({Vsn1, _}, {Vsn2, _}) ->
parse_vsn(Vsn1) < parse_vsn(Vsn2)
end, maps:to_list(NewVsnMap)),
{ok, FD} = file:open(VsnDB, [write]),
io:format(FD, "%% -*- mode: erlang; -*-\n\n", []),
lists:foreach(
fun(Entry) ->
io:format(FD, "~p.~n", [Entry])
end,
NewVsnList),
file:close(FD),
halt(0);
main(["check-vsn-db", NewVsn0, VsnDB]) ->
VsnMap = read_db(VsnDB),
NewVsn = strip_pre_release(NewVsn0),
case check_all_vsns_schema(VsnMap) of
[] -> ok;
Problems ->
print_warning("Invalid Version DB ~s!~n", [VsnDB]),
print_warning("Problems found:~n"),
lists:foreach(
fun(Problem) ->
print_warning(" ~p~n", [Problem])
end, Problems),
halt(1)
end,
case VsnMap of
#{NewVsn := _} ->
io:format(user, "ok~n", []),
halt(0);
_ ->
Candidates = find_insertion_candidates(NewVsn, VsnMap),
print_warning("Version ~s not found in the version DB!~n", [NewVsn]),
[] =/= Candidates
andalso print_warning("Candidates for to insert this version into:~n"),
lists:foreach(
fun(Vsn) ->
io:format(user, " ~s~n", [Vsn])
end, Candidates),
print_warning(
"To insert this version automatically, run:~n"
"./scripts/relup-base-vsns insert-new-vsn NEW-VSN BASE-FROM-VSN NEW-OTP-VSN ~s~n"
"And commit the results. Be sure to revise the changes.~n"
"Otherwise, edit the file manually~n",
[VsnDB]),
halt(1)
end;
main(_) ->
io:format(user, usage(), []),
halt(1).
strip_pre_release(Vsn0) ->
case re:run(Vsn0, "[0-9]+\\.[0-9]+\\.[0-9]+", [{capture, all, binary}]) of
{match, [Vsn]} ->
Vsn;
_ ->
print_warning("Invalid Version: ~s ~n", [Vsn0]),
halt(1)
end.
fetch_version(Vsn, VsnMap) ->
case VsnMap of
#{Vsn := VsnData} ->
VsnData;
_ ->
print_warning("Version not found in releases: ~s ~n", [Vsn]),
halt(1)
end.
filter_froms(Froms0, AvailableVersionsIndex) ->
Froms1 =
case os:getenv("SYSTEM") of
%% debian11 is introduced since v4.4.2 and e4.4.2
%% exclude tags before them
"debian11" ->
lists:filter(
fun(Vsn) ->
not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>])
end, Froms0);
_ ->
Froms0
end,
lists:filter(
fun(V) -> maps:get(V, AvailableVersionsIndex, false) end,
Froms1).
%% assumes that's X.Y.Z, without pre-releases
parse_vsn(VsnBin) ->
{match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)",
[{capture, all_but_first, binary}]),
[Major, Minor, Patch] = lists:map(fun binary_to_integer/1, [Major0, Minor0, Patch0]),
{Major, Minor, Patch}.
parsed_vsn_to_bin({Major, Minor, Patch}) ->
iolist_to_binary(io_lib:format("~b.~b.~b", [Major, Minor, Patch])).
find_insertion_candidates(NewVsn, VsnMap) ->
ParsedNewVsn = parse_vsn(NewVsn),
[Vsn
|| Vsn <- maps:keys(VsnMap),
ParsedVsn <- [parse_vsn(Vsn)],
ParsedVsn > ParsedNewVsn].
check_all_vsns_schema(VsnMap) ->
maps:fold(
fun(Vsn, Val, Acc) ->
Problems =
[{Vsn, should_be_binary} || not is_binary(Vsn)] ++
[{Vsn, must_have_map_value} || not is_map(Val)] ++
[{Vsn, {must_contain_keys, [otp, from_versions]}}
|| case Val of
#{otp := _, from_versions := _} ->
false;
_ ->
true
end] ++
[{Vsn, otp_version_must_be_binary}
|| case Val of
#{otp := Otp} when is_binary(Otp) ->
false;
_ ->
true
end] ++
[{Vsn, versions_must_be_list_of_binaries}
|| case Val of
#{from_versions := Froms} when is_list(Froms) ->
not lists:all(fun is_binary/1, Froms);
_ ->
true
end],
Problems ++ Acc
end,
[],
VsnMap).
insert_new_vsn(VsnMap0, NewVsn, OtpVsn, BaseFromVsn) ->
ParsedNewVsn = parse_vsn(NewVsn),
ParsedBaseFromVsn = parse_vsn(BaseFromVsn),
%% candidates to insert this version into (they are "future" versions)
Candidates = find_insertion_candidates(NewVsn, VsnMap0),
%% Past versions we can upgrade from
Froms = [Vsn || Vsn <- maps:keys(VsnMap0),
ParsedVsn <- [parse_vsn(Vsn)],
ParsedVsn >= ParsedBaseFromVsn,
ParsedVsn < ParsedNewVsn],
VsnMap1 =
lists:foldl(
fun(FutureVsn, Acc) ->
FutureData0 = #{from_versions := Froms0} = maps:get(FutureVsn, Acc),
FutureData = FutureData0#{from_versions => lists:usort(Froms0 ++ [NewVsn])},
Acc#{FutureVsn => FutureData}
end,
VsnMap0,
Candidates),
VsnMap1#{NewVsn => #{otp => OtpVsn, from_versions => Froms}}.
validate_version(Vsn) ->
ParsedVsn = parse_vsn(Vsn),
VsnBack = parsed_vsn_to_bin(ParsedVsn),
case VsnBack =:= Vsn of
true -> ok;
false ->
print_warning("Invalid version ~p !~n", [Vsn]),
print_warning("Versions MUST be of the form X.Y.Z "
"and not prefixed by `e` or `v`~n"),
halt(1)
end.
available_versions_index() ->
Output = os:cmd("git tag -l"),
AllVersions =
lists:filtermap(
fun(Line) ->
case re:run(Line, "^[ve]([0-9]+)\\.([0-9]+)\\.([0-9]+)$",
[{capture, all_but_first, binary}]) of
{match, [Major, Minor, Patch]} ->
Vsn = iolist_to_binary(io_lib:format("~s.~s.~s", [Major, Minor, Patch])),
{true, Vsn};
_ -> false
end
end, string:split(Output, "\n", all)),
%% FIXME: `maps:from_keys' is available only in OTP 24, but we
%% still build with 23. Switch to that once we drop OTP 23.
maps:from_list([{Vsn, true} || Vsn <- AllVersions]).
read_db(VsnDB) ->
{ok, VsnList} = file:consult(VsnDB),
maps:from_list(VsnList).
print_warning(Msg) ->
print_warning(Msg, []).
print_warning(Msg, Args) ->
io:format(user, ?RED ++ Msg ++ ?RESET, Args).

View File

@ -41,14 +41,6 @@ if [ "${#CUR_SEMVER[@]}" -lt 3 ]; then
usage
fi
## when the current version has no suffix such as -abcdef00
## it is a formal release
if [ "${#CUR_SEMVER[@]}" -eq 3 ]; then
IS_RELEASE=true
else
IS_RELEASE=false
fi
case "${EDITION}" in
*enterprise*)
GIT_TAG_PREFIX="e"
@ -61,28 +53,11 @@ esac
# must not be empty for MacOS (bash 3.x)
TAGS=( 'dummy' )
TAGS_EXCLUDE=( 'dummy' )
while read -r git_tag; do
# shellcheck disable=SC2207
semver=($(parse_semver "$git_tag"))
if [ "${#semver[@]}" -eq 3 ] && [ "${semver[2]}" -le "${CUR_SEMVER[2]}" ]; then
if [ ${IS_RELEASE} = true ] && [ "${semver[2]}" -eq "${CUR_SEMVER[2]}" ] ; then
# do nothing
# exact match, do not print current version
# because current version is not an upgrade base
true
else
TAGS+=( "$git_tag" )
fi
fi
done < <(git tag -l "${GIT_TAG_PREFIX}${CUR_SEMVER[0]}.${CUR_SEMVER[1]}.*")
# debian11 is introduced since v4.4.2 and e4.4.2
# exclude tags before them
SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
if [ "$SYSTEM" = 'debian11' ]; then
TAGS_EXCLUDE+=( 'v4.4.0' 'v4.4.1' )
TAGS_EXCLUDE+=( 'e4.4.0' 'e4.4.1' )
fi
while read -r vsn; do
# shellcheck disable=SC2207
TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}"))
done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup-paths.eterm)
for tag_to_del in "${TAGS_EXCLUDE[@]}"; do
TAGS=( "${TAGS[@]/$tag_to_del}" )