From 4020db8fc1276718f414c7d41529cda2e6ac855d Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:06:16 +0200 Subject: [PATCH 01/11] feat(update_appup): Compare beam files --- scripts/update_appup.escript | 240 +++++++++++++++++++++++++++-------- 1 file changed, 190 insertions(+), 50 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 6a78012d2..436dff3a1 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,41 +1,41 @@ #!/usr/bin/env -S escript -c +%% -*- erlang-indent-level:4 -*- %% A script that adds changed modules to the corresponding appup files -main(_Args) -> - ChangedFiles = string:lexemes(os:cmd("git diff --name-only origin/master..HEAD"), "\n"), - AppModules0 = lists:filtermap(fun filter_erlang_modules/1, ChangedFiles), - %% emqx_app must always be included as we bump version number in emqx_release.hrl for each release - AppModules1 = [{emqx, emqx_app} | AppModules0], - AppModules = group_modules(AppModules1), - io:format("Changed modules: ~p~n", [AppModules]), - _ = maps:map(fun process_app/2, AppModules), - ok. +main(Args) -> + #{check := Check, current_release := CurrentRelease, prepare := Prepare} = + parse_args(Args, #{check => false, prepare => true}), + case find_pred_tag(CurrentRelease) of + {ok, Baseline} -> + {CurrDir, PredDir} = prepare(Baseline, Prepare), + Changed = diff_releases(CurrDir, PredDir), + _ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed), + ok; + undefined -> + io:format(standard_error, "No appup update is needed for this release, done~n", []), + ok + end. -process_app(App, Modules) -> +parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> + State#{current_release => CurrentRelease}; +parse_args(["--check"|Rest], State) -> + parse_args(Rest, State#{check => true}); +parse_args(["--skip-build"|Rest], State) -> + parse_args(Rest, State#{prepare => false}); +parse_args([], _) -> + fail("Usage:~n update_appup.escript [--check] [--skip-build] + + --check Don't update the appup files, just check that they are complete + --skip-build Don't rebuild the releases. May produce wrong appup files. +"). + +process_app(PredVersion, _Check, App, Changes) -> AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])), case AppupFiles of [AppupFile] -> - update_appup(AppupFile, Modules); - [] -> - io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App]) - end. - -filter_erlang_modules(Filename) -> - case lists:reverse(filename:split(Filename)) of - [Module, "src"] -> - erl_basename("emqx", Module); - [Module, "src", App|_] -> - erl_basename(App, Module); - [Module, _, "src", App|_] -> - erl_basename(App, Module); - _ -> - false - end. - -erl_basename(App, Name) -> - case filename:basename(Name, ".erl") of - Name -> false; - Module -> {true, {list_to_atom(App), list_to_atom(Module)}} + update_appup(PredVersion, AppupFile, Changes); + [] -> + io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App]) end. group_modules(L) -> @@ -43,33 +43,173 @@ group_modules(L) -> maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc) end, #{}, L). -update_appup(File, Modules) -> +update_appup(_, File, {[], [], []}) -> + %% No changes in the app. Just check syntax of the existing appup: + _ = read_appup(File); +update_appup(PredVersion, File, Changes) -> io:format("~nUpdating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), - Upgrade = update_actions(Modules, Upgrade0), - Downgrade = update_actions(Modules, Downgrade0), + Upgrade = update_actions(PredVersion, Changes, Upgrade0), + Downgrade = update_actions(PredVersion, Changes, Downgrade0), IOList = io_lib:format("%% -*- mode: erlang -*- {VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), - ok = file:write_file(File, IOList). + ok = file:write_file(File, IOList), + %% Check appup syntax: + _ = read_appup(File). -update_actions(Modules, Versions) -> - lists:map(fun(L) -> do_update_actions(Modules, L) end, Versions). +update_actions(PredVersion, Changes, Versions) -> + lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)). do_update_actions(_, Ret = {<<".*">>, _}) -> Ret; -do_update_actions(Modules, {Vsn, Actions}) -> - {Vsn, add_modules(Modules, Actions)}. +do_update_actions(Changes, {Vsn, Actions}) -> + {Vsn, process_changes(Changes, Actions)}. -add_modules(NewModules, OldActions) -> - OldModules = lists:map(fun(It) -> element(2, It) end, OldActions), - Modules = NewModules -- OldModules, - OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Modules]. +process_changes({New0, Changed0, Deleted0}, OldActions) -> + AlreadyHandled = lists:map(fun(It) -> element(2, It) end, OldActions), + New = New0 -- AlreadyHandled, + Changed = Changed0 -- AlreadyHandled, + Deleted = Deleted0 -- AlreadyHandled, + OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] + ++ [{delete_module, M} || M <- Deleted]. + +ensure_pred_version(PredVersion, Versions) -> + case lists:keyfind(PredVersion, 1, Versions) of + false -> + [{PredVersion, []}|Versions]; + _ -> + Versions + end. read_appup(File) -> - {ok, Bin0} = file:read_file(File), - %% Hack: - Bin1 = re:replace(Bin0, "VSN", "\"VSN\""), - TmpFile = filename:join("/tmp", filename:basename(File)), - ok = file:write_file(TmpFile, Bin1), - {ok, [Terms]} = file:consult(TmpFile), - Terms. + case file:script(File, [{'VSN', "VSN"}]) of + {ok, Terms} -> + Terms; + Error -> + fail("Failed to parse appup file ~s: ~p", [File, Error]) + end. + +diff_releases(CurrDir, OldDir) -> + Curr = hashsums(find_beams(CurrDir)), + Old = hashsums(find_beams(OldDir)), + Fun = fun(App, Modules, Acc) -> + OldModules = maps:get(App, Old, #{}), + Acc#{App => diff_app_modules(Modules, OldModules)} + end, + maps:fold(Fun, #{}, Curr). + +diff_app_modules(Modules, OldModules) -> + {New, Changed} = + maps:fold( fun(Mod, MD5, {New, Changed}) -> + case OldModules of + #{Mod := OldMD5} when MD5 =:= OldMD5 -> + {New, Changed}; + #{Mod := _} -> + {[Mod|New], Changed}; + _ -> {New, [Mod|Changed]} + end + end + , {[], []} + , Modules + ), + Deleted = maps:keys(maps:without(maps:keys(Modules), OldModules)), + {New, Changed, Deleted}. + +find_beams(Dir) -> + [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. + +prepare(Baseline, Prepare) -> + io:format("~n===================================~n" + "Baseline: ~s" + "~n===================================~n", [Baseline]), + io:format("Building the current version...~n"), + Prepare andalso success(cmd("make", #{args => ["emqx-rel"]}), "Failed to build HEAD"), + io:format("Downloading the preceding release...~n"), + {ok, PredRootDir} = build_pred_release(Baseline, Prepare), + BeamDir = "_build/emqx/rel/emqx/lib/", + {BeamDir, filename:join(PredRootDir, BeamDir)}. + +build_pred_release(Baseline, Prepare) -> + Repo = find_upstream_repo(), + BaseDir = "/tmp/emqx-baseline/", + Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], + %% TODO: shallow clone + Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && cd ${DIR} && make emqx-rel", + Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}], + Prepare andalso + success( cmd("bash", #{ args => ["-c", Script] + , env => Env + }) + , "Failed to build the baseline release" + ), + {ok, filename:join(BaseDir, Dir)}. + +%% @doc Find whether we are in emqx or emqx-ee +find_upstream_repo() -> + Str = os:cmd("git remote get-url origin"), + case re:run(Str, "/([^/]+).git$", [{capture, all_but_first, list}]) of + {match, ["emqx"]} -> "git@github.com:emqx/emqx.git"; + {match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git"; + Ret -> fail("Cannot detect the correct upstream repo: ~p", [Ret]) + end. + +find_pred_tag(CurrentRelease) -> + case re:run(CurrentRelease, "^([0-9]+)\.([0-9]+)\.([0-9]+)$", [{capture, all_but_first, list}]) of + {match, [Maj, Min, Patch]} -> + case list_to_integer(Patch) of + 0 -> undefined; + P -> {ok, lists:flatten(io_lib:format("~s.~s.~p", [Maj, Min, P - 1]))} + end; + Err -> + fail("The current release tag doesn't follow semver pattern: ~p", [Err]) + end. + +-spec hashsums(file:filename()) -> #{App => #{module() => binary()}} + when App :: atom(). +hashsums(Files) -> + hashsums(Files, #{}). + +hashsums([], Acc) -> + Acc; +hashsums([File|Rest], Acc0) -> + [_, "ebin", Dir|_] = lists:reverse(filename:split(File)), + {match, [AppStr]} = re:run(Dir, "^(.*)-[^-]+$", [{capture, all_but_first, list}]), + App = list_to_atom(AppStr), + {ok, {Module, MD5}} = beam_lib:md5(File), + Acc = maps:update_with( App + , fun(Old) -> Old #{Module => MD5} end + , #{Module => MD5} + , Acc0 + ), + hashsums(Rest, Acc). + +%% Spawn an executable and return the exit status +cmd(Exec, Params) -> + case os:find_executable(Exec) of + false -> + fail("Executable not found in $PATH: ~s", [Exec]); + Path -> + Params1 = maps:to_list(maps:with([env, args, cd], Params)), + Port = erlang:open_port( {spawn_executable, Path} + , [ exit_status + , nouse_stdio + | Params1 + ] + ), + receive + {Port, {exit_status, Status}} -> + Status + end + end. + +success(0, _) -> + true; +success(_, Msg) -> + fail(Msg). + +fail(Str) -> + fail(Str, []). + +fail(Str, Args) -> + io:format(standard_error, Str ++ "~n", Args), + halt(1). From aca6367561157a81e94d6f50fa4f5fdcc3a00ded Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 1 Oct 2021 18:35:17 +0200 Subject: [PATCH 02/11] feat(update_appup): Create stubs --- scripts/update_appup.escript | 60 ++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 436dff3a1..8f715cda9 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -23,25 +23,47 @@ parse_args(["--check"|Rest], State) -> parse_args(["--skip-build"|Rest], State) -> parse_args(Rest, State#{prepare => false}); parse_args([], _) -> - fail("Usage:~n update_appup.escript [--check] [--skip-build] + fail("A script that creates stubs for appup files + +Usage:~n update_appup.escript [--check] [--skip-build] --check Don't update the appup files, just check that they are complete --skip-build Don't rebuild the releases. May produce wrong appup files. "). +process_app(_, _, App, {[], [], []}) -> + %% No changes, just check the appup file if present: + case locate(App, ".appup.src") of + {ok, AppupFile} -> + _ = read_appup(AppupFile), + ok; + undefined -> + ok + end; process_app(PredVersion, _Check, App, Changes) -> - AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])), - case AppupFiles of - [AppupFile] -> + case locate(App, ".appup.src") of + {ok, AppupFile} -> update_appup(PredVersion, AppupFile, Changes); - [] -> - io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App]) + undefined -> + case create_stub(App) of + false -> + %% External dependency, skip + ok; + AppupFile -> + update_appup(PredVersion, AppupFile, Changes) + end end. -group_modules(L) -> - lists:foldl(fun({App, Mod}, Acc) -> - maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc) - end, #{}, L). +create_stub(App) -> + case locate(App, ".app.src") of + {ok, AppSrc} -> + AppupFile = filename:basename(AppSrc) ++ ".appup.src", + Default = {<<".*">>, []}, + render_appfile(AppupFile, [Default], [Default]), + AppupFile; + undefined -> + false + end. update_appup(_, File, {[], [], []}) -> %% No changes in the app. Just check syntax of the existing appup: @@ -51,12 +73,14 @@ update_appup(PredVersion, File, Changes) -> {_, Upgrade0, Downgrade0} = read_appup(File), Upgrade = update_actions(PredVersion, Changes, Upgrade0), Downgrade = update_actions(PredVersion, Changes, Downgrade0), - IOList = io_lib:format("%% -*- mode: erlang -*- -{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), - ok = file:write_file(File, IOList), + render_appfile(File, Upgrade, Downgrade), %% Check appup syntax: _ = read_appup(File). +render_appfile(File, Upgrade, Downgrade) -> + IOList = io_lib:format("%% -*- mode: erlang -*-\n{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), + ok = file:write_file(File, IOList). + update_actions(PredVersion, Changes, Versions) -> lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)). @@ -183,6 +207,16 @@ hashsums([File|Rest], Acc0) -> ), hashsums(Rest, Acc). +%% Locate a file in a specified application +locate(App, Suffix) -> + AppStr = atom_to_list(App), + case filelib:wildcard("{src,apps,lib-*}/**/" ++ AppStr ++ Suffix) of + [File] -> + {ok, File}; + [] -> + undefined + end. + %% Spawn an executable and return the exit status cmd(Exec, Params) -> case os:find_executable(Exec) of From f793883e35ae53dbb2eb19f96fd7e69e60e7c6b3 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 1 Oct 2021 22:58:13 +0200 Subject: [PATCH 03/11] refactor(update_appup): Minor code cleanup --- scripts/update_appup.escript | 102 ++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 8f715cda9..171b1a7e1 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -12,7 +12,7 @@ main(Args) -> _ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed), ok; undefined -> - io:format(standard_error, "No appup update is needed for this release, done~n", []), + log("No appup update is needed for this release, nothing to be done~n", []), ok end. @@ -25,10 +25,10 @@ parse_args(["--skip-build"|Rest], State) -> parse_args([], _) -> fail("A script that creates stubs for appup files -Usage:~n update_appup.escript [--check] [--skip-build] +Usage: update_appup.escript [--check] [--skip-build] --check Don't update the appup files, just check that they are complete - --skip-build Don't rebuild the releases. May produce wrong appup files. + --skip-build Don't rebuild the releases. May produce wrong appup files if changes are made. "). process_app(_, _, App, {[], [], []}) -> @@ -69,7 +69,7 @@ update_appup(_, File, {[], [], []}) -> %% No changes in the app. Just check syntax of the existing appup: _ = read_appup(File); update_appup(PredVersion, File, Changes) -> - io:format("~nUpdating appup: ~p~n", [File]), + log("Updating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), Upgrade = update_actions(PredVersion, Changes, Upgrade0), Downgrade = update_actions(PredVersion, Changes, Downgrade0), @@ -82,7 +82,7 @@ render_appfile(File, Upgrade, Downgrade) -> ok = file:write_file(File, IOList). update_actions(PredVersion, Changes, Versions) -> - lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)). + lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_versions(PredVersion, Versions)). do_update_actions(_, Ret = {<<".*">>, _}) -> Ret; @@ -97,10 +97,15 @@ process_changes({New0, Changed0, Deleted0}, OldActions) -> OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++ [{delete_module, M} || M <- Deleted]. -ensure_pred_version(PredVersion, Versions) -> - case lists:keyfind(PredVersion, 1, Versions) of +ensure_pred_versions(PredVersion, Versions) -> + {Maj, Min, Patch} = parse_semver(PredVersion), + PredVersions = [semver(Maj, Min, P) || P <- lists:seq(0, Patch)], + lists:foldl(fun ensure_version/2, Versions, PredVersions). + +ensure_version(Version, Versions) -> + case lists:keyfind(Version, 1, Versions) of false -> - [{PredVersion, []}|Versions]; + [{Version, []}|Versions]; _ -> Versions end. @@ -143,12 +148,12 @@ find_beams(Dir) -> [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. prepare(Baseline, Prepare) -> - io:format("~n===================================~n" - "Baseline: ~s" - "~n===================================~n", [Baseline]), - io:format("Building the current version...~n"), - Prepare andalso success(cmd("make", #{args => ["emqx-rel"]}), "Failed to build HEAD"), - io:format("Downloading the preceding release...~n"), + log("~n===================================~n" + "Baseline: ~s" + "~n===================================~n", [Baseline]), + log("Building the current version...~n"), + Prepare andalso bash("make emqx-rel"), + log("Downloading the preceding release...~n"), {ok, PredRootDir} = build_pred_release(Baseline, Prepare), BeamDir = "_build/emqx/rel/emqx/lib/", {BeamDir, filename:join(PredRootDir, BeamDir)}. @@ -158,34 +163,29 @@ build_pred_release(Baseline, Prepare) -> BaseDir = "/tmp/emqx-baseline/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], %% TODO: shallow clone - Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && cd ${DIR} && make emqx-rel", + Script = "mkdir -p ${BASEDIR} && + cd ${BASEDIR} && + { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && + cd ${DIR} && + make emqx-rel", Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}], - Prepare andalso - success( cmd("bash", #{ args => ["-c", Script] - , env => Env - }) - , "Failed to build the baseline release" - ), + Prepare andalso bash(Script, Env), {ok, filename:join(BaseDir, Dir)}. %% @doc Find whether we are in emqx or emqx-ee find_upstream_repo() -> Str = os:cmd("git remote get-url origin"), - case re:run(Str, "/([^/]+).git$", [{capture, all_but_first, list}]) of + case re(Str, "/([^/]+).git$") of {match, ["emqx"]} -> "git@github.com:emqx/emqx.git"; {match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git"; Ret -> fail("Cannot detect the correct upstream repo: ~p", [Ret]) end. find_pred_tag(CurrentRelease) -> - case re:run(CurrentRelease, "^([0-9]+)\.([0-9]+)\.([0-9]+)$", [{capture, all_but_first, list}]) of - {match, [Maj, Min, Patch]} -> - case list_to_integer(Patch) of - 0 -> undefined; - P -> {ok, lists:flatten(io_lib:format("~s.~s.~p", [Maj, Min, P - 1]))} - end; - Err -> - fail("The current release tag doesn't follow semver pattern: ~p", [Err]) + {Maj, Min, Patch} = parse_semver(CurrentRelease), + case Patch of + 0 -> undefined; + _ -> {ok, semver(Maj, Min, Patch - 1)} end. -spec hashsums(file:filename()) -> #{App => #{module() => binary()}} @@ -197,7 +197,7 @@ hashsums([], Acc) -> Acc; hashsums([File|Rest], Acc0) -> [_, "ebin", Dir|_] = lists:reverse(filename:split(File)), - {match, [AppStr]} = re:run(Dir, "^(.*)-[^-]+$", [{capture, all_but_first, list}]), + {match, [AppStr]} = re(Dir, "^(.*)-[^-]+$"), App = list_to_atom(AppStr), {ok, {Module, MD5}} = beam_lib:md5(File), Acc = maps:update_with( App @@ -207,6 +207,21 @@ hashsums([File|Rest], Acc0) -> ), hashsums(Rest, Acc). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Utility functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +parse_semver(Version) -> + case re(Version, "^([0-9]+)\.([0-9]+)\.([0-9]+)$") of + {match, [Maj, Min, Patch]} -> + {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; + Err -> + error({not_a_semver, Version}) + end. + +semver(Maj, Min, Patch) -> + lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])). + %% Locate a file in a specified application locate(App, Suffix) -> AppStr = atom_to_list(App), @@ -217,6 +232,15 @@ locate(App, Suffix) -> undefined end. +bash(Script) -> + bash(Script, []). + +bash(Script, Env) -> + case cmd("bash", #{args => ["-c", Script], env => Env}) of + 0 -> true; + _ -> fail("Failed to run command: ~s", [Script]) + end. + %% Spawn an executable and return the exit status cmd(Exec, Params) -> case os:find_executable(Exec) of @@ -236,14 +260,18 @@ cmd(Exec, Params) -> end end. -success(0, _) -> - true; -success(_, Msg) -> - fail(Msg). - fail(Str) -> fail(Str, []). fail(Str, Args) -> - io:format(standard_error, Str ++ "~n", Args), + log(Str ++ "~n", Args), halt(1). + +re(Subject, RE) -> + re:run(Subject, RE, [{capture, all_but_first, list}]). + +log(Msg) -> + log(Msg, []). + +log(Msg, Args) -> + io:format(standard_error, Msg, Args). From c50c72b18ea8d7d74b0e69363fc0d794a44dd06e Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 1 Oct 2021 23:50:45 +0200 Subject: [PATCH 04/11] fix(update_appup): Fix downgrade module loading --- scripts/update_appup.escript | 41 ++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 171b1a7e1..67b35f699 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -8,9 +8,16 @@ main(Args) -> case find_pred_tag(CurrentRelease) of {ok, Baseline} -> {CurrDir, PredDir} = prepare(Baseline, Prepare), - Changed = diff_releases(CurrDir, PredDir), - _ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed), - ok; + Upgrade = diff_releases(CurrDir, PredDir), + Downgrade = diff_releases(PredDir, CurrDir), + Apps = maps:keys(Upgrade), + lists:foreach( fun(App) -> + #{App := AppUpgrade} = Upgrade, + #{App := AppDowngrade} = Downgrade, + process_app(Baseline, Check, App, AppUpgrade, AppDowngrade) + end + , Apps + ); undefined -> log("No appup update is needed for this release, nothing to be done~n", []), ok @@ -31,7 +38,7 @@ Usage: update_appup.escript [--check] [--skip-build] --skip-build Don't rebuild the releases. May produce wrong appup files if changes are made. "). -process_app(_, _, App, {[], [], []}) -> +process_app(_, _, App, {[], [], []}, {[], [], []}) -> %% No changes, just check the appup file if present: case locate(App, ".appup.src") of {ok, AppupFile} -> @@ -40,17 +47,17 @@ process_app(_, _, App, {[], [], []}) -> undefined -> ok end; -process_app(PredVersion, _Check, App, Changes) -> +process_app(PredVersion, _Check, App, Upgrade, Downgrade) -> case locate(App, ".appup.src") of {ok, AppupFile} -> - update_appup(PredVersion, AppupFile, Changes); + update_appup(PredVersion, AppupFile, Upgrade, Downgrade); undefined -> case create_stub(App) of false -> %% External dependency, skip ok; AppupFile -> - update_appup(PredVersion, AppupFile, Changes) + update_appup(PredVersion, AppupFile, Upgrade, Downgrade) end end. @@ -65,14 +72,14 @@ create_stub(App) -> false end. -update_appup(_, File, {[], [], []}) -> +update_appup(_, File, {[], [], []}, {[], [], []}) -> %% No changes in the app. Just check syntax of the existing appup: _ = read_appup(File); -update_appup(PredVersion, File, Changes) -> +update_appup(PredVersion, File, UpgradeChanges, DowngradeChanges) -> log("Updating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), - Upgrade = update_actions(PredVersion, Changes, Upgrade0), - Downgrade = update_actions(PredVersion, Changes, Downgrade0), + Upgrade = update_actions(PredVersion, UpgradeChanges, Upgrade0), + Downgrade = update_actions(PredVersion, DowngradeChanges, Downgrade0), render_appfile(File, Upgrade, Downgrade), %% Check appup syntax: _ = read_appup(File). @@ -81,8 +88,10 @@ render_appfile(File, Upgrade, Downgrade) -> IOList = io_lib:format("%% -*- mode: erlang -*-\n{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), ok = file:write_file(File, IOList). -update_actions(PredVersion, Changes, Versions) -> - lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_versions(PredVersion, Versions)). +update_actions(PredVersion, Changes, Actions) -> + lists:map( fun(L) -> do_update_actions(Changes, L) end + , ensure_pred_versions(PredVersion, Actions) + ). do_update_actions(_, Ret = {<<".*">>, _}) -> Ret; @@ -134,8 +143,8 @@ diff_app_modules(Modules, OldModules) -> #{Mod := OldMD5} when MD5 =:= OldMD5 -> {New, Changed}; #{Mod := _} -> - {[Mod|New], Changed}; - _ -> {New, [Mod|Changed]} + {New, [Mod|Changed]}; + _ -> {[Mod|New], Changed} end end , {[], []} @@ -215,7 +224,7 @@ parse_semver(Version) -> case re(Version, "^([0-9]+)\.([0-9]+)\.([0-9]+)$") of {match, [Maj, Min, Patch]} -> {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; - Err -> + _ -> error({not_a_semver, Version}) end. From 6bee6279f8c7be26a3ba7336380db1a13e3be98d Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Sat, 2 Oct 2021 00:36:00 +0200 Subject: [PATCH 05/11] feat(update_appup): Make the script more generic --- scripts/update_appup.escript | 106 ++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 44 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 67b35f699..3bf856c3b 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -3,41 +3,70 @@ %% A script that adds changed modules to the corresponding appup files main(Args) -> - #{check := Check, current_release := CurrentRelease, prepare := Prepare} = - parse_args(Args, #{check => false, prepare => true}), + #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), case find_pred_tag(CurrentRelease) of {ok, Baseline} -> - {CurrDir, PredDir} = prepare(Baseline, Prepare), - Upgrade = diff_releases(CurrDir, PredDir), - Downgrade = diff_releases(PredDir, CurrDir), - Apps = maps:keys(Upgrade), - lists:foreach( fun(App) -> - #{App := AppUpgrade} = Upgrade, - #{App := AppDowngrade} = Downgrade, - process_app(Baseline, Check, App, AppUpgrade, AppDowngrade) - end - , Apps - ); + main(Options, Baseline); undefined -> log("No appup update is needed for this release, nothing to be done~n", []), ok end. +default_options() -> + #{ check => false + , prepare => true + , clone_url => find_upstream_repo("origin") + , make_command => "make emqx-rel" + , beams_dir => "_build/emqx/rel/emqx/lib/" + }. + parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> State#{current_release => CurrentRelease}; parse_args(["--check"|Rest], State) -> parse_args(Rest, State#{check => true}); parse_args(["--skip-build"|Rest], State) -> parse_args(Rest, State#{prepare => false}); -parse_args([], _) -> - fail("A script that creates stubs for appup files +parse_args(["--repo", Repo|Rest], State) -> + parse_args(Rest, State#{clone_url => Repo}); +parse_args(["--remote", Remote|Rest], State) -> + parse_args(Rest, State#{clone_url => find_upstream_repo(Remote)}); +parse_args(["--make-command", Command|Rest], State) -> + parse_args(Rest, State#{make_command => Command}); +parse_args(["--release-dir", Dir|Rest], State) -> + parse_args(Rest, State#{beams_dir => Dir}); +parse_args(_, _) -> + fail("A script that fills in boilerplate for appup files. +Note: The defaults are set up for emqx, but they can be tuned to support other repos too. -Usage: update_appup.escript [--check] [--skip-build] +Usage: - --check Don't update the appup files, just check that they are complete - --skip-build Don't rebuild the releases. May produce wrong appup files if changes are made. + update_appup.escript [--check] [--repo URL] [--remote NAME] [--skip-build] [--make-commad SCRIPT] [--release-dir DIR] + +Options: + + --check Don't update the appup files, just check that they are complete + --repo Upsteam git repo URL + --remote Get upstream repo URL from the specified git remote + --skip-build Don't rebuild the releases. May produce wrong results + --make-command A command used to assemble the release + --release-dir Directory where the release is build "). +main(Options = #{check := Check}, Baseline) -> + {CurrDir, PredDir} = prepare(Baseline, Options), + CurrBeams = hashsums(find_beams(CurrDir)), + PredBeams = hashsums(find_beams(PredDir)), + Upgrade = diff_releases(CurrBeams, PredBeams), + Downgrade = diff_releases(PredBeams, CurrBeams), + Apps = maps:keys(Upgrade), + lists:foreach( fun(App) -> + #{App := AppUpgrade} = Upgrade, + #{App := AppDowngrade} = Downgrade, + process_app(Baseline, Check, App, AppUpgrade, AppDowngrade) + end + , Apps + ). + process_app(_, _, App, {[], [], []}, {[], [], []}) -> %% No changes, just check the appup file if present: case locate(App, ".appup.src") of @@ -72,9 +101,6 @@ create_stub(App) -> false end. -update_appup(_, File, {[], [], []}, {[], [], []}) -> - %% No changes in the app. Just check syntax of the existing appup: - _ = read_appup(File); update_appup(PredVersion, File, UpgradeChanges, DowngradeChanges) -> log("Updating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), @@ -103,8 +129,9 @@ process_changes({New0, Changed0, Deleted0}, OldActions) -> New = New0 -- AlreadyHandled, Changed = Changed0 -- AlreadyHandled, Deleted = Deleted0 -- AlreadyHandled, - OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] - ++ [{delete_module, M} || M <- Deleted]. + [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++ + [{delete_module, M} || M <- Deleted] ++ + OldActions. ensure_pred_versions(PredVersion, Versions) -> {Maj, Min, Patch} = parse_semver(PredVersion), @@ -127,9 +154,7 @@ read_appup(File) -> fail("Failed to parse appup file ~s: ~p", [File, Error]) end. -diff_releases(CurrDir, OldDir) -> - Curr = hashsums(find_beams(CurrDir)), - Old = hashsums(find_beams(OldDir)), +diff_releases(Curr, Old) -> Fun = fun(App, Modules, Acc) -> OldModules = maps:get(App, Old, #{}), Acc#{App => diff_app_modules(Modules, OldModules)} @@ -156,39 +181,32 @@ diff_app_modules(Modules, OldModules) -> find_beams(Dir) -> [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. -prepare(Baseline, Prepare) -> +prepare(_, #{prepare := false}) -> + ok; +prepare(Baseline, #{clone_url := Repo, make_command := MakeCommand, beams_dir := BeamDir}) -> log("~n===================================~n" "Baseline: ~s" "~n===================================~n", [Baseline]), log("Building the current version...~n"), - Prepare andalso bash("make emqx-rel"), - log("Downloading the preceding release...~n"), - {ok, PredRootDir} = build_pred_release(Baseline, Prepare), - BeamDir = "_build/emqx/rel/emqx/lib/", + bash(MakeCommand), + log("Downloading and building the previous release...~n"), + {ok, PredRootDir} = build_pred_release(Baseline, Repo, MakeCommand), {BeamDir, filename:join(PredRootDir, BeamDir)}. -build_pred_release(Baseline, Prepare) -> - Repo = find_upstream_repo(), +build_pred_release(Baseline, Repo, MakeCommand) -> BaseDir = "/tmp/emqx-baseline/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], %% TODO: shallow clone Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && - cd ${DIR} && - make emqx-rel", + cd ${DIR} &&" ++ MakeCommand, Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}], - Prepare andalso bash(Script, Env), + bash(Script, Env), {ok, filename:join(BaseDir, Dir)}. -%% @doc Find whether we are in emqx or emqx-ee -find_upstream_repo() -> - Str = os:cmd("git remote get-url origin"), - case re(Str, "/([^/]+).git$") of - {match, ["emqx"]} -> "git@github.com:emqx/emqx.git"; - {match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git"; - Ret -> fail("Cannot detect the correct upstream repo: ~p", [Ret]) - end. +find_upstream_repo(Remote) -> + string:trim(os:cmd("git remote get-url " ++ Remote)). find_pred_tag(CurrentRelease) -> {Maj, Min, Patch} = parse_semver(CurrentRelease), From ca77749281712dfad99b13a10adb7aa4aeabd36f Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Sat, 2 Oct 2021 10:16:30 +0200 Subject: [PATCH 06/11] feat(update_appup): Add a full description of the algorithm --- scripts/update_appup.escript | 89 ++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 3bf856c3b..214c03141 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,42 +1,19 @@ #!/usr/bin/env -S escript -c %% -*- erlang-indent-level:4 -*- -%% A script that adds changed modules to the corresponding appup files -main(Args) -> - #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), - case find_pred_tag(CurrentRelease) of - {ok, Baseline} -> - main(Options, Baseline); - undefined -> - log("No appup update is needed for this release, nothing to be done~n", []), - ok - end. +usage() -> +"A script that fills in boilerplate for appup files. -default_options() -> - #{ check => false - , prepare => true - , clone_url => find_upstream_repo("origin") - , make_command => "make emqx-rel" - , beams_dir => "_build/emqx/rel/emqx/lib/" - }. +Algorithm: this script compares md5s of beam files of each +application, and creates a `{load_module, Module, brutal_purge, +soft_purge, []}` action for the changed and new modules. For deleted +modules it creates `{delete_module, M}` action. These entries are +added to each patch release preceding the current release. If an entry +for a module already exists, this module is ignored. The existing +actions are kept. -parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> - State#{current_release => CurrentRelease}; -parse_args(["--check"|Rest], State) -> - parse_args(Rest, State#{check => true}); -parse_args(["--skip-build"|Rest], State) -> - parse_args(Rest, State#{prepare => false}); -parse_args(["--repo", Repo|Rest], State) -> - parse_args(Rest, State#{clone_url => Repo}); -parse_args(["--remote", Remote|Rest], State) -> - parse_args(Rest, State#{clone_url => find_upstream_repo(Remote)}); -parse_args(["--make-command", Command|Rest], State) -> - parse_args(Rest, State#{make_command => Command}); -parse_args(["--release-dir", Dir|Rest], State) -> - parse_args(Rest, State#{beams_dir => Dir}); -parse_args(_, _) -> - fail("A script that fills in boilerplate for appup files. -Note: The defaults are set up for emqx, but they can be tuned to support other repos too. +Note: The defaults are set up for emqx, but they can be tuned to +support other repos too. Usage: @@ -49,8 +26,42 @@ Options: --remote Get upstream repo URL from the specified git remote --skip-build Don't rebuild the releases. May produce wrong results --make-command A command used to assemble the release - --release-dir Directory where the release is build -"). + --release-dir Release directory +". + +default_options() -> + #{ check => false + , clone_url => find_upstream_repo("origin") + , make_command => "make emqx-rel" + , beams_dir => "_build/emqx/rel/emqx/lib/" + }. + +main(Args) -> + #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), + case find_pred_tag(CurrentRelease) of + {ok, Baseline} -> + main(Options, Baseline); + undefined -> + log("No appup update is needed for this release, nothing to be done~n", []), + ok + end. + +parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> + State#{current_release => CurrentRelease}; +parse_args(["--check"|Rest], State) -> + parse_args(Rest, State#{check => true}); +parse_args(["--skip-build"|Rest], State) -> + parse_args(Rest, State#{make_command => "true"}); +parse_args(["--repo", Repo|Rest], State) -> + parse_args(Rest, State#{clone_url => Repo}); +parse_args(["--remote", Remote|Rest], State) -> + parse_args(Rest, State#{clone_url => find_upstream_repo(Remote)}); +parse_args(["--make-command", Command|Rest], State) -> + parse_args(Rest, State#{make_command => Command}); +parse_args(["--release-dir", Dir|Rest], State) -> + parse_args(Rest, State#{beams_dir => Dir}); +parse_args(_, _) -> + fail(usage()). main(Options = #{check := Check}, Baseline) -> {CurrDir, PredDir} = prepare(Baseline, Options), @@ -130,8 +141,8 @@ process_changes({New0, Changed0, Deleted0}, OldActions) -> Changed = Changed0 -- AlreadyHandled, Deleted = Deleted0 -- AlreadyHandled, [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++ - [{delete_module, M} || M <- Deleted] ++ - OldActions. + OldActions ++ + [{delete_module, M} || M <- Deleted]. ensure_pred_versions(PredVersion, Versions) -> {Maj, Min, Patch} = parse_semver(PredVersion), @@ -181,8 +192,6 @@ diff_app_modules(Modules, OldModules) -> find_beams(Dir) -> [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. -prepare(_, #{prepare := false}) -> - ok; prepare(Baseline, #{clone_url := Repo, make_command := MakeCommand, beams_dir := BeamDir}) -> log("~n===================================~n" "Baseline: ~s" From 89fbf5fea21305b708cbe600f1d028d517799183 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Mon, 4 Oct 2021 19:40:51 +0200 Subject: [PATCH 07/11] feat(update_appup): Add a warning message --- scripts/update_appup.escript | 43 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 214c03141..1eb769252 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -12,6 +12,10 @@ added to each patch release preceding the current release. If an entry for a module already exists, this module is ignored. The existing actions are kept. +Please note that it only compares the current release with its +predecessor, assuming that the upgrade actions for the older releases +are correct. + Note: The defaults are set up for emqx, but they can be tuned to support other repos too. @@ -76,7 +80,12 @@ main(Options = #{check := Check}, Baseline) -> process_app(Baseline, Check, App, AppUpgrade, AppDowngrade) end , Apps - ). + ), + log(" +NOTE: Please review the changes manually. This script does not know about NIF +changes, supervisor changes, process restarts and so on. Also the load order of +the beam files might need updating. +"). process_app(_, _, App, {[], [], []}, {[], [], []}) -> %% No changes, just check the appup file if present: @@ -87,17 +96,17 @@ process_app(_, _, App, {[], [], []}, {[], [], []}) -> undefined -> ok end; -process_app(PredVersion, _Check, App, Upgrade, Downgrade) -> +process_app(PredVersion, Check, App, Upgrade, Downgrade) -> case locate(App, ".appup.src") of {ok, AppupFile} -> - update_appup(PredVersion, AppupFile, Upgrade, Downgrade); + update_appup(Check, PredVersion, AppupFile, Upgrade, Downgrade); undefined -> case create_stub(App) of false -> %% External dependency, skip ok; AppupFile -> - update_appup(PredVersion, AppupFile, Upgrade, Downgrade) + update_appup(Check, PredVersion, AppupFile, Upgrade, Downgrade) end end. @@ -112,7 +121,7 @@ create_stub(App) -> false end. -update_appup(PredVersion, File, UpgradeChanges, DowngradeChanges) -> +update_appup(Check, PredVersion, File, UpgradeChanges, DowngradeChanges) -> log("Updating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), Upgrade = update_actions(PredVersion, UpgradeChanges, Upgrade0), @@ -136,7 +145,7 @@ do_update_actions(Changes, {Vsn, Actions}) -> {Vsn, process_changes(Changes, Actions)}. process_changes({New0, Changed0, Deleted0}, OldActions) -> - AlreadyHandled = lists:map(fun(It) -> element(2, It) end, OldActions), + AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)), New = New0 -- AlreadyHandled, Changed = Changed0 -- AlreadyHandled, Deleted = Deleted0 -- AlreadyHandled, @@ -144,6 +153,18 @@ process_changes({New0, Changed0, Deleted0}, OldActions) -> OldActions ++ [{delete_module, M} || M <- Deleted]. +%% @doc Process the existing actions to exclude modules that are +%% already handled +process_old_action({purge, Modules}) -> + Modules; +process_old_action({delete_module, Module}) -> + [Module]; +process_old_action(LoadModule) when is_tuple(LoadModule) andalso + element(1, LoadModule) =:= load_module -> + element(2, LoadModule); +process_old_action(_) -> + []. + ensure_pred_versions(PredVersion, Versions) -> {Maj, Min, Patch} = parse_semver(PredVersion), PredVersions = [semver(Maj, Min, P) || P <- lists:seq(0, Patch)], @@ -192,17 +213,17 @@ diff_app_modules(Modules, OldModules) -> find_beams(Dir) -> [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. -prepare(Baseline, #{clone_url := Repo, make_command := MakeCommand, beams_dir := BeamDir}) -> +prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> log("~n===================================~n" "Baseline: ~s" "~n===================================~n", [Baseline]), log("Building the current version...~n"), bash(MakeCommand), log("Downloading and building the previous release...~n"), - {ok, PredRootDir} = build_pred_release(Baseline, Repo, MakeCommand), + {ok, PredRootDir} = build_pred_release(Baseline, Options), {BeamDir, filename:join(PredRootDir, BeamDir)}. -build_pred_release(Baseline, Repo, MakeCommand) -> +build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> BaseDir = "/tmp/emqx-baseline/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], %% TODO: shallow clone @@ -248,8 +269,8 @@ hashsums([File|Rest], Acc0) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% parse_semver(Version) -> - case re(Version, "^([0-9]+)\.([0-9]+)\.([0-9]+)$") of - {match, [Maj, Min, Patch]} -> + case re(Version, "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.[0-9]+)?$") of + {match, [Maj, Min, Patch|_]} -> {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; _ -> error({not_a_semver, Version}) From ecf4d196eb0e3b511064df1aec5026a562f69fb7 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:11:20 +0200 Subject: [PATCH 08/11] feat(update_appup): Return error when missing appup.src --- scripts/update_appup.escript | 50 ++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 1eb769252..8c3702644 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -25,7 +25,6 @@ Usage: Options: - --check Don't update the appup files, just check that they are complete --repo Upsteam git repo URL --remote Get upstream repo URL from the specified git remote --skip-build Don't rebuild the releases. May produce wrong results @@ -34,13 +33,13 @@ Options: ". default_options() -> - #{ check => false - , clone_url => find_upstream_repo("origin") + #{ clone_url => find_upstream_repo("origin") , make_command => "make emqx-rel" , beams_dir => "_build/emqx/rel/emqx/lib/" }. main(Args) -> + put(update_appup_valid, true), #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), case find_pred_tag(CurrentRelease) of {ok, Baseline} -> @@ -52,8 +51,6 @@ main(Args) -> parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> State#{current_release => CurrentRelease}; -parse_args(["--check"|Rest], State) -> - parse_args(Rest, State#{check => true}); parse_args(["--skip-build"|Rest], State) -> parse_args(Rest, State#{make_command => "true"}); parse_args(["--repo", Repo|Rest], State) -> @@ -67,27 +64,37 @@ parse_args(["--release-dir", Dir|Rest], State) -> parse_args(_, _) -> fail(usage()). -main(Options = #{check := Check}, Baseline) -> +main(Options, Baseline) -> {CurrDir, PredDir} = prepare(Baseline, Options), + log("~n===================================~n" + "Processing changes..." + "~n===================================~n"), CurrBeams = hashsums(find_beams(CurrDir)), PredBeams = hashsums(find_beams(PredDir)), Upgrade = diff_releases(CurrBeams, PredBeams), Downgrade = diff_releases(PredBeams, CurrBeams), Apps = maps:keys(Upgrade), lists:foreach( fun(App) -> + %% TODO: Here we can find new and deleted apps and handle them accordingly #{App := AppUpgrade} = Upgrade, #{App := AppDowngrade} = Downgrade, - process_app(Baseline, Check, App, AppUpgrade, AppDowngrade) + process_app(Baseline, App, AppUpgrade, AppDowngrade) end , Apps ), + warn_and_exit(is_valid()). + +warn_and_exit(true) -> log(" NOTE: Please review the changes manually. This script does not know about NIF changes, supervisor changes, process restarts and so on. Also the load order of -the beam files might need updating. -"). +the beam files might need updating.~n"), + halt(0); +warn_and_exit(false) -> + log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"), + halt(1). -process_app(_, _, App, {[], [], []}, {[], [], []}) -> +process_app(_, App, {[], [], []}, {[], [], []}) -> %% No changes, just check the appup file if present: case locate(App, ".appup.src") of {ok, AppupFile} -> @@ -96,17 +103,19 @@ process_app(_, _, App, {[], [], []}, {[], [], []}) -> undefined -> ok end; -process_app(PredVersion, Check, App, Upgrade, Downgrade) -> +process_app(PredVersion, App, Upgrade, Downgrade) -> case locate(App, ".appup.src") of {ok, AppupFile} -> - update_appup(Check, PredVersion, AppupFile, Upgrade, Downgrade); + update_appup(PredVersion, AppupFile, Upgrade, Downgrade); undefined -> case create_stub(App) of false -> - %% External dependency, skip + set_invalid(), + log("ERROR: External dependency '~p' contains changes, but the appup.src file is NOT updated. + Create a patch to the upstream to resolve this issue.~n", [App]), ok; AppupFile -> - update_appup(Check, PredVersion, AppupFile, Upgrade, Downgrade) + update_appup(PredVersion, AppupFile, Upgrade, Downgrade) end end. @@ -121,8 +130,8 @@ create_stub(App) -> false end. -update_appup(Check, PredVersion, File, UpgradeChanges, DowngradeChanges) -> - log("Updating appup: ~p~n", [File]), +update_appup(PredVersion, File, UpgradeChanges, DowngradeChanges) -> + log("INFO: Updating appup: ~s~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), Upgrade = update_actions(PredVersion, UpgradeChanges, Upgrade0), Downgrade = update_actions(PredVersion, DowngradeChanges, Downgrade0), @@ -229,7 +238,7 @@ build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) %% TODO: shallow clone Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && - { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && + { [ -d ${DIR} ] || git clone --branch ${TAG} ${REPO} ${DIR}; } && cd ${DIR} &&" ++ MakeCommand, Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}], bash(Script, Env), @@ -264,6 +273,13 @@ hashsums([File|Rest], Acc0) -> ), hashsums(Rest, Acc). +%% Set a global flag that something about the appfiles is invalid +set_invalid() -> + put(update_appup_invalid, false). + +is_valid() -> + get(update_appup_invalid). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utility functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% From 29ad2c04da048d89bed0ca55f02be32a287865eb Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 6 Oct 2021 17:37:33 +0200 Subject: [PATCH 09/11] fix: Use application version instead of the release version --- scripts/update_appup.escript | 407 +++++++++++++++++++++-------------- 1 file changed, 240 insertions(+), 167 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 8c3702644..de8605f5a 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -25,22 +25,33 @@ Usage: Options: + --check Don't update the appfile, just check that they are complete + --prev-tag Specify the previous release tag. Otherwise the previous patch version is used --repo Upsteam git repo URL --remote Get upstream repo URL from the specified git remote --skip-build Don't rebuild the releases. May produce wrong results --make-command A command used to assemble the release --release-dir Release directory + --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' ". +-record(app, + { modules :: #{module() => binary()} + , version :: string() + }). + default_options() -> #{ clone_url => find_upstream_repo("origin") , make_command => "make emqx-rel" , beams_dir => "_build/emqx/rel/emqx/lib/" + , check => false + , prev_tag => undefined + , src_dirs => "{src,apps,lib-*}/**/" }. main(Args) -> - put(update_appup_valid, true), #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), + init_globals(Options), case find_pred_tag(CurrentRelease) of {ok, Baseline} -> main(Options, Baseline); @@ -51,6 +62,8 @@ main(Args) -> parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> State#{current_release => CurrentRelease}; +parse_args(["--check"|Rest], State) -> + parse_args(Rest, State#{check => true}); parse_args(["--skip-build"|Rest], State) -> parse_args(Rest, State#{make_command => "true"}); parse_args(["--repo", Repo|Rest], State) -> @@ -65,23 +78,27 @@ parse_args(_, _) -> fail(usage()). main(Options, Baseline) -> - {CurrDir, PredDir} = prepare(Baseline, Options), + {CurrRelDir, PredRelDir} = prepare(Baseline, Options), log("~n===================================~n" "Processing changes..." "~n===================================~n"), - CurrBeams = hashsums(find_beams(CurrDir)), - PredBeams = hashsums(find_beams(PredDir)), - Upgrade = diff_releases(CurrBeams, PredBeams), - Downgrade = diff_releases(PredBeams, CurrBeams), - Apps = maps:keys(Upgrade), - lists:foreach( fun(App) -> - %% TODO: Here we can find new and deleted apps and handle them accordingly - #{App := AppUpgrade} = Upgrade, - #{App := AppDowngrade} = Downgrade, - process_app(Baseline, App, AppUpgrade, AppDowngrade) - end - , Apps - ), + CurrAppsIdx = index_apps(CurrRelDir), + PredAppsIdx = index_apps(PredRelDir), + %% log("Curr: ~p~nPred: ~p~n", [CurrApps, PredApps]), + AppupChanges = find_appup_actions(CurrAppsIdx, PredAppsIdx), + case getopt(check) of + true -> + case AppupChanges of + [] -> + ok; + _ -> + set_invalid(), + log("ERROR: The appup files are incomplete. Missing changes:~n ~p", [AppupChanges]) + end; + false -> + update_appups(AppupChanges) + end, + check_appup_files(), warn_and_exit(is_valid()). warn_and_exit(true) -> @@ -94,134 +111,6 @@ warn_and_exit(false) -> log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"), halt(1). -process_app(_, App, {[], [], []}, {[], [], []}) -> - %% No changes, just check the appup file if present: - case locate(App, ".appup.src") of - {ok, AppupFile} -> - _ = read_appup(AppupFile), - ok; - undefined -> - ok - end; -process_app(PredVersion, App, Upgrade, Downgrade) -> - case locate(App, ".appup.src") of - {ok, AppupFile} -> - update_appup(PredVersion, AppupFile, Upgrade, Downgrade); - undefined -> - case create_stub(App) of - false -> - set_invalid(), - log("ERROR: External dependency '~p' contains changes, but the appup.src file is NOT updated. - Create a patch to the upstream to resolve this issue.~n", [App]), - ok; - AppupFile -> - update_appup(PredVersion, AppupFile, Upgrade, Downgrade) - end - end. - -create_stub(App) -> - case locate(App, ".app.src") of - {ok, AppSrc} -> - AppupFile = filename:basename(AppSrc) ++ ".appup.src", - Default = {<<".*">>, []}, - render_appfile(AppupFile, [Default], [Default]), - AppupFile; - undefined -> - false - end. - -update_appup(PredVersion, File, UpgradeChanges, DowngradeChanges) -> - log("INFO: Updating appup: ~s~n", [File]), - {_, Upgrade0, Downgrade0} = read_appup(File), - Upgrade = update_actions(PredVersion, UpgradeChanges, Upgrade0), - Downgrade = update_actions(PredVersion, DowngradeChanges, Downgrade0), - render_appfile(File, Upgrade, Downgrade), - %% Check appup syntax: - _ = read_appup(File). - -render_appfile(File, Upgrade, Downgrade) -> - IOList = io_lib:format("%% -*- mode: erlang -*-\n{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), - ok = file:write_file(File, IOList). - -update_actions(PredVersion, Changes, Actions) -> - lists:map( fun(L) -> do_update_actions(Changes, L) end - , ensure_pred_versions(PredVersion, Actions) - ). - -do_update_actions(_, Ret = {<<".*">>, _}) -> - Ret; -do_update_actions(Changes, {Vsn, Actions}) -> - {Vsn, process_changes(Changes, Actions)}. - -process_changes({New0, Changed0, Deleted0}, OldActions) -> - AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)), - New = New0 -- AlreadyHandled, - Changed = Changed0 -- AlreadyHandled, - Deleted = Deleted0 -- AlreadyHandled, - [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++ - OldActions ++ - [{delete_module, M} || M <- Deleted]. - -%% @doc Process the existing actions to exclude modules that are -%% already handled -process_old_action({purge, Modules}) -> - Modules; -process_old_action({delete_module, Module}) -> - [Module]; -process_old_action(LoadModule) when is_tuple(LoadModule) andalso - element(1, LoadModule) =:= load_module -> - element(2, LoadModule); -process_old_action(_) -> - []. - -ensure_pred_versions(PredVersion, Versions) -> - {Maj, Min, Patch} = parse_semver(PredVersion), - PredVersions = [semver(Maj, Min, P) || P <- lists:seq(0, Patch)], - lists:foldl(fun ensure_version/2, Versions, PredVersions). - -ensure_version(Version, Versions) -> - case lists:keyfind(Version, 1, Versions) of - false -> - [{Version, []}|Versions]; - _ -> - Versions - end. - -read_appup(File) -> - case file:script(File, [{'VSN', "VSN"}]) of - {ok, Terms} -> - Terms; - Error -> - fail("Failed to parse appup file ~s: ~p", [File, Error]) - end. - -diff_releases(Curr, Old) -> - Fun = fun(App, Modules, Acc) -> - OldModules = maps:get(App, Old, #{}), - Acc#{App => diff_app_modules(Modules, OldModules)} - end, - maps:fold(Fun, #{}, Curr). - -diff_app_modules(Modules, OldModules) -> - {New, Changed} = - maps:fold( fun(Mod, MD5, {New, Changed}) -> - case OldModules of - #{Mod := OldMD5} when MD5 =:= OldMD5 -> - {New, Changed}; - #{Mod := _} -> - {New, [Mod|Changed]}; - _ -> {[Mod|New], Changed} - end - end - , {[], []} - , Modules - ), - Deleted = maps:keys(maps:without(maps:keys(Modules), OldModules)), - {New, Changed, Deleted}. - -find_beams(Dir) -> - [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. - prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> log("~n===================================~n" "Baseline: ~s" @@ -248,37 +137,220 @@ find_upstream_repo(Remote) -> string:trim(os:cmd("git remote get-url " ++ Remote)). find_pred_tag(CurrentRelease) -> - {Maj, Min, Patch} = parse_semver(CurrentRelease), - case Patch of - 0 -> undefined; - _ -> {ok, semver(Maj, Min, Patch - 1)} + case getopt(prev_tag) of + undefined -> + {Maj, Min, Patch} = parse_semver(CurrentRelease), + case Patch of + 0 -> undefined; + _ -> {ok, semver(Maj, Min, Patch - 1)} + end; + Tag -> + {ok, Tag} end. --spec hashsums(file:filename()) -> #{App => #{module() => binary()}} - when App :: atom(). -hashsums(Files) -> - hashsums(Files, #{}). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Appup action creation and updating +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -hashsums([], Acc) -> - Acc; -hashsums([File|Rest], Acc0) -> - [_, "ebin", Dir|_] = lists:reverse(filename:split(File)), - {match, [AppStr]} = re(Dir, "^(.*)-[^-]+$"), - App = list_to_atom(AppStr), - {ok, {Module, MD5}} = beam_lib:md5(File), - Acc = maps:update_with( App - , fun(Old) -> Old #{Module => MD5} end - , #{Module => MD5} - , Acc0 - ), - hashsums(Rest, Acc). +find_appup_actions(CurrApps, PredApps) -> + maps:fold( + fun(App, CurrAppIdx, Acc) -> + case PredApps of + #{App := PredAppIdx} -> find_appup_actions(App, CurrAppIdx, PredAppIdx) ++ Acc; + _ -> Acc %% New app, nothing to upgrade here. + end + end, + [], + CurrApps). + +find_appup_actions(_App, AppIdx, AppIdx) -> + %% No changes to the app, ignore: + []; +find_appup_actions(App, CurrAppIdx, PredAppIdx = #app{version = PredVersion}) -> + {OldUpgrade, OldDowngrade} = find_old_appup_actions(App, PredVersion), + Upgrade = merge_update_actions(diff_app(App, CurrAppIdx, PredAppIdx), OldUpgrade), + Downgrade = merge_update_actions(diff_app(App, PredAppIdx, CurrAppIdx), OldDowngrade), + if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade -> + %% The appup file has been already updated: + []; + true -> + [{App, {Upgrade, Downgrade}}] + end. + +find_old_appup_actions(App, PredVersion) -> + {Upgrade0, Downgrade0} = + case locate(App, ".appup.src") of + {ok, AppupFile} -> + {_, U, D} = read_appup(AppupFile), + {U, D}; + undefined -> + {[], []} + end, + {ensure_version(PredVersion, Upgrade0), ensure_version(PredVersion, Downgrade0)}. + +merge_update_actions(Changes, Vsns) -> + lists:map(fun(Ret = {<<".*">>, _}) -> + Ret; + ({Vsn, Actions}) -> + {Vsn, do_merge_update_actions(Changes, Actions)} + end, + Vsns). + +do_merge_update_actions({New0, Changed0, Deleted0}, OldActions) -> + AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)), + New = New0 -- AlreadyHandled, + Changed = Changed0 -- AlreadyHandled, + Deleted = Deleted0 -- AlreadyHandled, + [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] ++ + OldActions ++ + [{delete_module, M} || M <- Deleted]. + + +%% @doc Process the existing actions to exclude modules that are +%% already handled +process_old_action({purge, Modules}) -> + Modules; +process_old_action({delete_module, Module}) -> + [Module]; +process_old_action(LoadModule) when is_tuple(LoadModule) andalso + element(1, LoadModule) =:= load_module -> + element(2, LoadModule); +process_old_action(_) -> + []. + +ensure_version(Version, Versions) -> + case lists:keyfind(Version, 1, Versions) of + false -> + [{Version, []}|Versions]; + _ -> + Versions + end. + +read_appup(File) -> + %% NOTE: appup file is a script, it may contain variables or functions. + case file:script(File, [{'VSN', "VSN"}]) of + {ok, Terms} -> + Terms; + Error -> + fail("Failed to parse appup file ~s: ~p", [File, Error]) + end. + +check_appup_files() -> + AppupFiles = filelib:wildcard(getopt(src_dirs) ++ "/*.appup.src"), + lists:foreach(fun read_appup/1, AppupFiles). + +update_appups(Changes) -> + lists:foreach( + fun({App, {Upgrade, Downgrade}}) -> + do_update_appup(App, Upgrade, Downgrade) + end, + Changes). + +do_update_appup(App, Upgrade, Downgrade) -> + case locate(App, ".appup.src") of + {ok, AppupFile} -> + render_appfile(AppupFile, Upgrade, Downgrade); + undefined -> + case create_stub(App) of + {ok, AppupFile} -> + render_appfile(AppupFile, Upgrade, Downgrade); + false -> + set_invalid(), + log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p", [App, Upgrade]) + end + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Appup file creation +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +render_appfile(File, Upgrade, Downgrade) -> + IOList = io_lib:format("%% -*- mode: erlang -*-\n{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), + ok = file:write_file(File, IOList). + +create_stub(App) -> + case locate(App, ".app.src") of + {ok, AppSrc} -> + AppupFile = filename:basename(AppSrc) ++ ".appup.src", + Default = {<<".*">>, []}, + render_appfile(AppupFile, [Default], [Default]), + AppupFile; + undefined -> + false + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% application and release indexing +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +index_apps(ReleaseDir) -> + maps:from_list([index_app(filename:join(ReleaseDir, AppFile)) || + AppFile <- filelib:wildcard("**/ebin/*.app", ReleaseDir)]). + +index_app(AppFile) -> + {ok, [{application, App, Properties}]} = file:consult(AppFile), + Vsn = proplists:get_value(vsn, Properties), + %% Note: assuming that beams are always located in the same directory where app file is: + EbinDir = filename:dirname(AppFile), + Modules = hashsums(EbinDir), + {App, #app{ version = Vsn + , modules = Modules + }}. + +diff_app(App, #app{version = NewVersion, modules = NewModules}, #app{version = OldVersion, modules = OldModules}) -> + {New, Changed} = + maps:fold( fun(Mod, MD5, {New, Changed}) -> + case OldModules of + #{Mod := OldMD5} when MD5 =:= OldMD5 -> + {New, Changed}; + #{Mod := _} -> + {New, [Mod|Changed]}; + _ -> + {[Mod|New], Changed} + end + end + , {[], []} + , NewModules + ), + Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)), + NChanges = length(New) + length(Changed) + length(Deleted), + if NewVersion =:= OldVersion andalso NChanges > 0 -> + set_invalid(), + log("ERROR: Application '~p' contains changes, but its version is not updated", [App]); + true -> + ok + end, + {New, Changed, Deleted}. + +-spec hashsums(file:filename()) -> #{module() => binary()}. +hashsums(EbinDir) -> + maps:from_list(lists:map( + fun(Beam) -> + File = filename:join(EbinDir, Beam), + {ok, Ret = {_Module, _MD5}} = beam_lib:md5(File), + Ret + end, + filelib:wildcard("*.beam", EbinDir) + )). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Global state +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +init_globals(Options) -> + ets:new(globals, [named_table, set, public]), + ets:insert(globals, {valid, true}), + ets:insert(globals, {options, Options}). + +getopt(Option) -> + maps:get(Option, ets:lookup_element(globals, options, 2)). %% Set a global flag that something about the appfiles is invalid set_invalid() -> - put(update_appup_invalid, false). + ets:insert(globals, {valid, false}). is_valid() -> - get(update_appup_invalid). + ets:lookup_element(globals, valid, 2). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utility functions @@ -298,7 +370,8 @@ semver(Maj, Min, Patch) -> %% Locate a file in a specified application locate(App, Suffix) -> AppStr = atom_to_list(App), - case filelib:wildcard("{src,apps,lib-*}/**/" ++ AppStr ++ Suffix) of + SrcDirs = getopt(src_dirs), + case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of [File] -> {ok, File}; [] -> From 20ee42be876dc0a597c534d32a5ebf3aa7e467e0 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 6 Oct 2021 23:44:53 +0200 Subject: [PATCH 10/11] chore(update-appup): s/Pred/Prev/g --- scripts/update_appup.escript | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index de8605f5a..9572c5387 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -74,18 +74,20 @@ parse_args(["--make-command", Command|Rest], State) -> parse_args(Rest, State#{make_command => Command}); parse_args(["--release-dir", Dir|Rest], State) -> parse_args(Rest, State#{beams_dir => Dir}); +parse_args(["--src-dirs", Pattern|Rest], State) -> + parse_args(Rest, State#{src_dirs => Pattern}); parse_args(_, _) -> fail(usage()). main(Options, Baseline) -> - {CurrRelDir, PredRelDir} = prepare(Baseline, Options), + {CurrRelDir, PrevRelDir} = prepare(Baseline, Options), log("~n===================================~n" "Processing changes..." "~n===================================~n"), CurrAppsIdx = index_apps(CurrRelDir), - PredAppsIdx = index_apps(PredRelDir), - %% log("Curr: ~p~nPred: ~p~n", [CurrApps, PredApps]), - AppupChanges = find_appup_actions(CurrAppsIdx, PredAppsIdx), + PrevAppsIdx = index_apps(PrevRelDir), + %% log("Curr: ~p~nPrev: ~p~n", [CurrApps, PrevApps]), + AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx), case getopt(check) of true -> case AppupChanges of @@ -118,8 +120,8 @@ prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir} log("Building the current version...~n"), bash(MakeCommand), log("Downloading and building the previous release...~n"), - {ok, PredRootDir} = build_pred_release(Baseline, Options), - {BeamDir, filename:join(PredRootDir, BeamDir)}. + {ok, PrevRootDir} = build_pred_release(Baseline, Options), + {BeamDir, filename:join(PrevRootDir, BeamDir)}. build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> BaseDir = "/tmp/emqx-baseline/", @@ -152,11 +154,11 @@ find_pred_tag(CurrentRelease) -> %% Appup action creation and updating %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -find_appup_actions(CurrApps, PredApps) -> +find_appup_actions(CurrApps, PrevApps) -> maps:fold( fun(App, CurrAppIdx, Acc) -> - case PredApps of - #{App := PredAppIdx} -> find_appup_actions(App, CurrAppIdx, PredAppIdx) ++ Acc; + case PrevApps of + #{App := PrevAppIdx} -> find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc; _ -> Acc %% New app, nothing to upgrade here. end end, @@ -166,10 +168,10 @@ find_appup_actions(CurrApps, PredApps) -> find_appup_actions(_App, AppIdx, AppIdx) -> %% No changes to the app, ignore: []; -find_appup_actions(App, CurrAppIdx, PredAppIdx = #app{version = PredVersion}) -> - {OldUpgrade, OldDowngrade} = find_old_appup_actions(App, PredVersion), - Upgrade = merge_update_actions(diff_app(App, CurrAppIdx, PredAppIdx), OldUpgrade), - Downgrade = merge_update_actions(diff_app(App, PredAppIdx, CurrAppIdx), OldDowngrade), +find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) -> + {OldUpgrade, OldDowngrade} = find_old_appup_actions(App, PrevVersion), + Upgrade = merge_update_actions(diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade), + Downgrade = merge_update_actions(diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade), if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade -> %% The appup file has been already updated: []; @@ -177,7 +179,7 @@ find_appup_actions(App, CurrAppIdx, PredAppIdx = #app{version = PredVersion}) -> [{App, {Upgrade, Downgrade}}] end. -find_old_appup_actions(App, PredVersion) -> +find_old_appup_actions(App, PrevVersion) -> {Upgrade0, Downgrade0} = case locate(App, ".appup.src") of {ok, AppupFile} -> @@ -186,7 +188,7 @@ find_old_appup_actions(App, PredVersion) -> undefined -> {[], []} end, - {ensure_version(PredVersion, Upgrade0), ensure_version(PredVersion, Downgrade0)}. + {ensure_version(PrevVersion, Upgrade0), ensure_version(PrevVersion, Downgrade0)}. merge_update_actions(Changes, Vsns) -> lists:map(fun(Ret = {<<".*">>, _}) -> From 668bcad4e0fcfeca0975a904650e30131e731b57 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Thu, 7 Oct 2021 09:59:32 +0200 Subject: [PATCH 11/11] fix(update_appup): Add forgotten CLI parameter --- scripts/update_appup.escript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 9572c5387..61e6ae717 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -76,6 +76,8 @@ parse_args(["--release-dir", Dir|Rest], State) -> parse_args(Rest, State#{beams_dir => Dir}); parse_args(["--src-dirs", Pattern|Rest], State) -> parse_args(Rest, State#{src_dirs => Pattern}); +parse_args(["--prev-tag", Tag|Rest], State) -> + parse_args(Rest, State#{prev_tag => Tag}); parse_args(_, _) -> fail(usage()).