From 4020db8fc1276718f414c7d41529cda2e6ac855d Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:06:16 +0200 Subject: [PATCH] feat(update_appup): Compare beam files --- scripts/update_appup.escript | 240 +++++++++++++++++++++++++++-------- 1 file changed, 190 insertions(+), 50 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 6a78012d2..436dff3a1 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,41 +1,41 @@ #!/usr/bin/env -S escript -c +%% -*- erlang-indent-level:4 -*- %% A script that adds changed modules to the corresponding appup files -main(_Args) -> - ChangedFiles = string:lexemes(os:cmd("git diff --name-only origin/master..HEAD"), "\n"), - AppModules0 = lists:filtermap(fun filter_erlang_modules/1, ChangedFiles), - %% emqx_app must always be included as we bump version number in emqx_release.hrl for each release - AppModules1 = [{emqx, emqx_app} | AppModules0], - AppModules = group_modules(AppModules1), - io:format("Changed modules: ~p~n", [AppModules]), - _ = maps:map(fun process_app/2, AppModules), - ok. +main(Args) -> + #{check := Check, current_release := CurrentRelease, prepare := Prepare} = + parse_args(Args, #{check => false, prepare => true}), + case find_pred_tag(CurrentRelease) of + {ok, Baseline} -> + {CurrDir, PredDir} = prepare(Baseline, Prepare), + Changed = diff_releases(CurrDir, PredDir), + _ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed), + ok; + undefined -> + io:format(standard_error, "No appup update is needed for this release, done~n", []), + ok + end. -process_app(App, Modules) -> +parse_args([CurrentRelease = [A|_]], State) when A =/= $- -> + State#{current_release => CurrentRelease}; +parse_args(["--check"|Rest], State) -> + parse_args(Rest, State#{check => true}); +parse_args(["--skip-build"|Rest], State) -> + parse_args(Rest, State#{prepare => false}); +parse_args([], _) -> + fail("Usage:~n update_appup.escript [--check] [--skip-build] + + --check Don't update the appup files, just check that they are complete + --skip-build Don't rebuild the releases. May produce wrong appup files. +"). + +process_app(PredVersion, _Check, App, Changes) -> AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])), case AppupFiles of [AppupFile] -> - update_appup(AppupFile, Modules); - [] -> - io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App]) - end. - -filter_erlang_modules(Filename) -> - case lists:reverse(filename:split(Filename)) of - [Module, "src"] -> - erl_basename("emqx", Module); - [Module, "src", App|_] -> - erl_basename(App, Module); - [Module, _, "src", App|_] -> - erl_basename(App, Module); - _ -> - false - end. - -erl_basename(App, Name) -> - case filename:basename(Name, ".erl") of - Name -> false; - Module -> {true, {list_to_atom(App), list_to_atom(Module)}} + update_appup(PredVersion, AppupFile, Changes); + [] -> + io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App]) end. group_modules(L) -> @@ -43,33 +43,173 @@ group_modules(L) -> maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc) end, #{}, L). -update_appup(File, Modules) -> +update_appup(_, File, {[], [], []}) -> + %% No changes in the app. Just check syntax of the existing appup: + _ = read_appup(File); +update_appup(PredVersion, File, Changes) -> io:format("~nUpdating appup: ~p~n", [File]), {_, Upgrade0, Downgrade0} = read_appup(File), - Upgrade = update_actions(Modules, Upgrade0), - Downgrade = update_actions(Modules, Downgrade0), + Upgrade = update_actions(PredVersion, Changes, Upgrade0), + Downgrade = update_actions(PredVersion, Changes, Downgrade0), IOList = io_lib:format("%% -*- mode: erlang -*- {VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]), - ok = file:write_file(File, IOList). + ok = file:write_file(File, IOList), + %% Check appup syntax: + _ = read_appup(File). -update_actions(Modules, Versions) -> - lists:map(fun(L) -> do_update_actions(Modules, L) end, Versions). +update_actions(PredVersion, Changes, Versions) -> + lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)). do_update_actions(_, Ret = {<<".*">>, _}) -> Ret; -do_update_actions(Modules, {Vsn, Actions}) -> - {Vsn, add_modules(Modules, Actions)}. +do_update_actions(Changes, {Vsn, Actions}) -> + {Vsn, process_changes(Changes, Actions)}. -add_modules(NewModules, OldActions) -> - OldModules = lists:map(fun(It) -> element(2, It) end, OldActions), - Modules = NewModules -- OldModules, - OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Modules]. +process_changes({New0, Changed0, Deleted0}, OldActions) -> + AlreadyHandled = lists:map(fun(It) -> element(2, It) end, OldActions), + New = New0 -- AlreadyHandled, + Changed = Changed0 -- AlreadyHandled, + Deleted = Deleted0 -- AlreadyHandled, + OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New] + ++ [{delete_module, M} || M <- Deleted]. + +ensure_pred_version(PredVersion, Versions) -> + case lists:keyfind(PredVersion, 1, Versions) of + false -> + [{PredVersion, []}|Versions]; + _ -> + Versions + end. read_appup(File) -> - {ok, Bin0} = file:read_file(File), - %% Hack: - Bin1 = re:replace(Bin0, "VSN", "\"VSN\""), - TmpFile = filename:join("/tmp", filename:basename(File)), - ok = file:write_file(TmpFile, Bin1), - {ok, [Terms]} = file:consult(TmpFile), - Terms. + case file:script(File, [{'VSN', "VSN"}]) of + {ok, Terms} -> + Terms; + Error -> + fail("Failed to parse appup file ~s: ~p", [File, Error]) + end. + +diff_releases(CurrDir, OldDir) -> + Curr = hashsums(find_beams(CurrDir)), + Old = hashsums(find_beams(OldDir)), + Fun = fun(App, Modules, Acc) -> + OldModules = maps:get(App, Old, #{}), + Acc#{App => diff_app_modules(Modules, OldModules)} + end, + maps:fold(Fun, #{}, Curr). + +diff_app_modules(Modules, OldModules) -> + {New, Changed} = + maps:fold( fun(Mod, MD5, {New, Changed}) -> + case OldModules of + #{Mod := OldMD5} when MD5 =:= OldMD5 -> + {New, Changed}; + #{Mod := _} -> + {[Mod|New], Changed}; + _ -> {New, [Mod|Changed]} + end + end + , {[], []} + , Modules + ), + Deleted = maps:keys(maps:without(maps:keys(Modules), OldModules)), + {New, Changed, Deleted}. + +find_beams(Dir) -> + [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)]. + +prepare(Baseline, Prepare) -> + io:format("~n===================================~n" + "Baseline: ~s" + "~n===================================~n", [Baseline]), + io:format("Building the current version...~n"), + Prepare andalso success(cmd("make", #{args => ["emqx-rel"]}), "Failed to build HEAD"), + io:format("Downloading the preceding release...~n"), + {ok, PredRootDir} = build_pred_release(Baseline, Prepare), + BeamDir = "_build/emqx/rel/emqx/lib/", + {BeamDir, filename:join(PredRootDir, BeamDir)}. + +build_pred_release(Baseline, Prepare) -> + Repo = find_upstream_repo(), + BaseDir = "/tmp/emqx-baseline/", + Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], + %% TODO: shallow clone + Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && cd ${DIR} && make emqx-rel", + Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}], + Prepare andalso + success( cmd("bash", #{ args => ["-c", Script] + , env => Env + }) + , "Failed to build the baseline release" + ), + {ok, filename:join(BaseDir, Dir)}. + +%% @doc Find whether we are in emqx or emqx-ee +find_upstream_repo() -> + Str = os:cmd("git remote get-url origin"), + case re:run(Str, "/([^/]+).git$", [{capture, all_but_first, list}]) of + {match, ["emqx"]} -> "git@github.com:emqx/emqx.git"; + {match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git"; + Ret -> fail("Cannot detect the correct upstream repo: ~p", [Ret]) + end. + +find_pred_tag(CurrentRelease) -> + case re:run(CurrentRelease, "^([0-9]+)\.([0-9]+)\.([0-9]+)$", [{capture, all_but_first, list}]) of + {match, [Maj, Min, Patch]} -> + case list_to_integer(Patch) of + 0 -> undefined; + P -> {ok, lists:flatten(io_lib:format("~s.~s.~p", [Maj, Min, P - 1]))} + end; + Err -> + fail("The current release tag doesn't follow semver pattern: ~p", [Err]) + end. + +-spec hashsums(file:filename()) -> #{App => #{module() => binary()}} + when App :: atom(). +hashsums(Files) -> + hashsums(Files, #{}). + +hashsums([], Acc) -> + Acc; +hashsums([File|Rest], Acc0) -> + [_, "ebin", Dir|_] = lists:reverse(filename:split(File)), + {match, [AppStr]} = re:run(Dir, "^(.*)-[^-]+$", [{capture, all_but_first, list}]), + App = list_to_atom(AppStr), + {ok, {Module, MD5}} = beam_lib:md5(File), + Acc = maps:update_with( App + , fun(Old) -> Old #{Module => MD5} end + , #{Module => MD5} + , Acc0 + ), + hashsums(Rest, Acc). + +%% Spawn an executable and return the exit status +cmd(Exec, Params) -> + case os:find_executable(Exec) of + false -> + fail("Executable not found in $PATH: ~s", [Exec]); + Path -> + Params1 = maps:to_list(maps:with([env, args, cd], Params)), + Port = erlang:open_port( {spawn_executable, Path} + , [ exit_status + , nouse_stdio + | Params1 + ] + ), + receive + {Port, {exit_status, Status}} -> + Status + end + end. + +success(0, _) -> + true; +success(_, Msg) -> + fail(Msg). + +fail(Str) -> + fail(Str, []). + +fail(Str, Args) -> + io:format(standard_error, Str ++ "~n", Args), + halt(1).