feat(update_appup): Compare beam files
This commit is contained in:
k32
parent
c8dda45c55
commit
4020db8fc1
|
|
@ -1,75 +1,215 @@
|
||||||
#!/usr/bin/env -S escript -c
|
#!/usr/bin/env -S escript -c
|
||||||
|
%% -*- erlang-indent-level:4 -*-
|
||||||
%% A script that adds changed modules to the corresponding appup files
|
%% A script that adds changed modules to the corresponding appup files
|
||||||
|
|
||||||
main(_Args) ->
|
main(Args) ->
|
||||||
ChangedFiles = string:lexemes(os:cmd("git diff --name-only origin/master..HEAD"), "\n"),
|
#{check := Check, current_release := CurrentRelease, prepare := Prepare} =
|
||||||
AppModules0 = lists:filtermap(fun filter_erlang_modules/1, ChangedFiles),
|
parse_args(Args, #{check => false, prepare => true}),
|
||||||
%% emqx_app must always be included as we bump version number in emqx_release.hrl for each release
|
case find_pred_tag(CurrentRelease) of
|
||||||
AppModules1 = [{emqx, emqx_app} | AppModules0],
|
{ok, Baseline} ->
|
||||||
AppModules = group_modules(AppModules1),
|
{CurrDir, PredDir} = prepare(Baseline, Prepare),
|
||||||
io:format("Changed modules: ~p~n", [AppModules]),
|
Changed = diff_releases(CurrDir, PredDir),
|
||||||
_ = maps:map(fun process_app/2, AppModules),
|
_ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed),
|
||||||
ok.
|
ok;
|
||||||
|
undefined ->
|
||||||
|
io:format(standard_error, "No appup update is needed for this release, done~n", []),
|
||||||
|
ok
|
||||||
|
end.
|
||||||
|
|
||||||
process_app(App, Modules) ->
|
parse_args([CurrentRelease = [A|_]], State) when A =/= $- ->
|
||||||
|
State#{current_release => CurrentRelease};
|
||||||
|
parse_args(["--check"|Rest], State) ->
|
||||||
|
parse_args(Rest, State#{check => true});
|
||||||
|
parse_args(["--skip-build"|Rest], State) ->
|
||||||
|
parse_args(Rest, State#{prepare => false});
|
||||||
|
parse_args([], _) ->
|
||||||
|
fail("Usage:~n update_appup.escript [--check] [--skip-build] <current_release_tag>
|
||||||
|
|
||||||
|
--check Don't update the appup files, just check that they are complete
|
||||||
|
--skip-build Don't rebuild the releases. May produce wrong appup files.
|
||||||
|
").
|
||||||
|
|
||||||
|
process_app(PredVersion, _Check, App, Changes) ->
|
||||||
AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])),
|
AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])),
|
||||||
case AppupFiles of
|
case AppupFiles of
|
||||||
[AppupFile] ->
|
[AppupFile] ->
|
||||||
update_appup(AppupFile, Modules);
|
update_appup(PredVersion, AppupFile, Changes);
|
||||||
[] ->
|
[] ->
|
||||||
io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App])
|
io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App])
|
||||||
end.
|
end.
|
||||||
|
|
||||||
filter_erlang_modules(Filename) ->
|
|
||||||
case lists:reverse(filename:split(Filename)) of
|
|
||||||
[Module, "src"] ->
|
|
||||||
erl_basename("emqx", Module);
|
|
||||||
[Module, "src", App|_] ->
|
|
||||||
erl_basename(App, Module);
|
|
||||||
[Module, _, "src", App|_] ->
|
|
||||||
erl_basename(App, Module);
|
|
||||||
_ ->
|
|
||||||
false
|
|
||||||
end.
|
|
||||||
|
|
||||||
erl_basename(App, Name) ->
|
|
||||||
case filename:basename(Name, ".erl") of
|
|
||||||
Name -> false;
|
|
||||||
Module -> {true, {list_to_atom(App), list_to_atom(Module)}}
|
|
||||||
end.
|
|
||||||
|
|
||||||
group_modules(L) ->
|
group_modules(L) ->
|
||||||
lists:foldl(fun({App, Mod}, Acc) ->
|
lists:foldl(fun({App, Mod}, Acc) ->
|
||||||
maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc)
|
maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc)
|
||||||
end, #{}, L).
|
end, #{}, L).
|
||||||
|
|
||||||
update_appup(File, Modules) ->
|
update_appup(_, File, {[], [], []}) ->
|
||||||
|
%% No changes in the app. Just check syntax of the existing appup:
|
||||||
|
_ = read_appup(File);
|
||||||
|
update_appup(PredVersion, File, Changes) ->
|
||||||
io:format("~nUpdating appup: ~p~n", [File]),
|
io:format("~nUpdating appup: ~p~n", [File]),
|
||||||
{_, Upgrade0, Downgrade0} = read_appup(File),
|
{_, Upgrade0, Downgrade0} = read_appup(File),
|
||||||
Upgrade = update_actions(Modules, Upgrade0),
|
Upgrade = update_actions(PredVersion, Changes, Upgrade0),
|
||||||
Downgrade = update_actions(Modules, Downgrade0),
|
Downgrade = update_actions(PredVersion, Changes, Downgrade0),
|
||||||
IOList = io_lib:format("%% -*- mode: erlang -*-
|
IOList = io_lib:format("%% -*- mode: erlang -*-
|
||||||
{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]),
|
{VSN,~n ~p,~n ~p}.~n", [Upgrade, Downgrade]),
|
||||||
ok = file:write_file(File, IOList).
|
ok = file:write_file(File, IOList),
|
||||||
|
%% Check appup syntax:
|
||||||
|
_ = read_appup(File).
|
||||||
|
|
||||||
update_actions(Modules, Versions) ->
|
update_actions(PredVersion, Changes, Versions) ->
|
||||||
lists:map(fun(L) -> do_update_actions(Modules, L) end, Versions).
|
lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)).
|
||||||
|
|
||||||
do_update_actions(_, Ret = {<<".*">>, _}) ->
|
do_update_actions(_, Ret = {<<".*">>, _}) ->
|
||||||
Ret;
|
Ret;
|
||||||
do_update_actions(Modules, {Vsn, Actions}) ->
|
do_update_actions(Changes, {Vsn, Actions}) ->
|
||||||
{Vsn, add_modules(Modules, Actions)}.
|
{Vsn, process_changes(Changes, Actions)}.
|
||||||
|
|
||||||
add_modules(NewModules, OldActions) ->
|
process_changes({New0, Changed0, Deleted0}, OldActions) ->
|
||||||
OldModules = lists:map(fun(It) -> element(2, It) end, OldActions),
|
AlreadyHandled = lists:map(fun(It) -> element(2, It) end, OldActions),
|
||||||
Modules = NewModules -- OldModules,
|
New = New0 -- AlreadyHandled,
|
||||||
OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Modules].
|
Changed = Changed0 -- AlreadyHandled,
|
||||||
|
Deleted = Deleted0 -- AlreadyHandled,
|
||||||
|
OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New]
|
||||||
|
++ [{delete_module, M} || M <- Deleted].
|
||||||
|
|
||||||
|
ensure_pred_version(PredVersion, Versions) ->
|
||||||
|
case lists:keyfind(PredVersion, 1, Versions) of
|
||||||
|
false ->
|
||||||
|
[{PredVersion, []}|Versions];
|
||||||
|
_ ->
|
||||||
|
Versions
|
||||||
|
end.
|
||||||
|
|
||||||
read_appup(File) ->
|
read_appup(File) ->
|
||||||
{ok, Bin0} = file:read_file(File),
|
case file:script(File, [{'VSN', "VSN"}]) of
|
||||||
%% Hack:
|
{ok, Terms} ->
|
||||||
Bin1 = re:replace(Bin0, "VSN", "\"VSN\""),
|
Terms;
|
||||||
TmpFile = filename:join("/tmp", filename:basename(File)),
|
Error ->
|
||||||
ok = file:write_file(TmpFile, Bin1),
|
fail("Failed to parse appup file ~s: ~p", [File, Error])
|
||||||
{ok, [Terms]} = file:consult(TmpFile),
|
end.
|
||||||
Terms.
|
|
||||||
|
diff_releases(CurrDir, OldDir) ->
|
||||||
|
Curr = hashsums(find_beams(CurrDir)),
|
||||||
|
Old = hashsums(find_beams(OldDir)),
|
||||||
|
Fun = fun(App, Modules, Acc) ->
|
||||||
|
OldModules = maps:get(App, Old, #{}),
|
||||||
|
Acc#{App => diff_app_modules(Modules, OldModules)}
|
||||||
|
end,
|
||||||
|
maps:fold(Fun, #{}, Curr).
|
||||||
|
|
||||||
|
diff_app_modules(Modules, OldModules) ->
|
||||||
|
{New, Changed} =
|
||||||
|
maps:fold( fun(Mod, MD5, {New, Changed}) ->
|
||||||
|
case OldModules of
|
||||||
|
#{Mod := OldMD5} when MD5 =:= OldMD5 ->
|
||||||
|
{New, Changed};
|
||||||
|
#{Mod := _} ->
|
||||||
|
{[Mod|New], Changed};
|
||||||
|
_ -> {New, [Mod|Changed]}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
, {[], []}
|
||||||
|
, Modules
|
||||||
|
),
|
||||||
|
Deleted = maps:keys(maps:without(maps:keys(Modules), OldModules)),
|
||||||
|
{New, Changed, Deleted}.
|
||||||
|
|
||||||
|
find_beams(Dir) ->
|
||||||
|
[filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)].
|
||||||
|
|
||||||
|
prepare(Baseline, Prepare) ->
|
||||||
|
io:format("~n===================================~n"
|
||||||
|
"Baseline: ~s"
|
||||||
|
"~n===================================~n", [Baseline]),
|
||||||
|
io:format("Building the current version...~n"),
|
||||||
|
Prepare andalso success(cmd("make", #{args => ["emqx-rel"]}), "Failed to build HEAD"),
|
||||||
|
io:format("Downloading the preceding release...~n"),
|
||||||
|
{ok, PredRootDir} = build_pred_release(Baseline, Prepare),
|
||||||
|
BeamDir = "_build/emqx/rel/emqx/lib/",
|
||||||
|
{BeamDir, filename:join(PredRootDir, BeamDir)}.
|
||||||
|
|
||||||
|
build_pred_release(Baseline, Prepare) ->
|
||||||
|
Repo = find_upstream_repo(),
|
||||||
|
BaseDir = "/tmp/emqx-baseline/",
|
||||||
|
Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
|
||||||
|
%% TODO: shallow clone
|
||||||
|
Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && cd ${DIR} && make emqx-rel",
|
||||||
|
Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}],
|
||||||
|
Prepare andalso
|
||||||
|
success( cmd("bash", #{ args => ["-c", Script]
|
||||||
|
, env => Env
|
||||||
|
})
|
||||||
|
, "Failed to build the baseline release"
|
||||||
|
),
|
||||||
|
{ok, filename:join(BaseDir, Dir)}.
|
||||||
|
|
||||||
|
%% @doc Find whether we are in emqx or emqx-ee
|
||||||
|
find_upstream_repo() ->
|
||||||
|
Str = os:cmd("git remote get-url origin"),
|
||||||
|
case re:run(Str, "/([^/]+).git$", [{capture, all_but_first, list}]) of
|
||||||
|
{match, ["emqx"]} -> "git@github.com:emqx/emqx.git";
|
||||||
|
{match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git";
|
||||||
|
Ret -> fail("Cannot detect the correct upstream repo: ~p", [Ret])
|
||||||
|
end.
|
||||||
|
|
||||||
|
find_pred_tag(CurrentRelease) ->
|
||||||
|
case re:run(CurrentRelease, "^([0-9]+)\.([0-9]+)\.([0-9]+)$", [{capture, all_but_first, list}]) of
|
||||||
|
{match, [Maj, Min, Patch]} ->
|
||||||
|
case list_to_integer(Patch) of
|
||||||
|
0 -> undefined;
|
||||||
|
P -> {ok, lists:flatten(io_lib:format("~s.~s.~p", [Maj, Min, P - 1]))}
|
||||||
|
end;
|
||||||
|
Err ->
|
||||||
|
fail("The current release tag doesn't follow semver pattern: ~p", [Err])
|
||||||
|
end.
|
||||||
|
|
||||||
|
-spec hashsums(file:filename()) -> #{App => #{module() => binary()}}
|
||||||
|
when App :: atom().
|
||||||
|
hashsums(Files) ->
|
||||||
|
hashsums(Files, #{}).
|
||||||
|
|
||||||
|
hashsums([], Acc) ->
|
||||||
|
Acc;
|
||||||
|
hashsums([File|Rest], Acc0) ->
|
||||||
|
[_, "ebin", Dir|_] = lists:reverse(filename:split(File)),
|
||||||
|
{match, [AppStr]} = re:run(Dir, "^(.*)-[^-]+$", [{capture, all_but_first, list}]),
|
||||||
|
App = list_to_atom(AppStr),
|
||||||
|
{ok, {Module, MD5}} = beam_lib:md5(File),
|
||||||
|
Acc = maps:update_with( App
|
||||||
|
, fun(Old) -> Old #{Module => MD5} end
|
||||||
|
, #{Module => MD5}
|
||||||
|
, Acc0
|
||||||
|
),
|
||||||
|
hashsums(Rest, Acc).
|
||||||
|
|
||||||
|
%% Spawn an executable and return the exit status
|
||||||
|
cmd(Exec, Params) ->
|
||||||
|
case os:find_executable(Exec) of
|
||||||
|
false ->
|
||||||
|
fail("Executable not found in $PATH: ~s", [Exec]);
|
||||||
|
Path ->
|
||||||
|
Params1 = maps:to_list(maps:with([env, args, cd], Params)),
|
||||||
|
Port = erlang:open_port( {spawn_executable, Path}
|
||||||
|
, [ exit_status
|
||||||
|
, nouse_stdio
|
||||||
|
| Params1
|
||||||
|
]
|
||||||
|
),
|
||||||
|
receive
|
||||||
|
{Port, {exit_status, Status}} ->
|
||||||
|
Status
|
||||||
|
end
|
||||||
|
end.
|
||||||
|
|
||||||
|
success(0, _) ->
|
||||||
|
true;
|
||||||
|
success(_, Msg) ->
|
||||||
|
fail(Msg).
|
||||||
|
|
||||||
|
fail(Str) ->
|
||||||
|
fail(Str, []).
|
||||||
|
|
||||||
|
fail(Str, Args) ->
|
||||||
|
io:format(standard_error, Str ++ "~n", Args),
|
||||||
|
halt(1).
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue