yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5923 from k32/update-appup-binary 

feat(update_appup): Support binary releases (.zip)
This commit is contained in:
k32 2021-10-14 17:14:11 +02:00 committed by GitHub
parent 4b098ce3af 14aaa4affe
commit 44d666f62b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 64 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
scripts/update_appup.escript
View File

@ -25,14 +25,16 @@ Usage:
Options: Options:
--check Don't update the appfile, just check that they are complete --check Don't update the appfile, just check that they are complete
--prev-tag Specify the previous release tag. Otherwise the previous patch version is used --prev-tag Specify the previous release tag. Otherwise the previous patch version is used
--repo Upsteam git repo URL --repo Upsteam git repo URL
--remote Get upstream repo URL from the specified git remote --remote Get upstream repo URL from the specified git remote
--skip-build Don't rebuild the releases. May produce wrong results --skip-build Don't rebuild the releases. May produce wrong results
--make-command A command used to assemble the release --make-command A command used to assemble the release
--release-dir Release directory --release-dir Release directory
--src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
--binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag.
E.g. \"https://github.com/emqx/emqx/releases/download/v4.3.8/emqx-centos7-%TAG%-amd64.zip\"
". ".
-record(app, -record(app,
@ -41,12 +43,13 @@ Options:
}). }).
default_options() -> default_options() ->
#{ clone_url => find_upstream_repo("origin") #{ clone_url => find_upstream_repo("origin")
, make_command => "make emqx-rel" , make_command => "make emqx-rel"
, beams_dir => "_build/emqx/rel/emqx/lib/" , beams_dir => "_build/emqx/rel/emqx/lib/"
, check => false , check => false
, prev_tag => undefined , prev_tag => undefined
, src_dirs => "{src,apps,lib-*}/**/" , src_dirs => "{src,apps,lib-*}/**/"
, binary_rel_url => undefined
}. }.
%% App-specific actions that should be added unconditionally to any update/downgrade: %% App-specific actions that should be added unconditionally to any update/downgrade:
@ -54,12 +57,12 @@ app_specific_actions(_) ->
[]. [].
ignored_apps() -> ignored_apps() ->
[emqx_dashboard, emqx_management]. [emqx_dashboard, emqx_management] ++ otp_standard_apps().
main(Args) -> main(Args) ->
#{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()),
init_globals(Options), init_globals(Options),
case find_pred_tag(CurrentRelease) of case find_prev_tag(CurrentRelease) of
{ok, Baseline} -> {ok, Baseline} ->
main(Options, Baseline); main(Options, Baseline);
undefined -> undefined ->
@ -85,6 +88,8 @@ parse_args(["--src-dirs", Pattern|Rest], State) ->
parse_args(Rest, State#{src_dirs => Pattern}); parse_args(Rest, State#{src_dirs => Pattern});
parse_args(["--prev-tag", Tag|Rest], State) -> parse_args(["--prev-tag", Tag|Rest], State) ->
parse_args(Rest, State#{prev_tag => Tag}); parse_args(Rest, State#{prev_tag => Tag});
parse_args(["--binary-rel-url", URL|Rest], State) ->
parse_args(Rest, State#{binary_rel_url => {ok, URL}});
parse_args(_, _) -> parse_args(_, _) ->
fail(usage()). fail(usage()).
@ -95,7 +100,7 @@ main(Options, Baseline) ->
"~n===================================~n"), "~n===================================~n"),
CurrAppsIdx = index_apps(CurrRelDir), CurrAppsIdx = index_apps(CurrRelDir),
PrevAppsIdx = index_apps(PrevRelDir), PrevAppsIdx = index_apps(PrevRelDir),
%% log("Curr: ~p~nPrev: ~p~n", [CurrApps, PrevApps]), %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]),
AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx), AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx),
case getopt(check) of case getopt(check) of
true -> true ->
@ -122,17 +127,25 @@ warn_and_exit(false) ->
log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"), log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"),
halt(1). halt(1).
prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir, binary_rel_url := BinRel}) ->
log("~n===================================~n" log("~n===================================~n"
"Baseline: ~s" "Baseline: ~s"
"~n===================================~n", [Baseline]), "~n===================================~n", [Baseline]),
log("Building the current version...~n"), log("Building the current version...~n"),
bash(MakeCommand), bash(MakeCommand),
log("Downloading and building the previous release...~n"), log("Downloading and building the previous release...~n"),
{ok, PrevRootDir} = build_pred_release(Baseline, Options), PrevRelDir =
{BeamDir, filename:join(PrevRootDir, BeamDir)}. case BinRel of
undefined ->
{ok, PrevRootDir} = build_prev_release(Baseline, Options),
filename:join(PrevRootDir, BeamDir);
{ok, URL} ->
{ok, PrevRootDir} = download_prev_release(Baseline, Options),
PrevRootDir
end,
{BeamDir, PrevRelDir}.
build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) ->
BaseDir = "/tmp/emqx-baseline/", BaseDir = "/tmp/emqx-baseline/",
Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
%% TODO: shallow clone %% TODO: shallow clone
@ -144,10 +157,22 @@ build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand})
bash(Script, Env), bash(Script, Env),
{ok, filename:join(BaseDir, Dir)}. {ok, filename:join(BaseDir, Dir)}.
download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) ->
URL = string:replace(URL0, "%TAG%", Tag, all),
BaseDir = "/tmp/emqx-baseline-bin/",
Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
Filename = filename:join(BaseDir, Dir),
Script = "mkdir -p ${OUTFILE} &&
{ [ -f ${OUTFILE}.zip ] || wget -O ${OUTFILE}.zip ${URL}; } &&
unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
bash(Script, Env),
{ok, Filename}.
find_upstream_repo(Remote) -> find_upstream_repo(Remote) ->
string:trim(os:cmd("git remote get-url " ++ Remote)). string:trim(os:cmd("git remote get-url " ++ Remote)).
find_pred_tag(CurrentRelease) -> find_prev_tag(CurrentRelease) ->
case getopt(prev_tag) of case getopt(prev_tag) of
undefined -> undefined ->
{Maj, Min, Patch} = parse_semver(CurrentRelease), {Maj, Min, Patch} = parse_semver(CurrentRelease),
@ -190,7 +215,7 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
find_old_appup_actions(App, PrevVersion) -> find_old_appup_actions(App, PrevVersion) ->
{Upgrade0, Downgrade0} = {Upgrade0, Downgrade0} =
case locate(App, ".appup.src") of case locate(ebin_current, App, ".appup") of
{ok, AppupFile} -> {ok, AppupFile} ->
{_, U, D} = read_appup(AppupFile), {_, U, D} = read_appup(AppupFile),
{U, D}; {U, D};
@ -261,7 +286,7 @@ update_appups(Changes) ->
Changes). Changes).
do_update_appup(App, Upgrade, Downgrade) -> do_update_appup(App, Upgrade, Downgrade) ->
case locate(App, ".appup.src") of case locate(src, App, ".appup.src") of
{ok, AppupFile} -> {ok, AppupFile} ->
render_appfile(AppupFile, Upgrade, Downgrade); render_appfile(AppupFile, Upgrade, Downgrade);
undefined -> undefined ->
@ -270,7 +295,7 @@ do_update_appup(App, Upgrade, Downgrade) ->
render_appfile(AppupFile, Upgrade, Downgrade); render_appfile(AppupFile, Upgrade, Downgrade);
false -> false ->
set_invalid(), set_invalid(),
log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p", [App, Upgrade]) log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p~n", [App, Upgrade])
end end
end. end.
@ -283,7 +308,7 @@ render_appfile(File, Upgrade, Downgrade) ->
ok = file:write_file(File, IOList). ok = file:write_file(File, IOList).
create_stub(App) -> create_stub(App) ->
case locate(App, ".app.src") of case locate(src, App, ".app.src") of
{ok, AppSrc} -> {ok, AppSrc} ->
AppupFile = filename:basename(AppSrc) ++ ".appup.src", AppupFile = filename:basename(AppSrc) ++ ".appup.src",
Default = {<<".*">>, []}, Default = {<<".*">>, []},
@ -386,7 +411,16 @@ semver(Maj, Min, Patch) ->
lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])). lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])).
%% Locate a file in a specified application %% Locate a file in a specified application
locate(App, Suffix) -> locate(ebin_current, App, Suffix) ->
ReleaseDir = getopt(beams_dir),
AppStr = atom_to_list(App),
case filelib:wildcard(ReleaseDir ++ "/**/ebin/" ++ AppStr ++ Suffix) of
[File] ->
{ok, File};
[] ->
undefined
end;
locate(src, App, Suffix) ->
AppStr = atom_to_list(App), AppStr = atom_to_list(App),
SrcDirs = getopt(src_dirs), SrcDirs = getopt(src_dirs),
case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of
@ -444,3 +478,6 @@ ensure_string(Str) when is_binary(Str) ->
binary_to_list(Str); binary_to_list(Str);
ensure_string(Str) when is_list(Str) -> ensure_string(Str) when is_list(Str) ->
Str. Str.
otp_standard_apps() ->
[ssl, mnesia, kernel, asn1, stdlib].