From 703f52cec717eab65ed6e02753e30819a5dfa7b2 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:17:22 +0200 Subject: [PATCH 1/3] feat(update_appup): Support binary releases (.zip) --- scripts/update_appup.escript | 72 +++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 653bcf949..c2641f001 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -25,14 +25,16 @@ Usage: Options: - --check Don't update the appfile, just check that they are complete - --prev-tag Specify the previous release tag. Otherwise the previous patch version is used - --repo Upsteam git repo URL - --remote Get upstream repo URL from the specified git remote - --skip-build Don't rebuild the releases. May produce wrong results - --make-command A command used to assemble the release - --release-dir Release directory - --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' + --check Don't update the appfile, just check that they are complete + --prev-tag Specify the previous release tag. Otherwise the previous patch version is used + --repo Upsteam git repo URL + --remote Get upstream repo URL from the specified git remote + --skip-build Don't rebuild the releases. May produce wrong results + --make-command A command used to assemble the release + --release-dir Release directory + --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' + --binary-rel-url Binary release URL pattern. ${TAG} variable is substituted with the release tag. + E.g. \"https://github.com/emqx/emqx/releases/download/v4.3.8/emqx-centos7-${TAG}-amd64.zip\" ". -record(app, @@ -41,12 +43,13 @@ Options: }). default_options() -> - #{ clone_url => find_upstream_repo("origin") - , make_command => "make emqx-rel" - , beams_dir => "_build/emqx/rel/emqx/lib/" - , check => false - , prev_tag => undefined - , src_dirs => "{src,apps,lib-*}/**/" + #{ clone_url => find_upstream_repo("origin") + , make_command => "make emqx-rel" + , beams_dir => "_build/emqx/rel/emqx/lib/" + , check => false + , prev_tag => undefined + , src_dirs => "{src,apps,lib-*}/**/" + , binary_rel_url => undefined }. %% App-specific actions that should be added unconditionally to any update/downgrade: @@ -54,12 +57,12 @@ app_specific_actions(_) -> []. ignored_apps() -> - [emqx_dashboard, emqx_management]. + [emqx_dashboard, emqx_management] ++ otp_standard_apps(). main(Args) -> #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()), init_globals(Options), - case find_pred_tag(CurrentRelease) of + case find_prev_tag(CurrentRelease) of {ok, Baseline} -> main(Options, Baseline); undefined -> @@ -85,6 +88,8 @@ parse_args(["--src-dirs", Pattern|Rest], State) -> parse_args(Rest, State#{src_dirs => Pattern}); parse_args(["--prev-tag", Tag|Rest], State) -> parse_args(Rest, State#{prev_tag => Tag}); +parse_args(["--binary-rel-url", URL|Rest], State) -> + parse_args(Rest, State#{binary_rel_url => {ok, URL}}); parse_args(_, _) -> fail(usage()). @@ -122,17 +127,25 @@ warn_and_exit(false) -> log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"), halt(1). -prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> +prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir, binary_rel_url := BinRel}) -> log("~n===================================~n" "Baseline: ~s" "~n===================================~n", [Baseline]), log("Building the current version...~n"), bash(MakeCommand), log("Downloading and building the previous release...~n"), - {ok, PrevRootDir} = build_pred_release(Baseline, Options), - {BeamDir, filename:join(PrevRootDir, BeamDir)}. + PrevRelDir = + case BinRel of + undefined -> + {ok, PrevRootDir} = build_prev_release(Baseline, Options), + filename:join(PrevRootDir, BeamDir); + {ok, URL} -> + {ok, PrevRootDir} = download_prev_release(Baseline, Options), + PrevRootDir + end, + {BeamDir, PrevRelDir}. -build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> +build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> BaseDir = "/tmp/emqx-baseline/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], %% TODO: shallow clone @@ -144,10 +157,22 @@ build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) bash(Script, Env), {ok, filename:join(BaseDir, Dir)}. +download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -> + URL = string:replace(URL0, "${TAG}", Tag, all), + BaseDir = "/tmp/emqx-baseline-bin/", + Dir = filename:basename(Repo, ".git") ++ [$-|Tag], + Filename = filename:join(BaseDir, Dir), + Script = "mkdir -p ${OUTFILE} && + { [ -f ${OUTFILE}.zip ] || wget -O ${OUTFILE}.zip ${URL}; } && + unzip -n -d ${OUTFILE} ${OUTFILE}.zip", + Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], + bash(Script, Env), + {ok, Filename}. + find_upstream_repo(Remote) -> string:trim(os:cmd("git remote get-url " ++ Remote)). -find_pred_tag(CurrentRelease) -> +find_prev_tag(CurrentRelease) -> case getopt(prev_tag) of undefined -> {Maj, Min, Patch} = parse_semver(CurrentRelease), @@ -270,7 +295,7 @@ do_update_appup(App, Upgrade, Downgrade) -> render_appfile(AppupFile, Upgrade, Downgrade); false -> set_invalid(), - log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p", [App, Upgrade]) + log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p~n", [App, Upgrade]) end end. @@ -444,3 +469,6 @@ ensure_string(Str) when is_binary(Str) -> binary_to_list(Str); ensure_string(Str) when is_list(Str) -> Str. + +otp_standard_apps() -> + [ssl, mnesia, kernel, asn1, stdlib]. From cb5db8059b7943889ada0c31b4cd017ab92f5f1f Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Thu, 14 Oct 2021 13:47:04 +0200 Subject: [PATCH 2/3] fix(update_appup): Use a different syntax for substitution --- scripts/update_appup.escript | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index c2641f001..ffbfcb810 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -33,8 +33,8 @@ Options: --make-command A command used to assemble the release --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' - --binary-rel-url Binary release URL pattern. ${TAG} variable is substituted with the release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v4.3.8/emqx-centos7-${TAG}-amd64.zip\" + --binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. + E.g. \"https://github.com/emqx/emqx/releases/download/v4.3.8/emqx-centos7-%TAG%-amd64.zip\" ". -record(app, @@ -158,7 +158,7 @@ build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) {ok, filename:join(BaseDir, Dir)}. download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -> - URL = string:replace(URL0, "${TAG}", Tag, all), + URL = string:replace(URL0, "%TAG%", Tag, all), BaseDir = "/tmp/emqx-baseline-bin/", Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), From 14aaa4affed9ea3911f5757519f259cfa8937791 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Thu, 14 Oct 2021 14:18:09 +0200 Subject: [PATCH 3/3] fix(update_appup): Fix dependency check --- scripts/update_appup.escript | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index ffbfcb810..c947eee87 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -100,7 +100,7 @@ main(Options, Baseline) -> "~n===================================~n"), CurrAppsIdx = index_apps(CurrRelDir), PrevAppsIdx = index_apps(PrevRelDir), - %% log("Curr: ~p~nPrev: ~p~n", [CurrApps, PrevApps]), + %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]), AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx), case getopt(check) of true -> @@ -215,7 +215,7 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) -> find_old_appup_actions(App, PrevVersion) -> {Upgrade0, Downgrade0} = - case locate(App, ".appup.src") of + case locate(ebin_current, App, ".appup") of {ok, AppupFile} -> {_, U, D} = read_appup(AppupFile), {U, D}; @@ -286,7 +286,7 @@ update_appups(Changes) -> Changes). do_update_appup(App, Upgrade, Downgrade) -> - case locate(App, ".appup.src") of + case locate(src, App, ".appup.src") of {ok, AppupFile} -> render_appfile(AppupFile, Upgrade, Downgrade); undefined -> @@ -308,7 +308,7 @@ render_appfile(File, Upgrade, Downgrade) -> ok = file:write_file(File, IOList). create_stub(App) -> - case locate(App, ".app.src") of + case locate(src, App, ".app.src") of {ok, AppSrc} -> AppupFile = filename:basename(AppSrc) ++ ".appup.src", Default = {<<".*">>, []}, @@ -411,7 +411,16 @@ semver(Maj, Min, Patch) -> lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])). %% Locate a file in a specified application -locate(App, Suffix) -> +locate(ebin_current, App, Suffix) -> + ReleaseDir = getopt(beams_dir), + AppStr = atom_to_list(App), + case filelib:wildcard(ReleaseDir ++ "/**/ebin/" ++ AppStr ++ Suffix) of + [File] -> + {ok, File}; + [] -> + undefined + end; +locate(src, App, Suffix) -> AppStr = atom_to_list(App), SrcDirs = getopt(src_dirs), case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of