Merge pull request #5923 from k32/update-appup-binary

feat(update_appup): Support binary releases (.zip)

scripts/update_appup.escript

@@ -33,6 +33,8 @@ Options:
   --make-command    A command used to assemble the release
   --release-dir     Release directory
   --src-dirs        Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
+  --binary-rel-url  Binary release URL pattern. %TAG% variable is substituted with the release tag.
+                    E.g. \"https://github.com/emqx/emqx/releases/download/v4.3.8/emqx-centos7-%TAG%-amd64.zip\"
 ".
 
 -record(app,
@@ -47,6 +49,7 @@ default_options() ->
      , check          => false
      , prev_tag       => undefined
      , src_dirs       => "{src,apps,lib-*}/**/"
+     , binary_rel_url => undefined
      }.
 
 %% App-specific actions that should be added unconditionally to any update/downgrade:
@@ -54,12 +57,12 @@ app_specific_actions(_) ->
     [].
 
 ignored_apps() ->
-    [emqx_dashboard, emqx_management].
+    [emqx_dashboard, emqx_management] ++ otp_standard_apps().
 
 main(Args) ->
     #{current_release := CurrentRelease} = Options = parse_args(Args, default_options()),
     init_globals(Options),
-    case find_pred_tag(CurrentRelease) of
+    case find_prev_tag(CurrentRelease) of
         {ok, Baseline} ->
             main(Options, Baseline);
         undefined ->
@@ -85,6 +88,8 @@ parse_args(["--src-dirs", Pattern|Rest], State) ->
     parse_args(Rest, State#{src_dirs => Pattern});
 parse_args(["--prev-tag", Tag|Rest], State) ->
     parse_args(Rest, State#{prev_tag => Tag});
+parse_args(["--binary-rel-url", URL|Rest], State) ->
+    parse_args(Rest, State#{binary_rel_url => {ok, URL}});
 parse_args(_, _) ->
     fail(usage()).
 
@@ -95,7 +100,7 @@ main(Options, Baseline) ->
         "~n===================================~n"),
     CurrAppsIdx = index_apps(CurrRelDir),
     PrevAppsIdx = index_apps(PrevRelDir),
-    %% log("Curr: ~p~nPrev: ~p~n", [CurrApps, PrevApps]),
+    %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]),
     AppupChanges = find_appup_actions(CurrAppsIdx, PrevAppsIdx),
     case getopt(check) of
         true ->
@@ -122,17 +127,25 @@ warn_and_exit(false) ->
     log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"),
     halt(1).
 
-prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) ->
+prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir, binary_rel_url := BinRel}) ->
     log("~n===================================~n"
         "Baseline: ~s"
         "~n===================================~n", [Baseline]),
     log("Building the current version...~n"),
     bash(MakeCommand),
     log("Downloading and building the previous release...~n"),
-    {ok, PrevRootDir} = build_pred_release(Baseline, Options),
+    PrevRelDir =
-    {BeamDir, filename:join(PrevRootDir, BeamDir)}.
+        case BinRel of
+            undefined ->
+                {ok, PrevRootDir} = build_prev_release(Baseline, Options),
+                filename:join(PrevRootDir, BeamDir);
+            {ok, URL} ->
+                {ok, PrevRootDir} = download_prev_release(Baseline, Options),
+                PrevRootDir
+        end,
+    {BeamDir, PrevRelDir}.
 
-build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) ->
+build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) ->
     BaseDir = "/tmp/emqx-baseline/",
     Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
     %% TODO: shallow clone
@@ -144,10 +157,22 @@ build_pred_release(Baseline, #{clone_url := Repo, make_command := MakeCommand})
     bash(Script, Env),
     {ok, filename:join(BaseDir, Dir)}.
 
+download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) ->
+    URL = string:replace(URL0, "%TAG%", Tag, all),
+    BaseDir = "/tmp/emqx-baseline-bin/",
+    Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
+    Filename = filename:join(BaseDir, Dir),
+    Script = "mkdir -p ${OUTFILE} &&
+              { [ -f ${OUTFILE}.zip ] || wget -O ${OUTFILE}.zip ${URL}; } &&
+              unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
+    Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
+    bash(Script, Env),
+    {ok, Filename}.
+
 find_upstream_repo(Remote) ->
     string:trim(os:cmd("git remote get-url " ++ Remote)).
 
-find_pred_tag(CurrentRelease) ->
+find_prev_tag(CurrentRelease) ->
     case getopt(prev_tag) of
         undefined ->
             {Maj, Min, Patch} = parse_semver(CurrentRelease),
@@ -190,7 +215,7 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
 
 find_old_appup_actions(App, PrevVersion) ->
     {Upgrade0, Downgrade0} =
-        case locate(App, ".appup.src") of
+        case locate(ebin_current, App, ".appup") of
             {ok, AppupFile} ->
                 {_, U, D} = read_appup(AppupFile),
                 {U, D};
@@ -261,7 +286,7 @@ update_appups(Changes) ->
       Changes).
 
 do_update_appup(App, Upgrade, Downgrade) ->
-    case locate(App, ".appup.src") of
+    case locate(src, App, ".appup.src") of
         {ok, AppupFile} ->
             render_appfile(AppupFile, Upgrade, Downgrade);
         undefined ->
@@ -270,7 +295,7 @@ do_update_appup(App, Upgrade, Downgrade) ->
                     render_appfile(AppupFile, Upgrade, Downgrade);
                 false ->
                     set_invalid(),
-                    log("ERROR: Appup file for the external dependency '~p' is not complete.~n       Missing changes: ~p", [App, Upgrade])
+                    log("ERROR: Appup file for the external dependency '~p' is not complete.~n       Missing changes: ~p~n", [App, Upgrade])
             end
     end.
 
@@ -283,7 +308,7 @@ render_appfile(File, Upgrade, Downgrade) ->
     ok = file:write_file(File, IOList).
 
 create_stub(App) ->
-    case locate(App, ".app.src") of
+    case locate(src, App, ".app.src") of
         {ok, AppSrc} ->
             AppupFile = filename:basename(AppSrc) ++ ".appup.src",
             Default = {<<".*">>, []},
@@ -386,7 +411,16 @@ semver(Maj, Min, Patch) ->
     lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])).
 
 %% Locate a file in a specified application
-locate(App, Suffix) ->
+locate(ebin_current, App, Suffix) ->
+    ReleaseDir = getopt(beams_dir),
+    AppStr = atom_to_list(App),
+    case filelib:wildcard(ReleaseDir ++ "/**/ebin/" ++ AppStr ++ Suffix) of
+        [File] ->
+            {ok, File};
+        [] ->
+            undefined
+    end;
+locate(src, App, Suffix) ->
     AppStr = atom_to_list(App),
     SrcDirs = getopt(src_dirs),
     case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of
@@ -444,3 +478,6 @@ ensure_string(Str) when is_binary(Str) ->
     binary_to_list(Str);
 ensure_string(Str) when is_list(Str) ->
     Str.
+
+otp_standard_apps() ->
+    [ssl, mnesia, kernel, asn1, stdlib].