yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style: erlfmt all remaining escripts

This commit is contained in:
JimMoen 2023-11-08 11:51:10 +08:00
parent 3fd5ab2782
commit 72eb34658d
No known key found for this signature in database
GPG Key ID: 87A520B4F76BA86D
6 changed files with 444 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
bin/install_upgrade.escript
View File

@ -6,7 +6,9 @@
-define(TIMEOUT, 300000).
-define(INFO(Fmt, Args), io:format(standard_io, Fmt ++ "~n", Args)).
-define(ERROR(Fmt, Args), io:format(standard_error, "ERROR: " ++ Fmt ++ "~n", Args)).
-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>).
-define(SEMVER_RE,
<<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>
).
-mode(compile).
@ -17,7 +19,8 @@ main([Command0, DistInfoStr | CommandArgs]) ->
%% convert arguments into a proplist
Opts = parse_arguments(CommandArgs),
%% invoke the command passed as argument
F = case Command0 of
F =
case Command0 of
"install" -> fun(A, B) -> install(A, B) end;
"unpack" -> fun(A, B) -> unpack(A, B) end;
"upgrade" -> fun(A, B) -> upgrade(A, B) end;
@ -74,23 +77,31 @@ install({RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
current ->
case proplists:get_value(permanent, Opts, true) of
true ->
?INFO("Release ~s is already installed and current, making permanent.",
[Version]),
?INFO(
"Release ~s is already installed and current, making permanent.",
[Version]
),
permafy(TargetNode, RelName, Version);
false ->
?INFO("Release ~s is already installed and current.",
[Version])
?INFO(
"Release ~s is already installed and current.",
[Version]
)
end;
permanent ->
%% this release is marked permanent, however it might not the
%% one currently running
case current_release_version(TargetNode) of
Version ->
?INFO("Release ~s is already installed, running and set permanent.",
[Version]);
?INFO(
"Release ~s is already installed, running and set permanent.",
[Version]
);
CurrentVersion ->
?INFO("Release ~s is the currently running version.",
[CurrentVersion]),
?INFO(
"Release ~s is the currently running version.",
[CurrentVersion]
),
check_and_install(TargetNode, Version),
maybe_permafy(TargetNode, RelName, Version, Opts)
end;
@ -119,8 +130,10 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
?INFO("Release ~s is marked old, uninstalling it.", [Version]),
remove_release(TargetNode, Version);
unpacked ->
?INFO("Release ~s is marked unpacked, uninstalling it",
[Version]),
?INFO(
"Release ~s is marked unpacked, uninstalling it",
[Version]
),
remove_release(TargetNode, Version);
current ->
?INFO("Uninstall failed: Release ~s is marked current.", [Version]),
@ -140,7 +153,8 @@ parse_arguments(Args) ->
IsEnterprise = os:getenv("IS_ENTERPRISE") == "yes",
parse_arguments(Args, [{is_enterprise, IsEnterprise}]).
parse_arguments([], Acc) -> Acc;
parse_arguments([], Acc) ->
Acc;
parse_arguments(["--no-permanent" | Rest], Acc) ->
parse_arguments(Rest, [{permanent, false}] ++ Acc);
parse_arguments([VersionStr | Rest], Acc) ->
@ -162,18 +176,29 @@ unpack_release(RelName, TargetNode, Version, Opts) ->
{_, undefined} ->
{error, release_package_not_found};
{ReleasePackage, ReleasePackageLink} ->
?INFO("Release ~s not found, attempting to unpack ~s",
[Version, ReleasePackage]),
case rpc:call(TargetNode, release_handler, unpack_release,
[ReleasePackageLink], ?TIMEOUT) of
{ok, Vsn} -> {ok, Vsn};
?INFO(
"Release ~s not found, attempting to unpack ~s",
[Version, ReleasePackage]
),
case
rpc:call(
TargetNode,
release_handler,
unpack_release,
[ReleasePackageLink],
?TIMEOUT
)
of
{ok, Vsn} ->
{ok, Vsn};
{error, {existing_release, Vsn}} ->
%% sometimes the user may have removed the release/<vsn> dir
%% for an `unpacked` release, then we need to re-unpack it from
%% the .tar ball
untar_for_unpacked_release(str(RelName), Vsn),
{ok, Vsn};
{error, _} = Error -> Error
{error, _} = Error ->
Error
end
end;
Other ->
@ -198,8 +223,8 @@ untar_for_unpacked_release(RelName, Vsn) ->
extract_tar(Cwd, Tar) ->
case erl_tar:extract(Tar, [keep_old_files, {cwd, Cwd}, compressed]) of
ok -> ok;
{error, {Name, Reason}} -> % New erl_tar (R3A).
throw({error, {cannot_extract_file, Name, Reason}})
% New erl_tar (R3A).
{error, {Name, Reason}} -> throw({error, {cannot_extract_file, Name, Reason}})
end.
%% 1. look for a release package tarball with the provided version:
@ -217,8 +242,11 @@ find_and_link_release_package(Version, RelName, IsEnterprise) ->
ReleaseHandlerPackageLink = filename:join(Version, RelNameStr),
%% this is the symlink name we'll create once
%% we've found where the actual release package is located
ReleaseLink = filename:join(["releases", Version,
RelNameStr ++ ".tar.gz"]),
ReleaseLink = filename:join([
"releases",
Version,
RelNameStr ++ ".tar.gz"
]),
ReleaseNamePattern =
case IsEnterprise of
false -> RelNameStr;
@ -240,14 +268,18 @@ find_and_link_release_package(Version, RelName, IsEnterprise) ->
make_symlink_or_copy(filename:absname(Filename), ReleaseLink),
{Filename, ReleaseHandlerPackageLink};
Files ->
?ERROR("Found more than one package for version: '~s', "
"files: ~p", [Version, Files]),
?ERROR(
"Found more than one package for version: '~s', "
"files: ~p",
[Version, Files]
),
erlang:halt(47)
end.
make_symlink_or_copy(Filename, ReleaseLink) ->
case file:make_symlink(Filename, ReleaseLink) of
ok -> ok;
ok ->
ok;
{error, eexist} ->
?INFO("Symlink ~p already exists, recreate it", [ReleaseLink]),
ok = file:delete(ReleaseLink),
@ -267,29 +299,48 @@ check_and_install(TargetNode, Vsn) ->
%% NOTE: We cannot backup the old sys.config directly, because the
%% configs for plugins are only in app-envs, not in the old sys.config
Configs0 =
[{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)}
|| {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)],
[
{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)}
|| {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)
],
Configs1 = [{AppName, Conf} || {AppName, Conf} <- Configs0, Conf =/= []],
ok = file:write_file(filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])),
ok = file:write_file(
filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])
),
%% check and install release
case rpc:call(TargetNode, release_handler,
check_install_release, [Vsn], ?TIMEOUT) of
case
rpc:call(
TargetNode,
release_handler,
check_install_release,
[Vsn],
?TIMEOUT
)
of
{ok, _OtherVsn, _Desc} ->
ok;
{error, Reason} ->
?ERROR("Call release_handler:check_install_release failed: ~p.", [Reason]),
erlang:halt(3)
end,
case rpc:call(TargetNode, release_handler, install_release,
[Vsn, [{update_paths, true}]], ?TIMEOUT) of
case
rpc:call(
TargetNode,
release_handler,
install_release,
[Vsn, [{update_paths, true}]],
?TIMEOUT
)
of
{ok, _, _} ->
?INFO("Installed Release: ~s.", [Vsn]),
ok;
{error, {no_such_release, Vsn}} ->
VerList =
iolist_to_binary(
[io_lib:format("* ~s\t~s~n",[V,S]) || {V,S} <- which_releases(TargetNode)]),
[io_lib:format("* ~s\t~s~n", [V, S]) || {V, S} <- which_releases(TargetNode)]
),
?INFO("Installed versions:~n~s", [VerList]),
?ERROR("Unable to revert to '~s' - not installed.", [Vsn]),
erlang:halt(2);
@ -298,8 +349,10 @@ check_and_install(TargetNode, Vsn) ->
%% If the value is soft_purge, release_handler:install_release/1
%% returns {error,{old_processes,Mod}}
{error, {old_processes, Mod}} ->
?ERROR("Unable to install '~s' - old processes still running code from module ~p",
[Vsn, Mod]),
?ERROR(
"Unable to install '~s' - old processes still running code from module ~p",
[Vsn, Mod]
),
erlang:halt(3);
{error, Reason1} ->
?ERROR("Call release_handler:install_release failed: ~p", [Reason1]),
@ -310,22 +363,34 @@ maybe_permafy(TargetNode, RelName, Vsn, Opts) ->
case proplists:get_value(permanent, Opts, true) of
true ->
permafy(TargetNode, RelName, Vsn);
false -> ok
false ->
ok
end.
permafy(TargetNode, RelName, Vsn) ->
RelNameStr = atom_to_list(RelName),
ok = rpc:call(TargetNode, release_handler,
make_permanent, [Vsn], ?TIMEOUT),
ok = rpc:call(
TargetNode,
release_handler,
make_permanent,
[Vsn],
?TIMEOUT
),
?INFO("Made release permanent: ~p", [Vsn]),
%% upgrade/downgrade the scripts by replacing them
Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool", "install_upgrade.escript"],
[{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]),
filename:join(["bin", File]))
|| File <- Scripts],
[
{ok, _} = file:copy(
filename:join(["bin", File ++ "-" ++ Vsn]),
filename:join(["bin", File])
)
|| File <- Scripts
],
%% update the vars
UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]),
file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]).
file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [
append
]).
remove_release(TargetNode, Vsn) ->
case rpc:call(TargetNode, release_handler, remove_release, [Vsn], ?TIMEOUT) of
@ -344,17 +409,26 @@ which_releases(TargetNode) ->
%% the running release version is either the only one marked `current´
%% or, if none exists, the one marked `permanent`
current_release_version(TargetNode) ->
R = rpc:call(TargetNode, release_handler, which_releases,
[], ?TIMEOUT),
R = rpc:call(
TargetNode,
release_handler,
which_releases,
[],
?TIMEOUT
),
Versions = [{S, V} || {_, V, _, S} <- R],
%% current version takes priority over the permanent
proplists:get_value(current, Versions,
proplists:get_value(permanent, Versions)).
proplists:get_value(
current,
Versions,
proplists:get_value(permanent, Versions)
).
print_existing_versions(TargetNode) ->
VerList = iolist_to_binary([
io_lib:format("* ~s\t~s~n", [V, S])
|| {V,S} <- which_releases(TargetNode) ]),
|| {V, S} <- which_releases(TargetNode)
]),
?INFO("Installed versions:~n~s", [VerList]).
start_distribution(TargetNode, NameTypeArg, Cookie) ->
@ -393,11 +467,14 @@ erts_vsn() ->
validate_target_version(TargetVersion, TargetNode) ->
CurrentVersion = current_release_version(TargetNode),
case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of
{{Major, Minor}, {Major, Minor}} -> ok;
{{Major, Minor}, {Major, Minor}} ->
ok;
_ ->
?ERROR("Cannot upgrade/downgrade from '~s' to '~s'~n"
?ERROR(
"Cannot upgrade/downgrade from '~s' to '~s'~n"
"Hot upgrade is only supported between patch releases.",
[CurrentVersion, TargetVersion]),
[CurrentVersion, TargetVersion]
),
erlang:halt(48)
end.
@ -409,7 +486,8 @@ get_major_minor_vsn(Version) ->
parse_semver(Version) ->
case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of
{match, Parts} -> Parts;
{match, Parts} ->
Parts;
nomatch ->
?ERROR("Invalid semantic version: '~s'~n", [Version]),
erlang:halt(22)

15
scripts/check-deps-integrity.escript
View File

@ -20,7 +20,8 @@ apps_rebar_config(Dir) ->
%% collect a kv-list of {DepName, [{DepReference, RebarConfigFile}]}
%% the value part should have unique DepReference
collect_deps([], Acc) -> maps:to_list(Acc);
collect_deps([], Acc) ->
maps:to_list(Acc);
collect_deps([File | Files], Acc) ->
Deps =
try
@ -33,7 +34,8 @@ collect_deps([File | Files], Acc) ->
end,
collect_deps(Files, do_collect_deps(Deps, File, Acc)).
do_collect_deps([], _File, Acc) -> Acc;
do_collect_deps([], _File, Acc) ->
Acc;
%% ignore relative app dependencies
do_collect_deps([{_Name, {path, _Path}} | Deps], File, Acc) ->
do_collect_deps(Deps, File, Acc);
@ -41,7 +43,8 @@ do_collect_deps([{Name, Ref} | Deps], File, Acc) ->
Refs = maps:get(Name, Acc, []),
do_collect_deps(Deps, File, Acc#{Name => [{Ref, File} | Refs]}).
count_bad_deps([]) -> 0;
count_bad_deps([]) ->
0;
count_bad_deps([{Name, Refs0} | Rest]) ->
Refs = lists:keysort(1, Refs0),
case is_unique_ref(Refs) andalso not_branch_ref(Refs) of
@ -53,10 +56,8 @@ count_bad_deps([{Name, Refs0} | Rest]) ->
end.
is_unique_ref([_]) -> true;
is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) ->
is_unique_ref([{Ref, File2} | Rest]);
is_unique_ref(_) ->
false.
is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> is_unique_ref([{Ref, File2} | Rest]);
is_unique_ref(_) -> false.
not_branch_ref([]) -> true;
not_branch_ref([{{git, _Repo, {branch, _Branch}}, _File} | _Rest]) -> false;

8
scripts/check-i18n-style.escript
View File

@ -46,7 +46,6 @@ logerr(Fmt, Args) ->
_ = put(errors, N + 1),
ok.
check(File) ->
io:format(user, ".", []),
{ok, C} = hocon:load(File),
@ -54,9 +53,12 @@ check(File) ->
ok.
check_one_field(Name, Field) ->
maps:foreach(fun(SubName, DescAndLabel) ->
maps:foreach(
fun(SubName, DescAndLabel) ->
check_desc_and_label([Name, ".", SubName], DescAndLabel)
end, Field).
end,
Field
).
check_desc_and_label(Name, D) ->
case maps:keys(D) -- [<<"desc">>, <<"label">>] of

8
scripts/merge-config.escript
View File

@ -95,9 +95,13 @@ do_merge_desc_files(BaseConf, Cfgs) ->
true ->
{ok, Bin1} = file:read_file(CfgFile),
[Acc, io_lib:nl(), Bin1];
false -> Acc
false ->
Acc
end
end, BaseConf, Cfgs).
end,
BaseConf,
Cfgs
).
get_all_desc_files() ->
Dir = filename:join(["rel", "i18n"]),

79
scripts/relup-build/inject-relup.escript
View File

@ -20,8 +20,8 @@ inject_relup_file(File) ->
case file:script(File) of
{ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} ->
?INFO("Injecting instructions to: ~p", [File]),
UpdatedContent = {CurrRelVsn,
inject_relup_instrs(up, UpVsnRUs),
UpdatedContent =
{CurrRelVsn, inject_relup_instrs(up, UpVsnRUs),
inject_relup_instrs(down, DnVsnRUs)},
file:write_file(File, term_to_text(UpdatedContent));
{ok, _BadFormat} ->
@ -36,38 +36,49 @@ inject_relup_file(File) ->
end.
inject_relup_instrs(Type, RUs) ->
lists:map(fun({Vsn, Desc, Instrs}) ->
lists:map(
fun({Vsn, Desc, Instrs}) ->
{Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)}
end, RUs).
end,
RUs
).
append_emqx_relup_instrs(up, FromRelVsn, Instrs0) ->
{{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0),
Instrs1 ++
[ {load, {emqx_release, brutal_purge, soft_purge}}
, {load, {emqx_relup, brutal_purge, soft_purge}}
, {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}}
[
{load, {emqx_release, brutal_purge, soft_purge}},
{load, {emqx_relup, brutal_purge, soft_purge}},
{apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}}
];
append_emqx_relup_instrs(down, ToRelVsn, Instrs0) ->
{{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0),
%% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading
%% or removing the emqx_relup module.
Instrs2 = Instrs1 ++
[ {load, {emqx_release, brutal_purge, soft_purge}}
, {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}}
, {load, {emqx_relup, brutal_purge, soft_purge}}
Instrs2 =
Instrs1 ++
[
{load, {emqx_release, brutal_purge, soft_purge}},
{apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}},
{load, {emqx_relup, brutal_purge, soft_purge}}
],
Instrs2.
filter_and_check_instrs(Type, Instrs) ->
case filter_fetch_emqx_mods_and_extra(Instrs) of
{_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined ->
?ERROR("Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'"
" from the upgrade instruction list, should be 'post_release_upgrade'", []),
?ERROR(
"Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'"
" from the upgrade instruction list, should be 'post_release_upgrade'",
[]
),
error({instruction_not_found, load_object_code});
{UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined ->
?ERROR("Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'"
" from the downgrade instruction list, should be 'post_release_downgrade'", []),
?ERROR(
"Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'"
" from the downgrade instruction list, should be 'post_release_downgrade'",
[]
),
error({instruction_not_found, load_object_code});
{_, _, [], _} ->
?ERROR("Cannot find any 'load_object_code' instructions for app emqx", []),
@ -81,12 +92,15 @@ filter_fetch_emqx_mods_and_extra(Instrs) ->
lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs).
%% collect modules for emqx app
do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr,
{UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
do_filter_and_get(
{load_object_code, {emqx, _AppVsn, Mods}} = Instr,
{UpExtra, DnExtra, EmqxMods, RemainInstrs}
) ->
{UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]};
%% remove 'load' instrs for emqx_relup and emqx_release
do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs})
when Mod =:= emqx_relup; Mod =:= emqx_release ->
do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) when
Mod =:= emqx_relup; Mod =:= emqx_release
->
{UpExtra, DnExtra, EmqxMods, RemainInstrs};
%% remove 'remove' and 'purge' instrs for emqx_relup
do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
@ -94,22 +108,31 @@ do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, Rem
do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
{UpExtra, DnExtra, EmqxMods, RemainInstrs};
%% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter
do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}},
{_, DnExtra, EmqxMods, RemainInstrs}) ->
do_filter_and_get(
{apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}},
{_, DnExtra, EmqxMods, RemainInstrs}
) ->
{UpExtra0, DnExtra, EmqxMods, RemainInstrs};
%% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter
do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}},
{UpExtra, _, EmqxMods, RemainInstrs}) ->
do_filter_and_get(
{apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}},
{UpExtra, _, EmqxMods, RemainInstrs}
) ->
{UpExtra, DnExtra0, EmqxMods, RemainInstrs};
%% keep all other instrs unchanged
do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) ->
{UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}.
assert_mandatory_modules(_, Mods) ->
MandInstrs = [{load_module,emqx_release,brutal_purge,soft_purge,[]},
{load_module,emqx_relup}],
assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods),
"The following instructions are mandatory in every clause of the emqx.appup.src: ~p", [MandInstrs]).
MandInstrs = [
{load_module, emqx_release, brutal_purge, soft_purge, []},
{load_module, emqx_relup}
],
assert(
lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods),
"The following instructions are mandatory in every clause of the emqx.appup.src: ~p",
[MandInstrs]
).
assert(true, _, _) ->
ok;

166
scripts/update_appup.escript
View File

@ -1,6 +1,7 @@
#!/usr/bin/env -S escript -c
%% -*- erlang-indent-level:4 -*-
%% erlfmt-ignore
usage() ->
"A script that fills in boilerplate for appup files.
@ -35,19 +36,20 @@ Options:
--src-dirs Directories where source code is found. Defaults to '{src,apps}/**/'
".
-record(app,
{ modules :: #{module() => binary()}
, version :: string()
-record(app, {
modules :: #{module() => binary()},
version :: string()
}).
default_options() ->
#{ clone_url => find_upstream_repo("origin")
, make_command => "make emqx-rel"
, beams_dir => "_build/emqx/rel/emqx/lib/"
, check => false
, prev_tag => undefined
, src_dirs => "{src,apps}/**/"
, prev_beams_dir => undefined
#{
clone_url => find_upstream_repo("origin"),
make_command => "make emqx-rel",
beams_dir => "_build/emqx/rel/emqx/lib/",
check => false,
prev_tag => undefined,
src_dirs => "{src,apps}/**/",
prev_beams_dir => undefined
}.
%% App-specific actions that should be added unconditionally to any update/downgrade:
@ -55,8 +57,8 @@ app_specific_actions(_) ->
[].
ignored_apps() ->
[gpb %% only a build tool
] ++ otp_standard_apps().
%% only a build tool
[gpb] ++ otp_standard_apps().
main(Args) ->
#{prev_tag := Baseline} = Options = parse_args(Args, default_options()),
@ -87,9 +89,11 @@ parse_args(_, _) ->
main(Options, Baseline) ->
{CurrRelDir, PrevRelDir} = prepare(Baseline, Options),
putopt(prev_beams_dir, PrevRelDir),
log("~n===================================~n"
log(
"~n===================================~n"
"Processing changes..."
"~n===================================~n"),
"~n===================================~n"
),
CurrAppsIdx = index_apps(CurrRelDir),
PrevAppsIdx = index_apps(PrevRelDir),
%% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]),
@ -98,6 +102,7 @@ main(Options, Baseline) ->
ok = check_appup_files(),
ok = warn_and_exit(is_valid()).
%% erlfmt-ignore
warn_and_exit(true) ->
log("
NOTE: Please review the changes manually. This script does not know about NIF
@ -109,9 +114,12 @@ warn_and_exit(false) ->
halt(1).
prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) ->
log("~n===================================~n"
log(
"~n===================================~n"
"Baseline: ~s"
"~n===================================~n", [Baseline]),
"~n===================================~n",
[Baseline]
),
log("Building the current version...~n"),
ok = bash(MakeCommand),
PrevRelDir =
@ -126,6 +134,7 @@ prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}
end,
{BeamDir, PrevRelDir}.
%% erlfmt-ignore
build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) ->
BaseDir = "/tmp/emqx-appup-base/",
Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
@ -156,14 +165,17 @@ find_appup_actions(CurrApps, PrevApps) ->
end
end,
[],
CurrApps).
CurrApps
).
find_appup_actions(_App, AppIdx, AppIdx) ->
%% No changes to the app, ignore:
[];
find_appup_actions(App,
find_appup_actions(
App,
CurrAppIdx = #app{version = CurrVersion},
PrevAppIdx = #app{version = PrevVersion}) ->
PrevAppIdx = #app{version = PrevVersion}
) ->
{OldUpgrade0, OldDowngrade0} = find_base_appup_actions(App, PrevVersion),
OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0),
OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0),
@ -195,7 +207,10 @@ do_ensure_all_patch_versions(App, CurrVsn, OldActions) ->
{ok, ExpectedVsns} ->
CoveredVsns = [V || {V, _} <- OldActions, V =/= <<".*">>],
ExpectedVsnStrs = [vsn_number_to_string(V) || V <- ExpectedVsns],
MissingActions = [{V, []} || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns)],
MissingActions = [
{V, []}
|| V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns)
],
MissingActions ++ OldActions;
{error, bad_version} ->
log("WARN: Could not infer expected versions to upgrade from for ~p~n", [App]),
@ -222,7 +237,8 @@ diff_appup_instructions(ComputedChanges, PresentChanges) ->
end
end,
[],
ComputedChanges).
ComputedChanges
).
%% checks if any missing diffs are present
%% and groups them by `up' and `down' types.
@ -234,8 +250,9 @@ parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) ->
%% no diff for external dependency; ignore
ok;
_ ->
Diffs = #{ up => DiffUp
, down => DiffDown
Diffs = #{
up => DiffUp,
down => DiffDown
},
{diffs, Diffs}
end.
@ -260,7 +277,9 @@ find_base_appup_actions(App, PrevVersion) ->
{ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}.
merge_update_actions(App, Changes, Vsns, PrevVersion) ->
lists:map(fun(Ret = {<<".*">>, _}) ->
lists:map(
fun
(Ret = {<<".*">>, _}) ->
Ret;
({Vsn, Actions}) ->
case is_skipped_version(App, Vsn, PrevVersion) of
@ -271,7 +290,8 @@ merge_update_actions(App, Changes, Vsns, PrevVersion) ->
{Vsn, do_merge_update_actions(App, Changes, Actions)}
end
end,
Vsns).
Vsns
).
%% say current version is 1.1.3, and the compare base is version 1.1.1,
%% but there is a 1.1.2 in appup we may skip merging instructions for
@ -331,11 +351,14 @@ contains_restart_application(Application, Actions) ->
find_application_stop_instruction(Application, Actions) ->
{Before, After0} =
lists:splitwith(
fun({apply, {application, stop, [App]}}) when App =:= Application ->
fun
({apply, {application, stop, [App]}}) when App =:= Application ->
false;
(_) ->
true
end, Actions),
end,
Actions
),
case After0 of
[StopInst | After] ->
{Before ++ [StopInst], After};
@ -353,8 +376,10 @@ process_old_action({delete_module, Module}) ->
[Module];
process_old_action({update, Module, _Change}) ->
[Module];
process_old_action(LoadModule) when is_tuple(LoadModule) andalso
element(1, LoadModule) =:= load_module ->
process_old_action(LoadModule) when
is_tuple(LoadModule) andalso
element(1, LoadModule) =:= load_module
->
element(2, LoadModule);
process_old_action(_) ->
[].
@ -370,7 +395,8 @@ ensure_version(Version, OldInstructions) ->
contains_version(Needle, Haystack) when is_list(Needle) ->
lists:any(
fun(Regex) when is_binary(Regex) ->
fun
(Regex) when is_binary(Regex) ->
case re:run(Needle, Regex) of
{match, _} ->
true;
@ -380,7 +406,8 @@ contains_version(Needle, Haystack) when is_list(Needle) ->
(Vsn) ->
Vsn =:= Needle
end,
Haystack).
Haystack
).
%% As a best effort approach, we assume that we only bump patch
%% version numbers between release upgrades for our dependencies and
@ -437,7 +464,8 @@ update_appups(Changes) ->
fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) ->
do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade)
end,
Changes).
Changes
).
do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
case locate_current_src(App, ".appup.src") of
@ -469,8 +497,11 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
ok;
{diffs, Diffs} ->
set_invalid(),
log("ERROR: Appup file for '~p' is not complete.~n"
"Missing:~100p~n", [App, Diffs]),
log(
"ERROR: Appup file for '~p' is not complete.~n"
"Missing:~100p~n",
[App, Diffs]
),
notok
end.
@ -496,9 +527,12 @@ render_appup(App, File, Up, Down) ->
end.
do_render_appup(File, Up, Down) ->
IOList = io_lib:format("%% -*- mode: erlang -*-~n"
IOList = io_lib:format(
"%% -*- mode: erlang -*-~n"
"%% Unless you know what you are doing, DO NOT edit manually!!~n"
"{VSN,~n ~p,~n ~p}.~n", [Up, Down]),
"{VSN,~n ~p,~n ~p}.~n",
[Up, Down]
),
ok = file:write_file(File, IOList).
create_stub(App) ->
@ -544,15 +578,20 @@ index_app(AppFile) ->
%% Note: assuming that beams are always located in the same directory where app file is:
EbinDir = filename:dirname(AppFile),
Modules = hashsums(EbinDir),
{App, #app{ version = Vsn
, modules = Modules
{App, #app{
version = Vsn,
modules = Modules
}}.
diff_app(UpOrDown, App,
diff_app(
UpOrDown,
App,
#app{version = NewVersion, modules = NewModules},
#app{version = OldVersion, modules = OldModules}) ->
#app{version = OldVersion, modules = OldModules}
) ->
{New, Changed} =
maps:fold( fun(Mod, MD5, {New, Changed}) ->
maps:fold(
fun(Mod, MD5, {New, Changed}) ->
case OldModules of
#{Mod := OldMD5} when MD5 =:= OldMD5 ->
{New, Changed};
@ -561,13 +600,15 @@ diff_app(UpOrDown, App,
_ ->
{[Mod | New], Changed}
end
end
, {[], []}
, NewModules
end,
{[], []},
NewModules
),
Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)),
Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end,
[{added, New}, {changed, Changed}, {deleted, Deleted}]),
Changes = lists:filter(
fun({_T, L}) -> length(L) > 0 end,
[{added, New}, {changed, Changed}, {deleted, Deleted}]
),
case NewVersion =:= OldVersion of
true when Changes =:= [] ->
%% no change
@ -577,13 +618,17 @@ diff_app(UpOrDown, App,
case UpOrDown =:= up of
true ->
%% only log for the upgrade case because it would be the same result
log("ERROR: Application '~p' contains changes, but its version is not updated. ~s",
[App, format_changes(Changes)]);
log(
"ERROR: Application '~p' contains changes, but its version is not updated. ~s",
[App, format_changes(Changes)]
);
false ->
ok
end;
false ->
log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]),
log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [
App, OldVersion, UpOrDown, NewVersion
]),
log("INFO: changes [~p]: ~p~n", [UpOrDown, Changes]),
ok
end,
@ -594,14 +639,16 @@ format_changes(Changes) ->
-spec hashsums(file:filename()) -> #{module() => binary()}.
hashsums(EbinDir) ->
maps:from_list(lists:map(
maps:from_list(
lists:map(
fun(Beam) ->
File = filename:join(EbinDir, Beam),
{ok, Ret = {_Module, _MD5}} = beam_lib:md5(File),
Ret
end,
filelib:wildcard("*.beam", EbinDir)
)).
)
).
is_app_external(App) ->
Ext = ".app.src",
@ -674,12 +721,13 @@ do_locate(Dir, App, Suffix) ->
end.
find_app(Pattern) ->
lists:filter(fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end,
filelib:wildcard(Pattern)).
lists:filter(
fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end,
filelib:wildcard(Pattern)
).
bash(undefined) -> ok;
bash(Script) ->
bash(Script, []).
bash(Script) -> bash(Script, []).
bash(Script, Env) ->
log("+ ~s~n+ Env: ~p~n", [Script, Env]),
@ -695,9 +743,11 @@ cmd(Exec, Params) ->
fail("Executable not found in $PATH: ~s", [Exec]);
Path ->
Params1 = maps:to_list(maps:with([env, args, cd], Params)),
Port = erlang:open_port( {spawn_executable, Path}
, [ exit_status
, nouse_stdio
Port = erlang:open_port(
{spawn_executable, Path},
[
exit_status,
nouse_stdio
| Params1
]
),