diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 3e39c787b..421e63b21 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -4,9 +4,11 @@ %% ex: ft=erlang ts=4 sw=4 et -define(TIMEOUT, 300000). --define(INFO(Fmt,Args), io:format(standard_io, Fmt++"~n",Args)). --define(ERROR(Fmt,Args), io:format(standard_error, "ERROR: "++Fmt++"~n",Args)). --define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>). +-define(INFO(Fmt, Args), io:format(standard_io, Fmt ++ "~n", Args)). +-define(ERROR(Fmt, Args), io:format(standard_error, "ERROR: " ++ Fmt ++ "~n", Args)). +-define(SEMVER_RE, + <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">> +). -mode(compile). @@ -17,14 +19,15 @@ main([Command0, DistInfoStr | CommandArgs]) -> %% convert arguments into a proplist Opts = parse_arguments(CommandArgs), %% invoke the command passed as argument - F = case Command0 of - "install" -> fun(A, B) -> install(A, B) end; - "unpack" -> fun(A, B) -> unpack(A, B) end; - "upgrade" -> fun(A, B) -> upgrade(A, B) end; - "downgrade" -> fun(A, B) -> downgrade(A, B) end; - "uninstall" -> fun(A, B) -> uninstall(A, B) end; - "versions" -> fun(A, B) -> versions(A, B) end - end, + F = + case Command0 of + "install" -> fun(A, B) -> install(A, B) end; + "unpack" -> fun(A, B) -> unpack(A, B) end; + "upgrade" -> fun(A, B) -> upgrade(A, B) end; + "downgrade" -> fun(A, B) -> downgrade(A, B) end; + "uninstall" -> fun(A, B) -> uninstall(A, B) end; + "versions" -> fun(A, B) -> versions(A, B) end + end, F(DistInfo, Opts); main(Args) -> ?INFO("unknown args: ~p", [Args]), @@ -38,15 +41,15 @@ unpack({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> ?INFO("Unpacked successfully: ~p", [Vsn]); old -> %% no need to unpack, has been installed previously - ?INFO("Release ~s is marked old.",[Version]); + ?INFO("Release ~s is marked old.", [Version]); unpacked -> - ?INFO("Release ~s is already unpacked.",[Version]); + ?INFO("Release ~s is already unpacked.", [Version]); current -> - ?INFO("Release ~s is already installed and current.",[Version]); + ?INFO("Release ~s is already installed and current.", [Version]); permanent -> - ?INFO("Release ~s is already installed and set permanent.",[Version]); + ?INFO("Release ~s is already installed and set permanent.", [Version]); {error, Reason} -> - ?INFO("Unpack failed: ~p.",[Reason]), + ?INFO("Unpack failed: ~p.", [Reason]), print_existing_versions(TargetNode), erlang:halt(2) end; @@ -64,38 +67,46 @@ install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> maybe_permafy(TargetNode, RelName, Vsn, Opts); old -> %% no need to unpack, has been installed previously - ?INFO("Release ~s is marked old, switching to it.",[Version]), + ?INFO("Release ~s is marked old, switching to it.", [Version]), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts); unpacked -> - ?INFO("Release ~s is already unpacked, now installing.",[Version]), + ?INFO("Release ~s is already unpacked, now installing.", [Version]), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts); current -> case proplists:get_value(permanent, Opts, true) of true -> - ?INFO("Release ~s is already installed and current, making permanent.", - [Version]), + ?INFO( + "Release ~s is already installed and current, making permanent.", + [Version] + ), permafy(TargetNode, RelName, Version); false -> - ?INFO("Release ~s is already installed and current.", - [Version]) + ?INFO( + "Release ~s is already installed and current.", + [Version] + ) end; permanent -> %% this release is marked permanent, however it might not the %% one currently running case current_release_version(TargetNode) of Version -> - ?INFO("Release ~s is already installed, running and set permanent.", - [Version]); + ?INFO( + "Release ~s is already installed, running and set permanent.", + [Version] + ); CurrentVersion -> - ?INFO("Release ~s is the currently running version.", - [CurrentVersion]), + ?INFO( + "Release ~s is the currently running version.", + [CurrentVersion] + ), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts) end; {error, Reason} -> - ?INFO("Unpack failed: ~p",[Reason]), + ?INFO("Unpack failed: ~p", [Reason]), print_existing_versions(TargetNode), erlang:halt(2) end; @@ -119,8 +130,10 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) -> ?INFO("Release ~s is marked old, uninstalling it.", [Version]), remove_release(TargetNode, Version); unpacked -> - ?INFO("Release ~s is marked unpacked, uninstalling it", - [Version]), + ?INFO( + "Release ~s is marked unpacked, uninstalling it", + [Version] + ), remove_release(TargetNode, Version); current -> ?INFO("Uninstall failed: Release ~s is marked current.", [Version]), @@ -140,10 +153,11 @@ parse_arguments(Args) -> IsEnterprise = os:getenv("IS_ENTERPRISE") == "yes", parse_arguments(Args, [{is_enterprise, IsEnterprise}]). -parse_arguments([], Acc) -> Acc; -parse_arguments(["--no-permanent"|Rest], Acc) -> +parse_arguments([], Acc) -> + Acc; +parse_arguments(["--no-permanent" | Rest], Acc) -> parse_arguments(Rest, [{permanent, false}] ++ Acc); -parse_arguments([VersionStr|Rest], Acc) -> +parse_arguments([VersionStr | Rest], Acc) -> Version = parse_version(VersionStr), parse_arguments(Rest, [{version, Version}] ++ Acc). @@ -162,18 +176,29 @@ unpack_release(RelName, TargetNode, Version, Opts) -> {_, undefined} -> {error, release_package_not_found}; {ReleasePackage, ReleasePackageLink} -> - ?INFO("Release ~s not found, attempting to unpack ~s", - [Version, ReleasePackage]), - case rpc:call(TargetNode, release_handler, unpack_release, - [ReleasePackageLink], ?TIMEOUT) of - {ok, Vsn} -> {ok, Vsn}; + ?INFO( + "Release ~s not found, attempting to unpack ~s", + [Version, ReleasePackage] + ), + case + rpc:call( + TargetNode, + release_handler, + unpack_release, + [ReleasePackageLink], + ?TIMEOUT + ) + of + {ok, Vsn} -> + {ok, Vsn}; {error, {existing_release, Vsn}} -> %% sometimes the user may have removed the release/ dir %% for an `unpacked` release, then we need to re-unpack it from %% the .tar ball untar_for_unpacked_release(str(RelName), Vsn), {ok, Vsn}; - {error, _} = Error -> Error + {error, _} = Error -> + Error end end; Other -> @@ -198,8 +223,8 @@ untar_for_unpacked_release(RelName, Vsn) -> extract_tar(Cwd, Tar) -> case erl_tar:extract(Tar, [keep_old_files, {cwd, Cwd}, compressed]) of ok -> ok; - {error, {Name, Reason}} -> % New erl_tar (R3A). - throw({error, {cannot_extract_file, Name, Reason}}) + % New erl_tar (R3A). + {error, {Name, Reason}} -> throw({error, {cannot_extract_file, Name, Reason}}) end. %% 1. look for a release package tarball with the provided version: @@ -217,8 +242,11 @@ find_and_link_release_package(Version, RelName, IsEnterprise) -> ReleaseHandlerPackageLink = filename:join(Version, RelNameStr), %% this is the symlink name we'll create once %% we've found where the actual release package is located - ReleaseLink = filename:join(["releases", Version, - RelNameStr ++ ".tar.gz"]), + ReleaseLink = filename:join([ + "releases", + Version, + RelNameStr ++ ".tar.gz" + ]), ReleaseNamePattern = case IsEnterprise of false -> RelNameStr; @@ -240,14 +268,18 @@ find_and_link_release_package(Version, RelName, IsEnterprise) -> make_symlink_or_copy(filename:absname(Filename), ReleaseLink), {Filename, ReleaseHandlerPackageLink}; Files -> - ?ERROR("Found more than one package for version: '~s', " - "files: ~p", [Version, Files]), + ?ERROR( + "Found more than one package for version: '~s', " + "files: ~p", + [Version, Files] + ), erlang:halt(47) end. make_symlink_or_copy(Filename, ReleaseLink) -> case file:make_symlink(Filename, ReleaseLink) of - ok -> ok; + ok -> + ok; {error, eexist} -> ?INFO("Symlink ~p already exists, recreate it", [ReleaseLink]), ok = file:delete(ReleaseLink), @@ -260,36 +292,55 @@ make_symlink_or_copy(Filename, ReleaseLink) -> end. parse_version(V) when is_list(V) -> - hd(string:tokens(V,"/")). + hd(string:tokens(V, "/")). check_and_install(TargetNode, Vsn) -> %% Backup the sys.config, this will be used when we check and install release %% NOTE: We cannot backup the old sys.config directly, because the %% configs for plugins are only in app-envs, not in the old sys.config Configs0 = - [{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} - || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)], + [ + {AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} + || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT) + ], Configs1 = [{AppName, Conf} || {AppName, Conf} <- Configs0, Conf =/= []], - ok = file:write_file(filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])), + ok = file:write_file( + filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1]) + ), %% check and install release - case rpc:call(TargetNode, release_handler, - check_install_release, [Vsn], ?TIMEOUT) of + case + rpc:call( + TargetNode, + release_handler, + check_install_release, + [Vsn], + ?TIMEOUT + ) + of {ok, _OtherVsn, _Desc} -> ok; {error, Reason} -> ?ERROR("Call release_handler:check_install_release failed: ~p.", [Reason]), erlang:halt(3) end, - case rpc:call(TargetNode, release_handler, install_release, - [Vsn, [{update_paths, true}]], ?TIMEOUT) of + case + rpc:call( + TargetNode, + release_handler, + install_release, + [Vsn, [{update_paths, true}]], + ?TIMEOUT + ) + of {ok, _, _} -> ?INFO("Installed Release: ~s.", [Vsn]), ok; {error, {no_such_release, Vsn}} -> VerList = iolist_to_binary( - [io_lib:format("* ~s\t~s~n",[V,S]) || {V,S} <- which_releases(TargetNode)]), + [io_lib:format("* ~s\t~s~n", [V, S]) || {V, S} <- which_releases(TargetNode)] + ), ?INFO("Installed versions:~n~s", [VerList]), ?ERROR("Unable to revert to '~s' - not installed.", [Vsn]), erlang:halt(2); @@ -298,11 +349,13 @@ check_and_install(TargetNode, Vsn) -> %% If the value is soft_purge, release_handler:install_release/1 %% returns {error,{old_processes,Mod}} {error, {old_processes, Mod}} -> - ?ERROR("Unable to install '~s' - old processes still running code from module ~p", - [Vsn, Mod]), + ?ERROR( + "Unable to install '~s' - old processes still running code from module ~p", + [Vsn, Mod] + ), erlang:halt(3); {error, Reason1} -> - ?ERROR("Call release_handler:install_release failed: ~p",[Reason1]), + ?ERROR("Call release_handler:install_release failed: ~p", [Reason1]), erlang:halt(4) end. @@ -310,22 +363,34 @@ maybe_permafy(TargetNode, RelName, Vsn, Opts) -> case proplists:get_value(permanent, Opts, true) of true -> permafy(TargetNode, RelName, Vsn); - false -> ok + false -> + ok end. permafy(TargetNode, RelName, Vsn) -> RelNameStr = atom_to_list(RelName), - ok = rpc:call(TargetNode, release_handler, - make_permanent, [Vsn], ?TIMEOUT), + ok = rpc:call( + TargetNode, + release_handler, + make_permanent, + [Vsn], + ?TIMEOUT + ), ?INFO("Made release permanent: ~p", [Vsn]), %% upgrade/downgrade the scripts by replacing them - Scripts = [RelNameStr, RelNameStr++"_ctl", "nodetool", "install_upgrade.escript"], - [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]), - filename:join(["bin", File])) - || File <- Scripts], + Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool", "install_upgrade.escript"], + [ + {ok, _} = file:copy( + filename:join(["bin", File ++ "-" ++ Vsn]), + filename:join(["bin", File]) + ) + || File <- Scripts + ], %% update the vars UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]), - file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]). + file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [ + append + ]). remove_release(TargetNode, Vsn) -> case rpc:call(TargetNode, release_handler, remove_release, [Vsn], ?TIMEOUT) of @@ -339,22 +404,31 @@ remove_release(TargetNode, Vsn) -> which_releases(TargetNode) -> R = rpc:call(TargetNode, release_handler, which_releases, [], ?TIMEOUT), - [ {V, S} || {_,V,_, S} <- R ]. + [{V, S} || {_, V, _, S} <- R]. %% the running release version is either the only one marked `current´ %% or, if none exists, the one marked `permanent` current_release_version(TargetNode) -> - R = rpc:call(TargetNode, release_handler, which_releases, - [], ?TIMEOUT), - Versions = [ {S, V} || {_,V,_, S} <- R ], + R = rpc:call( + TargetNode, + release_handler, + which_releases, + [], + ?TIMEOUT + ), + Versions = [{S, V} || {_, V, _, S} <- R], %% current version takes priority over the permanent - proplists:get_value(current, Versions, - proplists:get_value(permanent, Versions)). + proplists:get_value( + current, + Versions, + proplists:get_value(permanent, Versions) + ). print_existing_versions(TargetNode) -> VerList = iolist_to_binary([ - io_lib:format("* ~s\t~s~n",[V,S]) - || {V,S} <- which_releases(TargetNode) ]), + io_lib:format("* ~s\t~s~n", [V, S]) + || {V, S} <- which_releases(TargetNode) + ]), ?INFO("Installed versions:~n~s", [VerList]). start_distribution(TargetNode, NameTypeArg, Cookie) -> @@ -378,12 +452,12 @@ make_script_node(Node) -> %% get name type from arg get_name_type(NameTypeArg) -> - case NameTypeArg of - "-sname" -> - shortnames; - _ -> - longnames - end. + case NameTypeArg of + "-sname" -> + shortnames; + _ -> + longnames + end. erts_vsn() -> {ok, Str} = file:read_file(filename:join(["releases", "start_erl.data"])), @@ -393,11 +467,14 @@ erts_vsn() -> validate_target_version(TargetVersion, TargetNode) -> CurrentVersion = current_release_version(TargetNode), case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of - {{Major, Minor}, {Major, Minor}} -> ok; + {{Major, Minor}, {Major, Minor}} -> + ok; _ -> - ?ERROR("Cannot upgrade/downgrade from '~s' to '~s'~n" - "Hot upgrade is only supported between patch releases.", - [CurrentVersion, TargetVersion]), + ?ERROR( + "Cannot upgrade/downgrade from '~s' to '~s'~n" + "Hot upgrade is only supported between patch releases.", + [CurrentVersion, TargetVersion] + ), erlang:halt(48) end. @@ -409,7 +486,8 @@ get_major_minor_vsn(Version) -> parse_semver(Version) -> case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of - {match, Parts} -> Parts; + {match, Parts} -> + Parts; nomatch -> ?ERROR("Invalid semantic version: '~s'~n", [Version]), erlang:halt(22) diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript index 03cd509de..304c771fd 100755 --- a/scripts/check-deps-integrity.escript +++ b/scripts/check-deps-integrity.escript @@ -20,7 +20,8 @@ apps_rebar_config(Dir) -> %% collect a kv-list of {DepName, [{DepReference, RebarConfigFile}]} %% the value part should have unique DepReference -collect_deps([], Acc) -> maps:to_list(Acc); +collect_deps([], Acc) -> + maps:to_list(Acc); collect_deps([File | Files], Acc) -> Deps = try @@ -28,12 +29,13 @@ collect_deps([File | Files], Acc) -> {deps, Deps0} = lists:keyfind(deps, 1, Config), Deps0 catch - C : E : St -> + C:E:St -> erlang:raise(C, {E, {failed_to_find_deps_in_rebar_config, File}}, St) end, collect_deps(Files, do_collect_deps(Deps, File, Acc)). -do_collect_deps([], _File, Acc) -> Acc; +do_collect_deps([], _File, Acc) -> + Acc; %% ignore relative app dependencies do_collect_deps([{_Name, {path, _Path}} | Deps], File, Acc) -> do_collect_deps(Deps, File, Acc); @@ -41,7 +43,8 @@ do_collect_deps([{Name, Ref} | Deps], File, Acc) -> Refs = maps:get(Name, Acc, []), do_collect_deps(Deps, File, Acc#{Name => [{Ref, File} | Refs]}). -count_bad_deps([]) -> 0; +count_bad_deps([]) -> + 0; count_bad_deps([{Name, Refs0} | Rest]) -> Refs = lists:keysort(1, Refs0), case is_unique_ref(Refs) andalso not_branch_ref(Refs) of @@ -53,10 +56,8 @@ count_bad_deps([{Name, Refs0} | Rest]) -> end. is_unique_ref([_]) -> true; -is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> - is_unique_ref([{Ref, File2} | Rest]); -is_unique_ref(_) -> - false. +is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> is_unique_ref([{Ref, File2} | Rest]); +is_unique_ref(_) -> false. not_branch_ref([]) -> true; not_branch_ref([{{git, _Repo, {branch, _Branch}}, _File} | _Rest]) -> false; diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index e7e0ea42e..4aea1b6d5 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -46,7 +46,6 @@ logerr(Fmt, Args) -> _ = put(errors, N + 1), ok. - check(File) -> io:format(user, ".", []), {ok, C} = hocon:load(File), @@ -54,9 +53,12 @@ check(File) -> ok. check_one_field(Name, Field) -> - maps:foreach(fun(SubName, DescAndLabel) -> - check_desc_and_label([Name, ".", SubName], DescAndLabel) - end, Field). + maps:foreach( + fun(SubName, DescAndLabel) -> + check_desc_and_label([Name, ".", SubName], DescAndLabel) + end, + Field + ). check_desc_and_label(Name, D) -> case maps:keys(D) -- [<<"desc">>, <<"label">>] of @@ -84,8 +86,8 @@ check_desc_string(Name, <<>>) -> check_desc_string(Name, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> - logerr("~s: ~s~n", [Name, Reason]) - end, + logerr("~s: ~s~n", [Name, Reason]) + end, case Str of [$\s | _] -> Err("remove leading whitespace"); diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index 25593a323..aad33f7ac 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -90,14 +90,18 @@ merge_desc_files() -> do_merge_desc_files(BaseConf, Cfgs) -> lists:foldl( - fun(CfgFile, Acc) -> - case filelib:is_regular(CfgFile) of - true -> - {ok, Bin1} = file:read_file(CfgFile), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Cfgs). + fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + [Acc, io_lib:nl(), Bin1]; + false -> + Acc + end + end, + BaseConf, + Cfgs + ). get_all_desc_files() -> Dir = filename:join(["rel", "i18n"]), diff --git a/scripts/relup-build/inject-relup.escript b/scripts/relup-build/inject-relup.escript index b7d905979..7e252f741 100755 --- a/scripts/relup-build/inject-relup.escript +++ b/scripts/relup-build/inject-relup.escript @@ -20,9 +20,9 @@ inject_relup_file(File) -> case file:script(File) of {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> ?INFO("Injecting instructions to: ~p", [File]), - UpdatedContent = {CurrRelVsn, - inject_relup_instrs(up, UpVsnRUs), - inject_relup_instrs(down, DnVsnRUs)}, + UpdatedContent = + {CurrRelVsn, inject_relup_instrs(up, UpVsnRUs), + inject_relup_instrs(down, DnVsnRUs)}, file:write_file(File, term_to_text(UpdatedContent)); {ok, _BadFormat} -> ?ERROR("Bad formatted relup file: ~p", [File]), @@ -36,38 +36,49 @@ inject_relup_file(File) -> end. inject_relup_instrs(Type, RUs) -> - lists:map(fun({Vsn, Desc, Instrs}) -> - {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} - end, RUs). + lists:map( + fun({Vsn, Desc, Instrs}) -> + {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} + end, + RUs + ). append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> - {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), - Instrs1 ++ - [ {load, {emqx_release, brutal_purge, soft_purge}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} + {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), + Instrs1 ++ + [ + {load, {emqx_release, brutal_purge, soft_purge}}, + {load, {emqx_relup, brutal_purge, soft_purge}}, + {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} ]; - append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0), %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading %% or removing the emqx_relup module. - Instrs2 = Instrs1 ++ - [ {load, {emqx_release, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - ], + Instrs2 = + Instrs1 ++ + [ + {load, {emqx_release, brutal_purge, soft_purge}}, + {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}}, + {load, {emqx_relup, brutal_purge, soft_purge}} + ], Instrs2. filter_and_check_instrs(Type, Instrs) -> case filter_fetch_emqx_mods_and_extra(Instrs) of {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined -> - ?ERROR("Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" - " from the upgrade instruction list, should be 'post_release_upgrade'", []), + ?ERROR( + "Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" + " from the upgrade instruction list, should be 'post_release_upgrade'", + [] + ), error({instruction_not_found, load_object_code}); {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined -> - ?ERROR("Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" - " from the downgrade instruction list, should be 'post_release_downgrade'", []), + ?ERROR( + "Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" + " from the downgrade instruction list, should be 'post_release_downgrade'", + [] + ), error({instruction_not_found, load_object_code}); {_, _, [], _} -> ?ERROR("Cannot find any 'load_object_code' instructions for app emqx", []), @@ -81,12 +92,15 @@ filter_fetch_emqx_mods_and_extra(Instrs) -> lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs). %% collect modules for emqx app -do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, - {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {load_object_code, {emqx, _AppVsn, Mods}} = Instr, + {UpExtra, DnExtra, EmqxMods, RemainInstrs} +) -> {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; %% remove 'load' instrs for emqx_relup and emqx_release -do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) - when Mod =:= emqx_relup; Mod =:= emqx_release -> +do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) when + Mod =:= emqx_relup; Mod =:= emqx_release +-> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; %% remove 'remove' and 'purge' instrs for emqx_relup do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> @@ -94,22 +108,31 @@ do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, Rem do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; %% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter -do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, - {_, DnExtra, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, + {_, DnExtra, EmqxMods, RemainInstrs} +) -> {UpExtra0, DnExtra, EmqxMods, RemainInstrs}; %% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter -do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, - {UpExtra, _, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, + {UpExtra, _, EmqxMods, RemainInstrs} +) -> {UpExtra, DnExtra0, EmqxMods, RemainInstrs}; %% keep all other instrs unchanged do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}. assert_mandatory_modules(_, Mods) -> - MandInstrs = [{load_module,emqx_release,brutal_purge,soft_purge,[]}, - {load_module,emqx_relup}], - assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods), - "The following instructions are mandatory in every clause of the emqx.appup.src: ~p", [MandInstrs]). + MandInstrs = [ + {load_module, emqx_release, brutal_purge, soft_purge, []}, + {load_module, emqx_relup} + ], + assert( + lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods), + "The following instructions are mandatory in every clause of the emqx.appup.src: ~p", + [MandInstrs] + ). assert(true, _, _) -> ok; diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 5ae0918bb..945a948b0 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,6 +1,7 @@ #!/usr/bin/env -S escript -c %% -*- erlang-indent-level:4 -*- +%% erlfmt-ignore usage() -> "A script that fills in boilerplate for appup files. @@ -35,51 +36,52 @@ Options: --src-dirs Directories where source code is found. Defaults to '{src,apps}/**/' ". --record(app, - { modules :: #{module() => binary()} - , version :: string() - }). +-record(app, { + modules :: #{module() => binary()}, + version :: string() +}). default_options() -> - #{ clone_url => find_upstream_repo("origin") - , make_command => "make emqx-rel" - , beams_dir => "_build/emqx/rel/emqx/lib/" - , check => false - , prev_tag => undefined - , src_dirs => "{src,apps}/**/" - , prev_beams_dir => undefined - }. + #{ + clone_url => find_upstream_repo("origin"), + make_command => "make emqx-rel", + beams_dir => "_build/emqx/rel/emqx/lib/", + check => false, + prev_tag => undefined, + src_dirs => "{src,apps}/**/", + prev_beams_dir => undefined + }. %% App-specific actions that should be added unconditionally to any update/downgrade: app_specific_actions(_) -> []. ignored_apps() -> - [gpb %% only a build tool - ] ++ otp_standard_apps(). + %% only a build tool + [gpb] ++ otp_standard_apps(). main(Args) -> #{prev_tag := Baseline} = Options = parse_args(Args, default_options()), init_globals(Options), main(Options, Baseline). -parse_args([PrevTag = [A|_]], State) when A =/= $- -> +parse_args([PrevTag = [A | _]], State) when A =/= $- -> State#{prev_tag => PrevTag}; -parse_args(["--check"|Rest], State) -> +parse_args(["--check" | Rest], State) -> parse_args(Rest, State#{check => true}); -parse_args(["--skip-build"|Rest], State) -> +parse_args(["--skip-build" | Rest], State) -> parse_args(Rest, State#{make_command => undefined}); -parse_args(["--repo", Repo|Rest], State) -> +parse_args(["--repo", Repo | Rest], State) -> parse_args(Rest, State#{clone_url => Repo}); -parse_args(["--remote", Remote|Rest], State) -> +parse_args(["--remote", Remote | Rest], State) -> parse_args(Rest, State#{clone_url => find_upstream_repo(Remote)}); -parse_args(["--make-command", Command|Rest], State) -> +parse_args(["--make-command", Command | Rest], State) -> parse_args(Rest, State#{make_command => Command}); -parse_args(["--release-dir", Dir|Rest], State) -> +parse_args(["--release-dir", Dir | Rest], State) -> parse_args(Rest, State#{beams_dir => Dir}); -parse_args(["--prev-release-dir", Dir|Rest], State) -> +parse_args(["--prev-release-dir", Dir | Rest], State) -> parse_args(Rest, State#{prev_beams_dir => Dir}); -parse_args(["--src-dirs", Pattern|Rest], State) -> +parse_args(["--src-dirs", Pattern | Rest], State) -> parse_args(Rest, State#{src_dirs => Pattern}); parse_args(_, _) -> fail(usage()). @@ -87,9 +89,11 @@ parse_args(_, _) -> main(Options, Baseline) -> {CurrRelDir, PrevRelDir} = prepare(Baseline, Options), putopt(prev_beams_dir, PrevRelDir), - log("~n===================================~n" + log( + "~n===================================~n" "Processing changes..." - "~n===================================~n"), + "~n===================================~n" + ), CurrAppsIdx = index_apps(CurrRelDir), PrevAppsIdx = index_apps(PrevRelDir), %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]), @@ -98,6 +102,7 @@ main(Options, Baseline) -> ok = check_appup_files(), ok = warn_and_exit(is_valid()). +%% erlfmt-ignore warn_and_exit(true) -> log(" NOTE: Please review the changes manually. This script does not know about NIF @@ -109,9 +114,12 @@ warn_and_exit(false) -> halt(1). prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> - log("~n===================================~n" + log( + "~n===================================~n" "Baseline: ~s" - "~n===================================~n", [Baseline]), + "~n===================================~n", + [Baseline] + ), log("Building the current version...~n"), ok = bash(MakeCommand), PrevRelDir = @@ -126,6 +134,7 @@ prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir} end, {BeamDir, PrevRelDir}. +%% erlfmt-ignore build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> BaseDir = "/tmp/emqx-appup-base/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], @@ -146,24 +155,27 @@ find_upstream_repo(Remote) -> find_appup_actions(CurrApps, PrevApps) -> maps:fold( - fun(App, CurrAppIdx, Acc) -> - case PrevApps of - #{App := PrevAppIdx} -> - find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc; - _ -> - %% New app, nothing to upgrade here. - Acc - end - end, - [], - CurrApps). + fun(App, CurrAppIdx, Acc) -> + case PrevApps of + #{App := PrevAppIdx} -> + find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc; + _ -> + %% New app, nothing to upgrade here. + Acc + end + end, + [], + CurrApps + ). find_appup_actions(_App, AppIdx, AppIdx) -> %% No changes to the app, ignore: []; -find_appup_actions(App, - CurrAppIdx = #app{version = CurrVersion}, - PrevAppIdx = #app{version = PrevVersion}) -> +find_appup_actions( + App, + CurrAppIdx = #app{version = CurrVersion}, + PrevAppIdx = #app{version = PrevVersion} +) -> {OldUpgrade0, OldDowngrade0} = find_base_appup_actions(App, PrevVersion), OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0), OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), @@ -195,7 +207,10 @@ do_ensure_all_patch_versions(App, CurrVsn, OldActions) -> {ok, ExpectedVsns} -> CoveredVsns = [V || {V, _} <- OldActions, V =/= <<".*">>], ExpectedVsnStrs = [vsn_number_to_string(V) || V <- ExpectedVsns], - MissingActions = [{V, []} || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns)], + MissingActions = [ + {V, []} + || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns) + ], MissingActions ++ OldActions; {error, bad_version} -> log("WARN: Could not infer expected versions to upgrade from for ~p~n", [App]), @@ -206,23 +221,24 @@ do_ensure_all_patch_versions(App, CurrVsn, OldActions) -> %% in their current appup. diff_appup_instructions(ComputedChanges, PresentChanges) -> lists:foldr( - fun({VsnOrRegex, ComputedActions}, Acc) -> - case find_matching_version(VsnOrRegex, PresentChanges) of - undefined -> - [{VsnOrRegex, ComputedActions} | Acc]; - PresentActions -> - DiffActions = ComputedActions -- PresentActions, - case DiffActions of - [] -> - %% no diff - Acc; - _ -> - [{VsnOrRegex, DiffActions} | Acc] - end - end - end, - [], - ComputedChanges). + fun({VsnOrRegex, ComputedActions}, Acc) -> + case find_matching_version(VsnOrRegex, PresentChanges) of + undefined -> + [{VsnOrRegex, ComputedActions} | Acc]; + PresentActions -> + DiffActions = ComputedActions -- PresentActions, + case DiffActions of + [] -> + %% no diff + Acc; + _ -> + [{VsnOrRegex, DiffActions} | Acc] + end + end + end, + [], + ComputedChanges + ). %% checks if any missing diffs are present %% and groups them by `up' and `down' types. @@ -234,9 +250,10 @@ parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) -> %% no diff for external dependency; ignore ok; _ -> - Diffs = #{ up => DiffUp - , down => DiffDown - }, + Diffs = #{ + up => DiffUp, + down => DiffDown + }, {diffs, Diffs} end. @@ -260,18 +277,21 @@ find_base_appup_actions(App, PrevVersion) -> {ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}. merge_update_actions(App, Changes, Vsns, PrevVersion) -> - lists:map(fun(Ret = {<<".*">>, _}) -> - Ret; - ({Vsn, Actions}) -> - case is_skipped_version(App, Vsn, PrevVersion) of - true -> - log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), - {Vsn, Actions}; - false -> - {Vsn, do_merge_update_actions(App, Changes, Actions)} - end - end, - Vsns). + lists:map( + fun + (Ret = {<<".*">>, _}) -> + Ret; + ({Vsn, Actions}) -> + case is_skipped_version(App, Vsn, PrevVersion) of + true -> + log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), + {Vsn, Actions}; + false -> + {Vsn, do_merge_update_actions(App, Changes, Actions)} + end + end, + Vsns + ). %% say current version is 1.1.3, and the compare base is version 1.1.1, %% but there is a 1.1.2 in appup we may skip merging instructions for @@ -306,7 +326,7 @@ do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) -> []; false -> [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed] ++ - [{add_module, M} || M <- New] + [{add_module, M} || M <- New] end, {OldActionsWithStop, OldActionsAfterStop} = find_application_stop_instruction(App, OldActions), @@ -331,11 +351,14 @@ contains_restart_application(Application, Actions) -> find_application_stop_instruction(Application, Actions) -> {Before, After0} = lists:splitwith( - fun({apply, {application, stop, [App]}}) when App =:= Application -> - false; - (_) -> - true - end, Actions), + fun + ({apply, {application, stop, [App]}}) when App =:= Application -> + false; + (_) -> + true + end, + Actions + ), case After0 of [StopInst | After] -> {Before ++ [StopInst], After}; @@ -353,8 +376,10 @@ process_old_action({delete_module, Module}) -> [Module]; process_old_action({update, Module, _Change}) -> [Module]; -process_old_action(LoadModule) when is_tuple(LoadModule) andalso - element(1, LoadModule) =:= load_module -> +process_old_action(LoadModule) when + is_tuple(LoadModule) andalso + element(1, LoadModule) =:= load_module +-> element(2, LoadModule); process_old_action(_) -> []. @@ -370,17 +395,19 @@ ensure_version(Version, OldInstructions) -> contains_version(Needle, Haystack) when is_list(Needle) -> lists:any( - fun(Regex) when is_binary(Regex) -> - case re:run(Needle, Regex) of - {match, _} -> - true; - nomatch -> - false - end; - (Vsn) -> - Vsn =:= Needle - end, - Haystack). + fun + (Regex) when is_binary(Regex) -> + case re:run(Needle, Regex) of + {match, _} -> + true; + nomatch -> + false + end; + (Vsn) -> + Vsn =:= Needle + end, + Haystack + ). %% As a best effort approach, we assume that we only bump patch %% version numbers between release upgrades for our dependencies and @@ -413,9 +440,9 @@ vsn_number_to_string({Major, Minor, Patch}) -> read_appup(File) -> %% NOTE: appup file is a script, it may contain variables or functions. - case do_read_appup(File) of - {ok, {U, D}} -> {U, D}; - {error, Reason} -> fail("Failed to parse appup file ~p~n~p", [File, Reason]) + case do_read_appup(File) of + {ok, {U, D}} -> {U, D}; + {error, Reason} -> fail("Failed to parse appup file ~p~n~p", [File, Reason]) end. do_read_appup(File) -> @@ -434,10 +461,11 @@ check_appup_files() -> update_appups(Changes) -> lists:foreach( - fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> - do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) - end, - Changes). + fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> + do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) + end, + Changes + ). do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> case locate_current_src(App, ".appup.src") of @@ -469,8 +497,11 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> ok; {diffs, Diffs} -> set_invalid(), - log("ERROR: Appup file for '~p' is not complete.~n" - "Missing:~100p~n", [App, Diffs]), + log( + "ERROR: Appup file for '~p' is not complete.~n" + "Missing:~100p~n", + [App, Diffs] + ), notok end. @@ -496,9 +527,12 @@ render_appup(App, File, Up, Down) -> end. do_render_appup(File, Up, Down) -> - IOList = io_lib:format("%% -*- mode: erlang -*-~n" - "%% Unless you know what you are doing, DO NOT edit manually!!~n" - "{VSN,~n ~p,~n ~p}.~n", [Up, Down]), + IOList = io_lib:format( + "%% -*- mode: erlang -*-~n" + "%% Unless you know what you are doing, DO NOT edit manually!!~n" + "{VSN,~n ~p,~n ~p}.~n", + [Up, Down] + ), ok = file:write_file(File, IOList). create_stub(App) -> @@ -544,30 +578,37 @@ index_app(AppFile) -> %% Note: assuming that beams are always located in the same directory where app file is: EbinDir = filename:dirname(AppFile), Modules = hashsums(EbinDir), - {App, #app{ version = Vsn - , modules = Modules - }}. + {App, #app{ + version = Vsn, + modules = Modules + }}. -diff_app(UpOrDown, App, - #app{version = NewVersion, modules = NewModules}, - #app{version = OldVersion, modules = OldModules}) -> +diff_app( + UpOrDown, + App, + #app{version = NewVersion, modules = NewModules}, + #app{version = OldVersion, modules = OldModules} +) -> {New, Changed} = - maps:fold( fun(Mod, MD5, {New, Changed}) -> - case OldModules of - #{Mod := OldMD5} when MD5 =:= OldMD5 -> - {New, Changed}; - #{Mod := _} -> - {New, [Mod | Changed]}; - _ -> - {[Mod | New], Changed} - end - end - , {[], []} - , NewModules - ), + maps:fold( + fun(Mod, MD5, {New, Changed}) -> + case OldModules of + #{Mod := OldMD5} when MD5 =:= OldMD5 -> + {New, Changed}; + #{Mod := _} -> + {New, [Mod | Changed]}; + _ -> + {[Mod | New], Changed} + end + end, + {[], []}, + NewModules + ), Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)), - Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end, - [{added, New}, {changed, Changed}, {deleted, Deleted}]), + Changes = lists:filter( + fun({_T, L}) -> length(L) > 0 end, + [{added, New}, {changed, Changed}, {deleted, Deleted}] + ), case NewVersion =:= OldVersion of true when Changes =:= [] -> %% no change @@ -577,13 +618,17 @@ diff_app(UpOrDown, App, case UpOrDown =:= up of true -> %% only log for the upgrade case because it would be the same result - log("ERROR: Application '~p' contains changes, but its version is not updated. ~s", - [App, format_changes(Changes)]); + log( + "ERROR: Application '~p' contains changes, but its version is not updated. ~s", + [App, format_changes(Changes)] + ); false -> ok end; false -> - log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]), + log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [ + App, OldVersion, UpOrDown, NewVersion + ]), log("INFO: changes [~p]: ~p~n", [UpOrDown, Changes]), ok end, @@ -594,14 +639,16 @@ format_changes(Changes) -> -spec hashsums(file:filename()) -> #{module() => binary()}. hashsums(EbinDir) -> - maps:from_list(lists:map( - fun(Beam) -> - File = filename:join(EbinDir, Beam), - {ok, Ret = {_Module, _MD5}} = beam_lib:md5(File), - Ret - end, - filelib:wildcard("*.beam", EbinDir) - )). + maps:from_list( + lists:map( + fun(Beam) -> + File = filename:join(EbinDir, Beam), + {ok, Ret = {_Module, _MD5}} = beam_lib:md5(File), + Ret + end, + filelib:wildcard("*.beam", EbinDir) + ) + ). is_app_external(App) -> Ext = ".app.src", @@ -674,12 +721,13 @@ do_locate(Dir, App, Suffix) -> end. find_app(Pattern) -> - lists:filter(fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end, - filelib:wildcard(Pattern)). + lists:filter( + fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end, + filelib:wildcard(Pattern) + ). bash(undefined) -> ok; -bash(Script) -> - bash(Script, []). +bash(Script) -> bash(Script, []). bash(Script, Env) -> log("+ ~s~n+ Env: ~p~n", [Script, Env]), @@ -695,12 +743,14 @@ cmd(Exec, Params) -> fail("Executable not found in $PATH: ~s", [Exec]); Path -> Params1 = maps:to_list(maps:with([env, args, cd], Params)), - Port = erlang:open_port( {spawn_executable, Path} - , [ exit_status - , nouse_stdio - | Params1 - ] - ), + Port = erlang:open_port( + {spawn_executable, Path}, + [ + exit_status, + nouse_stdio + | Params1 + ] + ), receive {Port, {exit_status, Status}} -> Status