Merge pull request #7186 from zmstone/structured-build-info

build: more structured build info

.github/workflows/build_slim_packages.yaml

@@ -127,9 +127,9 @@ jobs:
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
         echo "EMQX stopped"
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
-        echo "EQMX installed"
+        echo "EMQX installed"
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
-        echo "EQMX uninstaled"
+        echo "EMQX uninstalled"
 
   mac:
     strategy:

Makefile

@@ -10,6 +10,7 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v0.21.0
 export DOCKERFILE := deploy/docker/Dockerfile
+export EMQX_REL_FORM ?= tgz
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 	FIND=/usr/bin/find
@@ -192,7 +193,7 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-tgz-target,$(zt))))
 ## A pkg target depend on a regular release
 .PHONY: $(PKG_PROFILES)
 define gen-pkg-target
-$1: $1-rel
+$1:
 	@$(BUILD) $1 pkg
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
@@ -225,7 +226,7 @@ $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_C
 .PHONY: $(REL_PROFILES:%=%-elixir-pkg)
 define gen-elixir-pkg-target
 # the Elixir places the tar in a different path than Rebar3
-$1-elixir-pkg: $1-pkg-elixir
+$1-elixir-pkg:
 	@env TAR_PKG_DIR=_build/$1-pkg \
 	     IS_ELIXIR=yes \
 	     $(BUILD) $1-pkg pkg

apps/emqx/src/emqx_vm.erl

@@ -368,15 +368,16 @@ compat_windows(Fun) ->
 %% @doc Return on which Eralng/OTP the current vm is running.
 %% NOTE: This API reads a file, do not use it in critical code paths.
 get_otp_version() ->
-    parse_built_on(read_otp_version()).
+    read_otp_version().
 
 read_otp_version() ->
     ReleasesDir = filename:join([code:root_dir(), "releases"]),
-    Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILT_ON"]),
+    Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILD_INFO"]),
     case file:read_file(Filename) of
-        {ok, BuiltOn} ->
-            %% running on EQM X release
-            BuiltOn;
+        {ok, BuildInfo} ->
+            %% running on EMQX release
+            {ok, Fields} = hocon:binary(BuildInfo),
+            hocon_maps:get("erlang", Fields);
         {error, enoent} ->
             %% running tests etc.
             OtpMajor = erlang:system_info(otp_release),
@@ -384,11 +385,3 @@ read_otp_version() ->
             {ok, Vsn} = file:read_file(OtpVsnFile),
             Vsn
     end.
-
-parse_built_on(BuiltOn) ->
-    case binary:split(BuiltOn, <<"-">>, [global]) of
-        [Vsn, <<"emqx">>, N | _] ->
-            binary_to_list(Vsn) ++ "-emqx-" ++ binary_to_list(N);
-        [Vsn | _] ->
-            string:trim(binary_to_list(Vsn))
-    end.

apps/emqx_management/src/emqx_mgmt.erl

@@ -146,7 +146,7 @@ node_info() ->
     Info = maps:from_list([{K, list_to_binary(V)} || {K, V} <- emqx_vm:loads()]),
     BrokerInfo = emqx_sys:info(),
     Info#{node              => node(),
-          otp_release       => iolist_to_binary(otp_rel()),
+          otp_release       => otp_rel(),
           memory_total      => proplists:get_value(allocated, Memory),
           memory_used       => proplists:get_value(used, Memory),
           process_available => erlang:system_info(process_limit),
@@ -179,7 +179,7 @@ lookup_broker(Node) ->
 
 broker_info() ->
     Info = maps:from_list([{K, iolist_to_binary(V)} || {K, V} <- emqx_sys:info()]),
-    Info#{node => node(), otp_release => iolist_to_binary(otp_rel()), node_status => 'Running'}.
+    Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}.
 
 broker_info(Node) ->
     wrap_rpc(emqx_management_proto_v1:broker_info(Node)).
@@ -574,7 +574,7 @@ wrap_rpc(Res) ->
     Res.
 
 otp_rel() ->
-    lists:concat([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]).
+    iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]).
 
 check_row_limit(Tables) ->
     check_row_limit(Tables, max_row_limit()).

bin/emqx

@@ -219,7 +219,7 @@ if [ "${2:-}" = 'help' ]; then
 fi
 
 if ! check_erlang_start >/dev/null 2>&1; then
-    BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")"
+    BUILD_INFO="$(cat "${REL_DIR}/BUILD_INFO")"
     ## failed to start, might be due to missing libs, try to be portable
     export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-$DYNLIBS_DIR}"
     if [ "$LD_LIBRARY_PATH" != "$DYNLIBS_DIR" ]; then
@@ -229,8 +229,8 @@ if ! check_erlang_start >/dev/null 2>&1; then
         ## it's hopeless
         echoerr "FATAL: Unable to start Erlang."
         echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed."
-        echoerr "Also ensure it's running on the correct platform,"
-        echoerr "this EMQX release is built for $BUILT_ON"
+        echoerr "Also ensure it's running on the correct platform:"
+        echoerr "$BUILD_INFO"
         exit 1
     fi
     echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}"

build

@@ -284,10 +284,17 @@ case "$ARTIFACT" in
             log "Skipped making deb/rpm package for $SYSTEM"
             exit 0
         fi
+        export EMQX_REL_FORM="$PKGERDIR"
+        if [ "${IS_ELIXIR:-}" = 'yes' ]; then
+            make_elixir_rel
+        else
+            make_rel
+        fi
+        env EMQX_REL="$(pwd)" \
+            EMQX_BUILD="${PROFILE}" \
             make -C "deploy/packages/${PKGERDIR}" clean
         env EMQX_REL="$(pwd)" \
             EMQX_BUILD="${PROFILE}" \
-            SYSTEM="${SYSTEM}" \
             make -C "deploy/packages/${PKGERDIR}"
         ;;
     docker)

deploy/docker/Dockerfile

@@ -27,6 +27,7 @@ RUN export PROFILE="$EMQX_NAME" \
     && export EMQX_NAME=${EMQX_NAME%%-elixir} \
     && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \
     && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
+    && export EMQX_REL_FORM='docker' \
     && cd /emqx \
     && rm -rf $EMQX_LIB_PATH \
     && make $PROFILE \

deploy/packages/deb/Makefile

@@ -1,4 +1,3 @@
-ARCH ?= amd64
 TOPDIR := /tmp/emqx
 # Keep this short to avoid bloating beam files with long file path info
 SRCDIR := $(TOPDIR)/$(PKG_VSN)

deploy/packages/rpm/Makefile

@@ -9,11 +9,6 @@ space := $(none) $(none)
 RPM_VSN := $(subst -,_,$(PKG_VSN))
 RPM_REL := otp$(subst -,_,$(OTP_VSN))
 
-ARCH ?= amd64
-ifeq ($(ARCH),mips64)
-ARCH:=mips64el
-endif
-
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx

lib-ee/emqx_license/src/emqx_license_schema.erl

@@ -21,7 +21,7 @@ roots() -> [{license,
 A license is either a `key` or a `file`.
 When `key` and `file` are both configured, `key` is used.
 
-EQMX by default starts with a trial license.  For a different license,
+EMQX by default starts with a trial license.  For a different license,
 visit https://www.emqx.com/apply-licenses/emqx to apply.
 "})}
            ].

mix.exs

@@ -426,9 +426,9 @@ defmodule EMQXUmbrella.MixProject do
     File.chmod!(Path.join(bin, "node_dump"), 0o755)
 
     render_template(
-      "rel/BUILT_ON",
+      "rel/BUILD_INFO",
       assigns,
-      Path.join(release.version_path, "BUILT_ON")
+      Path.join(release.version_path, "BUILD_INFO")
     )
 
     release
@@ -550,10 +550,9 @@ defmodule EMQXUmbrella.MixProject do
       emqx_description: emqx_description(release_type, edition_type),
       emqx_schema_mod: emqx_schema_mod(edition_type),
       emqx_machine_boot_apps: emqx_machine_boot_app_list(edition_type),
-      built_on_arch: built_on(),
       is_elixir: "yes",
       is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
-    ]
+    ] ++ build_info()
   end
 
   defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
@@ -575,24 +574,23 @@ defmodule EMQXUmbrella.MixProject do
       # FIXME: this is empty in `make emqx` ???
       erl_opts: "",
       emqx_description: emqx_description(release_type, edition_type),
-      built_on_arch: built_on(),
       emqx_schema_mod: emqx_schema_mod(edition_type),
       emqx_machine_boot_apps: emqx_machine_boot_app_list(edition_type),
       is_elixir: "yes",
       is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
-    ]
+    ] ++ build_info()
   end
 
   defp emqx_description(release_type, edition_type) do
     case {release_type, edition_type} do
       {:cloud, :enterprise} ->
-        "EMQX Enterprise Edition"
+        "EMQX Enterprise"
 
       {:cloud, :community} ->
-        "EMQX Community Edition"
+        "EMQX"
 
       {:edge, :community} ->
-        "EMQX Edge Edition"
+        "EMQX Edge"
     end
   end
 
@@ -628,9 +626,12 @@ defmodule EMQXUmbrella.MixProject do
     %{edition_type: edition_type} = check_profile!()
     basedir = Path.dirname(__ENV__.file)
     script = Path.join(basedir, "pkg-vsn.sh")
-    {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)])
+    os_cmd(script, [Atom.to_string(edition_type)])
+  end
 
-    String.trim(str_vsn)
+  defp os_cmd(script, args) do
+    {str, 0} = System.cmd("bash", [script | args])
+    String.trim(str)
   end
 
   defp win32?(),
@@ -663,12 +664,15 @@ defmodule EMQXUmbrella.MixProject do
     )
   end
 
-  defp built_on() do
-    system_architecture = to_string(:erlang.system_info(:system_architecture))
-    elixir_version = System.version()
-    words = wordsize()
-
-    "#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}"
+  defp build_info() do
+    [
+      build_info_arch: to_string(:erlang.system_info(:system_architecture)),
+      build_info_wordsize: wordsize(),
+      build_info_os: os_cmd("./scripts/get-distro.sh", []),
+      build_info_erlang: otp_release(),
+      build_info_elixir: System.version(),
+      build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
+    ]
   end
 
   # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142

rebar.config.erl

@@ -212,11 +212,29 @@ relx(Vsn, RelType, PkgType, Edition) ->
     , {vm_args,false}
     , {release, {emqx, Vsn}, relx_apps(RelType, Edition)}
     , {overlay, relx_overlay(RelType, Edition)}
-    , {overlay_vars, [ {built_on_arch, rebar_utils:get_arch()}
-                     , {emqx_description, emqx_description(RelType, Edition)}
-                     | overlay_vars(RelType, PkgType, Edition)]}
+    , {overlay_vars, build_info() ++
+                     [ {emqx_description, emqx_description(RelType, Edition)}
+                     | overlay_vars(RelType, PkgType, Edition)
+                     ]}
     ].
 
+%% Make a HOCON compatible format
+build_info() ->
+    Os = os_cmd("./scripts/get-distro.sh"),
+    [ {build_info_arch, erlang:system_info(system_architecture)}
+    , {build_info_wordsize, rebar_utils:wordsize()}
+    , {build_info_os, Os}
+    , {build_info_erlang, rebar_utils:otp_release()}
+    , {build_info_elixir, none}
+    , {build_info_relform, relform()}
+    ].
+
+relform() ->
+    case os:getenv("EMQX_REL_FORM") of
+        false -> "tgz";
+        Other -> Other
+    end.
+
 emqx_description(cloud, ee) -> "EMQX Enterprise";
 emqx_description(cloud, ce) -> "EMQX";
 emqx_description(edge, ce)  -> "EMQX Edge".
@@ -394,7 +412,7 @@ relx_overlay(ReleaseType, Edition) ->
     , {mkdir, "data/patches"}
     , {mkdir, "data/scripts"}
     , {template, "rel/emqx_vars", "releases/emqx_vars"}
-    , {template, "rel/BUILT_ON", "releases/{{release_version}}/BUILT_ON"}
+    , {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"}
     , {copy, "bin/emqx", "bin/emqx"}
     , {copy, "bin/emqx_ctl", "bin/emqx_ctl"}
     , {copy, "bin/node_dump", "bin/node_dump"}
@@ -448,8 +466,11 @@ get_vsn(Profile) ->
     %% to make it compatible to Linux and Windows,
     %% we must use bash to execute the bash file
     %% because "./" will not be recognized as an internal or external command
-    PkgVsn = os:cmd("bash pkg-vsn.sh " ++ atom_to_list(Profile)),
-    re:replace(PkgVsn, "\n", "", [{return ,list}]).
+    os_cmd("pkg-vsn.sh " ++ atom_to_list(Profile)).
+
+os_cmd(Cmd) ->
+    Output = os:cmd("bash " ++ Cmd),
+    re:replace(Output, "\n", "", [{return ,list}]).
 
 maybe_dump(Config) ->
     is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),

rel/BUILD_INFO

@@ -0,0 +1,6 @@
+arch: "{{ build_info_arch }}"
+wordsize: {{ build_info_wordsize }}
+os: "{{ build_info_os }}"
+erlang: "{{ build_info_erlang }}"
+elixir: "{{ build_info_elixir }}"
+relform: "{{ build_info_relform }}"

rel/BUILT_ON

@@ -1 +0,0 @@
-{{ built_on_arch }}