Merge pull request #6887 from emqx/mix-use-mix_env

chore(mix): use MIX_ENV to define build profile and edition

Instead of reading some environment variables to define the build profile for the Elixir build, we use the MIX_ENV value to emulate Rebar3's profiles. Also, that makes the build output directory more similar to EMQ X's current scheme.

.github/workflows/elixir_release.yml

@@ -23,12 +23,12 @@ jobs:
         run: make emqx-elixir
       - name: start release
         run: |
-          cd _build/prod/rel/emqx
+          cd _build/emqx/rel/emqx
           bin/emqx start
       - name: check if started
         run: |
           sleep 10
           nc -zv localhost 1883
-          cd _build/prod/rel/emqx
+          cd _build/emqx/rel/emqx
           bin/emqx ping
           bin/emqx ctl status

.github/workflows/run_fvt_tests.yaml

@@ -99,10 +99,13 @@ jobs:
       working-directory: source
       run: |
         set -x
-        IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
         if [[ "${{ matrix.profile }}" = *-elixir ]]
         then
           export IS_ELIXIR=yes
+          PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g")
+          IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir
+        else
+          IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
         fi
         ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
     - name: make paho tests

Makefile

@@ -229,9 +229,9 @@ $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_C
 define gen-elixirpkg-target
 # the Elixir places the tar in a different path than Rebar3
 $1-elixirpkg: $1-pkg-elixir
-	@env TAR_PKG_DIR=_build/prod \
+	@env TAR_PKG_DIR=_build/$1-pkg \
 	     IS_ELIXIR=yes \
-	     $(BUILD) $1 pkg
+	     $(BUILD) $1-pkg pkg
 endef
 $(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))
 

build

@@ -86,7 +86,7 @@ make_rel() {
 
 make_elixir_rel() {
   export_release_vars "$PROFILE"
-  env MIX_ENV=prod mix release --overwrite
+  mix release --overwrite
 }
 
 ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
@@ -138,7 +138,7 @@ make_tgz() {
       # ensure tarball exists
       ELIXIR_MAKE_TAR=yes make_elixir_rel
 
-      local relpath="_build/prod"
+      local relpath="_build/${PROFILE}"
       target="${pkgpath}/${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
     else
       # build the tarball again to ensure relup is included
@@ -170,12 +170,18 @@ make_tgz() {
 make_docker() {
     EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
     EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
+
+    if [[ "$PROFILE" = *-elixir ]]
+    then
+      PKG_VSN="$PKG_VSN-elixir"
+    fi
+
     set -x
     docker build --no-cache --pull \
        --build-arg BUILD_FROM="${EMQX_BUILDER}" \
        --build-arg RUN_FROM="${EMQX_RUNNER}" \
        --build-arg EMQX_NAME="$PROFILE" \
-       --tag "emqx/$PROFILE:${PKG_VSN}" \
+       --tag "emqx/${PROFILE%%-elixir}:${PKG_VSN}" \
        -f "${DOCKERFILE}" .
 }
 
@@ -227,46 +233,17 @@ make_docker_testing() {
 export_release_vars() {
   local profile="$1"
   case "$profile" in
-    emqx)
+    emqx|emqx-edge|emqx-enterprise)
-      export EMQX_RLEASE_TYPE=cloud \
+      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
-             EMQX_PACKAGE_TYPE=bin \
-             EMQX_EDITION_TYPE=community \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
       ;;
-    emqx-edge)
+    emqx-pkg|emqx-edge-pkg|emqx-enterprise-pkg)
-      export EMQX_RLEASE_TYPE=edge \
+      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
-             EMQX_PACKAGE_TYPE=bin \
-             EMQX_EDITION_TYPE=community \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
-      ;;
-    emqx-enterprise)
-      export EMQX_RLEASE_TYPE=cloud \
-             EMQX_PACKAGE_TYPE=bin \
-             EMQX_EDITION_TYPE=enterprise \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
-      ;;
-    emqx-pkg)
-      export EMQX_RLEASE_TYPE=cloud \
-             EMQX_PACKAGE_TYPE=pkg \
-             EMQX_EDITION_TYPE=community \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
-      ;;
-    emqx-edge-pkg)
-      export EMQX_RLEASE_TYPE=edge \
-             EMQX_PACKAGE_TYPE=pkg \
-             EMQX_EDITION_TYPE=community \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
-      ;;
-    emqx-enterprise-pkg)
-      export EMQX_RLEASE_TYPE=cloud \
-             EMQX_PACKAGE_TYPE=pkg \
-             EMQX_EDITION_TYPE=enterprise \
-             ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
       ;;
     *)
       echo Invalid profile "$profile"
       exit 1
   esac
+  export MIX_ENV="$profile"
 }
 
 log "building artifact=$ARTIFACT for profile=$PROFILE"

deploy/docker/Dockerfile

@@ -22,16 +22,13 @@ COPY . /emqx
 
 ARG EMQX_NAME=emqx
 
-RUN if [[ "$EMQX_NAME" = *-elixir ]]; then \
+RUN export PROFILE="$EMQX_NAME" \
-      export EMQX_LIB_PATH="_build/prod/lib"; \
+    && export EMQX_NAME=${EMQX_NAME%%-elixir} \
-      export EMQX_REL_PATH="/emqx/_build/prod/rel/emqx"; \
+    && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \
-    else \
+    && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
-      export EMQX_LIB_PATH="_build/$EMQX_NAME/lib"; \
-      export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx"; \
-    fi \
     && cd /emqx \
     && rm -rf $EMQX_LIB_PATH \
-    && make $EMQX_NAME \
+    && make $PROFILE \
     && mkdir -p /emqx-rel \
     && mv $EMQX_REL_PATH /emqx-rel
 

mix.exs

@@ -8,23 +8,32 @@ defmodule EMQXUmbrella.MixProject do
   procedures, one cannot simply use `iex -S mix`.  Instead, it's
   recommendd to build and use the release.
 
+  ## Profiles
+
+  To control the profile and edition to build, we case split on the
+  MIX_ENV value.
+
+  The following profiles are valid:
+
+    * `emqx`
+    * `emqx-edge`
+    * `emqx-enterprise`
+    * `emqx-pkg`
+    * `emqx-edge-pkg`
+    * `emqx-enterprise-pkg`
+    * `dev` -> same as `emqx`, for convenience
+
   ## Release Environment Variables
 
   The release build is controlled by a few environment variables.
 
     * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
       tarball along with the release.
-    * `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`.  Controls a
-      few dependencies and the `vm.args` to be used.  Defaults to
-      `cloud`.
-    * `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`.  Controls
-      whether the build is intended for direct usage or for packaging.
-      Defaults to `bin`.
-    * `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`.
-      Defaults to `community`.
   """
 
   def project() do
+    check_profile!()
+
     [
       app: :emqx_mix,
       version: pkg_vsn(),
@@ -105,7 +114,7 @@ defmodule EMQXUmbrella.MixProject do
           release_type: release_type,
           package_type: package_type,
           edition_type: edition_type
-        } = read_inputs()
+        } = check_profile!()
 
         base_steps = [
           :assemble,
@@ -197,27 +206,58 @@ defmodule EMQXUmbrella.MixProject do
       )
   end
 
-  defp read_inputs() do
+  def check_profile!() do
-    release_type =
+    valid_envs = [
-      read_enum_env_var(
+      :dev,
-        "EMQX_RELEASE_TYPE",
+      :emqx,
-        [:cloud, :edge],
+      :"emqx-pkg",
-        :cloud
+      :"emqx-enterprise",
-      )
+      :"emqx-enterprise-pkg",
+      :"emqx-edge",
+      :"emqx-edge-pkg"
+    ]
 
-    package_type =
+    if Mix.env() not in valid_envs do
-      read_enum_env_var(
+      formatted_envs =
-        "EMQX_PACKAGE_TYPE",
+        valid_envs
-        [:bin, :pkg],
+        |> Enum.map(&"  * #{&1}")
-        :bin
+        |> Enum.join("\n")
-      )
 
-    edition_type =
+      Mix.raise("""
-      read_enum_env_var(
+      Invalid env #{Mix.env()}.  Valid options are:
-        "EMQX_EDITION_TYPE",
+      #{formatted_envs}
-        [:community, :enterprise],
+      """)
-        :community
+    end
-      )
+
+    {
+      release_type,
+      package_type,
+      edition_type
+    } =
+      case Mix.env() do
+        :dev ->
+          {:cloud, :bin, :community}
+
+        :emqx ->
+          {:cloud, :bin, :community}
+
+        :"emqx-edge" ->
+          {:edge, :bin, :community}
+
+        :"emqx-enterprise" ->
+          {:cloud, :bin, :enterprise}
+
+        :"emqx-pkg" ->
+          {:cloud, :pkg, :community}
+
+        :"emqx-edge-pkg" ->
+          {:edge, :pkg, :community}
+
+        :"emqx-enterprise-pkg" ->
+          {:cloud, :pkg, :enterprise}
+      end
+
+    normalize_env!()
 
     %{
       release_type: release_type,
@@ -477,28 +517,6 @@ defmodule EMQXUmbrella.MixProject do
     ]
   end
 
-  defp read_enum_env_var(env_var, allowed_values, default_value) do
-    case System.fetch_env(env_var) do
-      :error ->
-        default_value
-
-      {:ok, raw_value} ->
-        value =
-          raw_value
-          |> String.downcase()
-          |> String.to_atom()
-
-        if value not in allowed_values do
-          Mix.raise("""
-          Invalid value #{raw_value} for variable #{env_var}.
-          Allowed values are: #{inspect(allowed_values)}
-          """)
-        end
-
-        value
-    end
-  end
-
   defp emqx_description(release_type, edition_type) do
     case {release_type, edition_type} do
       {:cloud, :enterprise} ->
@@ -538,7 +556,7 @@ defmodule EMQXUmbrella.MixProject do
   end
 
   defp pkg_vsn() do
-    %{edition_type: edition_type} = read_inputs()
+    %{edition_type: edition_type} = check_profile!()
     basedir = Path.dirname(__ENV__.file)
     script = Path.join(basedir, "pkg-vsn.sh")
     {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)])
@@ -597,6 +615,19 @@ defmodule EMQXUmbrella.MixProject do
     to_string(8 * size)
   end
 
+  defp normalize_env!() do
+    env =
+      case Mix.env() do
+        :dev ->
+          :emqx
+
+        env ->
+          env
+      end
+
+    Mix.env(env)
+  end
+
   # As from Erlang/OTP 17, the OTP release number corresponds to the
   # major OTP version number. No erlang:system_info() argument gives
   # the exact OTP version.

rebar.config.erl

@@ -186,6 +186,7 @@ profiles_dev() ->
 
 %% RelType: cloud (full size) | edge (slim size)
 %% PkgType: bin | pkg
+%% Edition: ce (community) | ee (enterprise)
 relx(Vsn, RelType, PkgType, Edition) ->
     [ {include_src,false}
     , {include_erts, true}

scripts/check-elixir-applications.exs

@@ -1,13 +1,20 @@
 #!/usr/bin/env elixir
 
 defmodule CheckElixirApplications do
+  alias EMQXUmbrella.MixProject
+
   @default_applications [:kernel, :stdlib, :sasl]
 
   def main() do
     {:ok, _} = Application.ensure_all_started(:mix)
-    inputs = read_inputs()
+
+    File.cwd!()
+    |> Path.join("mix.exs")
+    |> Code.compile_file()
+
+    inputs = MixProject.check_profile!()
+    profile = Mix.env()
     # produce `rebar.config.rendered` to consult
-    profile = profile_of(inputs)
 
     File.cwd!()
     |> Path.join("rebar3")
@@ -15,10 +22,6 @@ defmodule CheckElixirApplications do
       env: [{"DEBUG", "1"}]
     )
 
-    File.cwd!()
-    |> Path.join("mix.exs")
-    |> Code.compile_file()
-
     mix_apps = mix_applications(inputs.release_type)
     rebar_apps = rebar_applications(profile)
     results = diff_apps(mix_apps, rebar_apps)
@@ -93,83 +96,6 @@ defmodule CheckElixirApplications do
     end)
   end
 
-  defp profile_of(%{
-         release_type: release_type,
-         package_type: package_type,
-         edition_type: edition_type
-       }) do
-    case {release_type, package_type, edition_type} do
-      {:cloud, :bin, :community} ->
-        :emqx
-
-      {:cloud, :pkg, :community} ->
-        :"emqx-pkg"
-
-      {:cloud, :bin, :enterprise} ->
-        :"emqx-enterprise"
-
-      {:cloud, :pkg, :enterprise} ->
-        :"emqx-enterprise-pkg"
-
-      {:edge, :bin, :community} ->
-        :"emqx-edge"
-
-      {:edge, :pkg, :community} ->
-        :"emqx-edge-pkg"
-    end
-  end
-
-  defp read_inputs() do
-    release_type =
-      read_enum_env_var(
-        "EMQX_RELEASE_TYPE",
-        [:cloud, :edge],
-        :cloud
-      )
-
-    package_type =
-      read_enum_env_var(
-        "EMQX_PACKAGE_TYPE",
-        [:bin, :pkg],
-        :bin
-      )
-
-    edition_type =
-      read_enum_env_var(
-        "EMQX_EDITION_TYPE",
-        [:community, :enterprise],
-        :community
-      )
-
-    %{
-      release_type: release_type,
-      package_type: package_type,
-      edition_type: edition_type
-    }
-  end
-
-  defp read_enum_env_var(env_var, allowed_values, default_value) do
-    case System.fetch_env(env_var) do
-      :error ->
-        default_value
-
-      {:ok, raw_value} ->
-        value =
-          raw_value
-          |> String.downcase()
-          |> String.to_atom()
-
-        if value not in allowed_values do
-          Mix.raise("""
-          Invalid value #{raw_value} for variable #{env_var}.
-          Allowed values are: #{inspect(allowed_values)}
-          """)
-        end
-
-        value
-    end
-  end
-
   defp diff_apps(mix_apps, rebar_apps) do
     app_names = Keyword.keys(rebar_apps)
     mix_apps = Keyword.filter(mix_apps, fn {app, _mode} -> app in app_names end)