From a597e9257632d4aa9fd673b7459736b05c8d1717 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Jan 2022 14:57:09 -0300 Subject: [PATCH] chore(mix): use MIX_ENV to define build profile and edition --- .github/workflows/elixir_release.yml | 4 +- .github/workflows/run_fvt_tests.yaml | 5 +- Makefile | 4 +- build | 51 +++------- deploy/docker/Dockerfile | 13 +-- mix.exs | 133 ++++++++++++++++---------- rebar.config.erl | 1 + scripts/check-elixir-applications.exs | 92 ++---------------- 8 files changed, 119 insertions(+), 184 deletions(-) diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 0fa97884d..a3d5a8814 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -23,12 +23,12 @@ jobs: run: make emqx-elixir - name: start release run: | - cd _build/prod/rel/emqx + cd _build/emqx/rel/emqx bin/emqx start - name: check if started run: | sleep 10 nc -zv localhost 1883 - cd _build/prod/rel/emqx + cd _build/emqx/rel/emqx bin/emqx ping bin/emqx ctl status diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 9570b68b3..bdab602b8 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -97,10 +97,13 @@ jobs: working-directory: source run: | set -x - IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) if [[ "${{ matrix.profile }}" = *-elixir ]] then export IS_ELIXIR=yes + PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g") + IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir + else + IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) fi ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - name: make paho tests diff --git a/Makefile b/Makefile index d26a09e1b..0d54c3273 100644 --- a/Makefile +++ b/Makefile @@ -229,9 +229,9 @@ $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_C define gen-elixirpkg-target # the Elixir places the tar in a different path than Rebar3 $1-elixirpkg: $1-pkg-elixir - @env TAR_PKG_DIR=_build/prod \ + @env TAR_PKG_DIR=_build/$1-pkg \ IS_ELIXIR=yes \ - $(BUILD) $1 pkg + $(BUILD) $1-pkg pkg endef $(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) diff --git a/build b/build index d5aeeb64f..e66df1f17 100755 --- a/build +++ b/build @@ -86,7 +86,7 @@ make_rel() { make_elixir_rel() { export_release_vars "$PROFILE" - env MIX_ENV=prod mix release --overwrite + mix release --overwrite } ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup @@ -138,7 +138,7 @@ make_tgz() { # ensure tarball exists ELIXIR_MAKE_TAR=yes make_elixir_rel - local relpath="_build/prod" + local relpath="_build/${PROFILE}" target="${pkgpath}/${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" else # build the tarball again to ensure relup is included @@ -170,12 +170,18 @@ make_tgz() { make_docker() { EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" + + if [[ "$PROFILE" = *-elixir ]] + then + PKG_VSN="$PKG_VSN-elixir" + fi + set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="${EMQX_RUNNER}" \ --build-arg EMQX_NAME="$PROFILE" \ - --tag "emqx/$PROFILE:${PKG_VSN}" \ + --tag "emqx/${PROFILE%%-elixir}:${PKG_VSN}" \ -f "${DOCKERFILE}" . } @@ -227,46 +233,17 @@ make_docker_testing() { export_release_vars() { local profile="$1" case "$profile" in - emqx) - export EMQX_RLEASE_TYPE=cloud \ - EMQX_PACKAGE_TYPE=bin \ - EMQX_EDITION_TYPE=community \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} + emqx|emqx-edge|emqx-enterprise) + export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} ;; - emqx-edge) - export EMQX_RLEASE_TYPE=edge \ - EMQX_PACKAGE_TYPE=bin \ - EMQX_EDITION_TYPE=community \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} - ;; - emqx-enterprise) - export EMQX_RLEASE_TYPE=cloud \ - EMQX_PACKAGE_TYPE=bin \ - EMQX_EDITION_TYPE=enterprise \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no} - ;; - emqx-pkg) - export EMQX_RLEASE_TYPE=cloud \ - EMQX_PACKAGE_TYPE=pkg \ - EMQX_EDITION_TYPE=community \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} - ;; - emqx-edge-pkg) - export EMQX_RLEASE_TYPE=edge \ - EMQX_PACKAGE_TYPE=pkg \ - EMQX_EDITION_TYPE=community \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} - ;; - emqx-enterprise-pkg) - export EMQX_RLEASE_TYPE=cloud \ - EMQX_PACKAGE_TYPE=pkg \ - EMQX_EDITION_TYPE=enterprise \ - ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} + emqx-pkg|emqx-edge-pkg|emqx-enterprise-pkg) + export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes} ;; *) echo Invalid profile "$profile" exit 1 esac + export MIX_ENV="$profile" } log "building artifact=$ARTIFACT for profile=$PROFILE" diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 68463fa5e..133e1a9af 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -22,16 +22,13 @@ COPY . /emqx ARG EMQX_NAME=emqx -RUN if [[ "$EMQX_NAME" = *-elixir ]]; then \ - export EMQX_LIB_PATH="_build/prod/lib"; \ - export EMQX_REL_PATH="/emqx/_build/prod/rel/emqx"; \ - else \ - export EMQX_LIB_PATH="_build/$EMQX_NAME/lib"; \ - export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx"; \ - fi \ +RUN export PROFILE="$EMQX_NAME" \ + && export EMQX_NAME=${EMQX_NAME%%-elixir} \ + && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ + && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ - && make $EMQX_NAME \ + && make $PROFILE \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel diff --git a/mix.exs b/mix.exs index dfab72031..16611cb04 100644 --- a/mix.exs +++ b/mix.exs @@ -8,23 +8,32 @@ defmodule EMQXUmbrella.MixProject do procedures, one cannot simply use `iex -S mix`. Instead, it's recommendd to build and use the release. + ## Profiles + + To control the profile and edition to build, we case split on the + MIX_ENV value. + + The following profiles are valid: + + * `emqx` + * `emqx-edge` + * `emqx-enterprise` + * `emqx-pkg` + * `emqx-edge-pkg` + * `emqx-enterprise-pkg` + * `dev` -> same as `emqx`, for convenience + ## Release Environment Variables The release build is controlled by a few environment variables. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz` tarball along with the release. - * `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`. Controls a - few dependencies and the `vm.args` to be used. Defaults to - `cloud`. - * `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`. Controls - whether the build is intended for direct usage or for packaging. - Defaults to `bin`. - * `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`. - Defaults to `community`. """ def project() do + check_profile!() + [ app: :emqx_mix, version: pkg_vsn(), @@ -105,7 +114,7 @@ defmodule EMQXUmbrella.MixProject do release_type: release_type, package_type: package_type, edition_type: edition_type - } = read_inputs() + } = check_profile!() base_steps = [ :assemble, @@ -197,27 +206,58 @@ defmodule EMQXUmbrella.MixProject do ) end - defp read_inputs() do - release_type = - read_enum_env_var( - "EMQX_RELEASE_TYPE", - [:cloud, :edge], - :cloud - ) + def check_profile!() do + valid_envs = [ + :dev, + :emqx, + :"emqx-pkg", + :"emqx-enterprise", + :"emqx-enterprise-pkg", + :"emqx-edge", + :"emqx-edge-pkg" + ] - package_type = - read_enum_env_var( - "EMQX_PACKAGE_TYPE", - [:bin, :pkg], - :bin - ) + if Mix.env() not in valid_envs do + formatted_envs = + valid_envs + |> Enum.map(&" * #{&1}") + |> Enum.join("\n") - edition_type = - read_enum_env_var( - "EMQX_EDITION_TYPE", - [:community, :enterprise], - :community - ) + Mix.raise(""" + Invalid env #{Mix.env()}. Valid options are: + #{formatted_envs} + """) + end + + { + release_type, + package_type, + edition_type + } = + case Mix.env() do + :dev -> + {:cloud, :bin, :community} + + :emqx -> + {:cloud, :bin, :community} + + :"emqx-edge" -> + {:edge, :bin, :community} + + :"emqx-enterprise" -> + {:cloud, :bin, :enterprise} + + :"emqx-pkg" -> + {:cloud, :pkg, :community} + + :"emqx-edge-pkg" -> + {:edge, :pkg, :community} + + :"emqx-enterprise-pkg" -> + {:cloud, :pkg, :enterprise} + end + + normalize_env!() %{ release_type: release_type, @@ -477,28 +517,6 @@ defmodule EMQXUmbrella.MixProject do ] end - defp read_enum_env_var(env_var, allowed_values, default_value) do - case System.fetch_env(env_var) do - :error -> - default_value - - {:ok, raw_value} -> - value = - raw_value - |> String.downcase() - |> String.to_atom() - - if value not in allowed_values do - Mix.raise(""" - Invalid value #{raw_value} for variable #{env_var}. - Allowed values are: #{inspect(allowed_values)} - """) - end - - value - end - end - defp emqx_description(release_type, edition_type) do case {release_type, edition_type} do {:cloud, :enterprise} -> @@ -538,7 +556,7 @@ defmodule EMQXUmbrella.MixProject do end defp pkg_vsn() do - %{edition_type: edition_type} = read_inputs() + %{edition_type: edition_type} = check_profile!() basedir = Path.dirname(__ENV__.file) script = Path.join(basedir, "pkg-vsn.sh") {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)]) @@ -597,6 +615,19 @@ defmodule EMQXUmbrella.MixProject do to_string(8 * size) end + defp normalize_env!() do + env = + case Mix.env() do + :dev -> + :emqx + + env -> + env + end + + Mix.env(env) + end + # As from Erlang/OTP 17, the OTP release number corresponds to the # major OTP version number. No erlang:system_info() argument gives # the exact OTP version. diff --git a/rebar.config.erl b/rebar.config.erl index ca3889f5b..7a0d5e26f 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -186,6 +186,7 @@ profiles_dev() -> %% RelType: cloud (full size) | edge (slim size) %% PkgType: bin | pkg +%% Edition: ce (community) | ee (enterprise) relx(Vsn, RelType, PkgType, Edition) -> [ {include_src,false} , {include_erts, true} diff --git a/scripts/check-elixir-applications.exs b/scripts/check-elixir-applications.exs index 3ada6d779..88602b3f5 100755 --- a/scripts/check-elixir-applications.exs +++ b/scripts/check-elixir-applications.exs @@ -1,13 +1,20 @@ #!/usr/bin/env elixir defmodule CheckElixirApplications do + alias EMQXUmbrella.MixProject + @default_applications [:kernel, :stdlib, :sasl] def main() do {:ok, _} = Application.ensure_all_started(:mix) - inputs = read_inputs() + + File.cwd!() + |> Path.join("mix.exs") + |> Code.compile_file() + + inputs = MixProject.check_profile!() + profile = Mix.env() # produce `rebar.config.rendered` to consult - profile = profile_of(inputs) File.cwd!() |> Path.join("rebar3") @@ -15,10 +22,6 @@ defmodule CheckElixirApplications do env: [{"DEBUG", "1"}] ) - File.cwd!() - |> Path.join("mix.exs") - |> Code.compile_file() - mix_apps = mix_applications(inputs.release_type) rebar_apps = rebar_applications(profile) results = diff_apps(mix_apps, rebar_apps) @@ -93,83 +96,6 @@ defmodule CheckElixirApplications do end) end - defp profile_of(%{ - release_type: release_type, - package_type: package_type, - edition_type: edition_type - }) do - case {release_type, package_type, edition_type} do - {:cloud, :bin, :community} -> - :emqx - - {:cloud, :pkg, :community} -> - :"emqx-pkg" - - {:cloud, :bin, :enterprise} -> - :"emqx-enterprise" - - {:cloud, :pkg, :enterprise} -> - :"emqx-enterprise-pkg" - - {:edge, :bin, :community} -> - :"emqx-edge" - - {:edge, :pkg, :community} -> - :"emqx-edge-pkg" - end - end - - defp read_inputs() do - release_type = - read_enum_env_var( - "EMQX_RELEASE_TYPE", - [:cloud, :edge], - :cloud - ) - - package_type = - read_enum_env_var( - "EMQX_PACKAGE_TYPE", - [:bin, :pkg], - :bin - ) - - edition_type = - read_enum_env_var( - "EMQX_EDITION_TYPE", - [:community, :enterprise], - :community - ) - - %{ - release_type: release_type, - package_type: package_type, - edition_type: edition_type - } - end - - defp read_enum_env_var(env_var, allowed_values, default_value) do - case System.fetch_env(env_var) do - :error -> - default_value - - {:ok, raw_value} -> - value = - raw_value - |> String.downcase() - |> String.to_atom() - - if value not in allowed_values do - Mix.raise(""" - Invalid value #{raw_value} for variable #{env_var}. - Allowed values are: #{inspect(allowed_values)} - """) - end - - value - end - end - defp diff_apps(mix_apps, rebar_apps) do app_names = Keyword.keys(rebar_apps) mix_apps = Keyword.filter(mix_apps, fn {app, _mode} -> app in app_names end)