defmodule EMQXUmbrella.MixProject do use Mix.Project @moduledoc """ The purpose of this file is to configure the release of EMQX under Mix. Since EMQX uses its own configuration conventions and startup procedures, one cannot simply use `iex -S mix`. Instead, it's recommendd to build and use the release. ## Release Environment Variables The release build is controlled by a few environment variables. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz` tarball along with the release. * `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`. Controls a few dependencies and the `vm.args` to be used. Defaults to `cloud`. * `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`. Controls whether the build is intended for direct usage or for packaging. Defaults to `bin`. * `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`. Defaults to `community`. """ # Temporary hack while 1.13.2 is not released System.version() |> Version.parse!() |> Version.compare(Version.parse!("1.13.2")) |> Kernel.==(:lt) |> if(do: Code.require_file("lib/mix/release.exs")) def project() do [ app: :emqx_mix, version: pkg_vsn(), deps: deps(), releases: releases() ] end defp deps() do # we need several overrides here because dependencies specify # other exact versions, and not ranges. [ {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true}, {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, {:ekka, github: "emqx/ekka", tag: "0.11.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, {:replayq, "0.3.3", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "k32/system_monitor", tag: "2.2.1"}, # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, {:hocon, github: "emqx/hocon", tag: "0.22.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, # in conflict by ehttpc and emqtt {:gun, github: "emqx/gun", tag: "1.3.6", override: true}, # in conflict by emqx_connectior and system_monitor {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, # in conflict by mongodb and eredis_cluster {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, # in conflict by emqx and observer_cli {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, # dependencies of dependencies; we choose specific refs to match # what rebar3 chooses. # in conflict by gun and emqtt {:cowlib, github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true}, # in conflict by cowboy_swagger and cowboy {:ranch, github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.11.2", override: true} ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep() end defp umbrella_apps() do "apps/*" |> Path.wildcard() |> Enum.map(fn path -> app = path |> String.trim_leading("apps/") |> String.to_atom() {app, path: path, manager: :rebar3, override: true} end) end defp releases() do [ emqx: fn -> %{ release_type: release_type, package_type: package_type, edition_type: edition_type } = read_inputs() base_steps = [ :assemble, &create_RELEASES/1, ©_files(&1, release_type, package_type, edition_type), ©_escript(&1, "nodetool"), ©_escript(&1, "install_upgrade.escript") ] steps = if System.get_env("ELIXIR_MAKE_TAR") == "yes" do base_steps ++ [:tar] else base_steps end [ applications: applications(release_type), skip_mode_validation_for: [ :emqx_gateway, :emqx_dashboard, :emqx_resource, :emqx_connector, :emqx_exhook, :emqx_bridge, :emqx_modules, :emqx_management, :emqx_statsd, :emqx_retainer, :emqx_prometheus, :emqx_plugins ], steps: steps, strip_beams: false ] end ] end def applications(release_type) do [ logger: :permanent, crypto: :permanent, public_key: :permanent, asn1: :permanent, syntax_tools: :permanent, ssl: :permanent, os_mon: :permanent, inets: :permanent, compiler: :permanent, runtime_tools: :permanent, hocon: :load, emqx: :load, emqx_conf: :load, emqx_machine: :permanent, mria: :load, mnesia: :load, ekka: :load, emqx_plugin_libs: :load, esasl: :load, observer_cli: :permanent, system_monitor: :permanent, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, emqx_authn: :permanent, emqx_authz: :permanent, emqx_auto_subscribe: :permanent, emqx_gateway: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, emqx_modules: :permanent, emqx_management: :permanent, emqx_dashboard: :permanent, emqx_retainer: :permanent, emqx_statsd: :permanent, emqx_prometheus: :permanent, emqx_psk: :permanent, emqx_slow_subs: :permanent, emqx_plugins: :permanent, emqx_mix: :none ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ if(release_type == :cloud, do: [xmerl: :permanent, observer: :load], else: [] ) end defp read_inputs() do release_type = read_enum_env_var( "EMQX_RELEASE_TYPE", [:cloud, :edge], :cloud ) package_type = read_enum_env_var( "EMQX_PACKAGE_TYPE", [:bin, :pkg], :bin ) edition_type = read_enum_env_var( "EMQX_EDITION_TYPE", [:community, :enterprise], :community ) %{ release_type: release_type, package_type: package_type, edition_type: edition_type } end defp copy_files(release, release_type, package_type, edition_type) do overwrite? = Keyword.get(release.options, :overwrite, false) bin = Path.join(release.path, "bin") etc = Path.join(release.path, "etc") Mix.Generator.create_directory(bin) Mix.Generator.create_directory(etc) Mix.Generator.create_directory(Path.join(etc, "certs")) Mix.Generator.copy_file( "apps/emqx_authz/etc/acl.conf", Path.join(etc, "acl.conf"), force: overwrite? ) # required by emqx_authz File.cp_r!( "apps/emqx/etc/certs", Path.join(etc, "certs") ) # this is required by the produced escript / nodetool Mix.Generator.copy_file( Path.join(release.version_path, "start_clean.boot"), Path.join(bin, "no_dot_erlang.boot"), force: overwrite? ) assigns = template_vars(release, release_type, package_type, edition_type) # This is generated by `scripts/merge-config.escript` or `make # conf-segs`. So, this should be run before the release. # TODO: run as a "compiler" step??? conf_rendered = File.read!("apps/emqx_conf/etc/emqx.conf.all") |> from_rebar_to_eex_template() |> EEx.eval_string(assigns) File.write!( Path.join(etc, "emqx.conf"), conf_rendered ) vars_rendered = File.read!("data/emqx_vars") |> from_rebar_to_eex_template() |> EEx.eval_string(assigns) File.write!( Path.join([release.path, "releases", "emqx_vars"]), vars_rendered ) vm_args_template_path = case release_type do :cloud -> "apps/emqx/etc/emqx_cloud/vm.args" :edge -> "apps/emqx/etc/emqx_edge/vm.args" end vm_args_rendered = File.read!(vm_args_template_path) |> from_rebar_to_eex_template() |> EEx.eval_string(assigns) File.write!( Path.join(etc, "vm.args"), vm_args_rendered ) File.write!( Path.join(release.version_path, "vm.args"), vm_args_rendered ) for name <- [ "emqx", "emqx_ctl" ] do Mix.Generator.copy_file( "bin/#{name}", Path.join(bin, name), force: overwrite? ) # Files with the version appended are expected by the release # upgrade script `install_upgrade.escript` Mix.Generator.copy_file( Path.join(bin, name), Path.join(bin, name <> "-#{release.version}"), force: overwrite? ) end for base_name <- ["emqx", "emqx_ctl"], suffix <- ["", "-#{release.version}"] do name = base_name <> suffix File.chmod!(Path.join(bin, name), 0o755) end built_on_rendered = File.read!("data/BUILT_ON") |> from_rebar_to_eex_template() |> EEx.eval_string(assigns) File.write!( Path.join([release.version_path, "BUILT_ON"]), built_on_rendered ) release end # needed by nodetool and by release_handler defp create_RELEASES(release) do apps = Enum.map(release.applications, fn {app_name, app_props} -> app_vsn = Keyword.fetch!(app_props, :vsn) app_path = "./lib" |> Path.join("#{app_name}-#{app_vsn}") |> to_charlist() {app_name, app_vsn, app_path} end) release_entry = [ { :release, to_charlist(release.name), to_charlist(release.version), release.erts_version, apps, :permanent } ] release.path |> Path.join("releases") |> Path.join("RELEASES") |> File.open!([:write, :utf8], fn handle -> IO.puts(handle, "%% coding: utf-8") :io.format(handle, '~tp.~n', [release_entry]) end) release end defp copy_escript(release, escript_name) do [shebang, rest] = "bin/#{escript_name}" |> File.read!() |> String.split("\n", parts: 2) # the elixir version of escript + start.boot required the boot_var # RELEASE_LIB to be defined. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib" # Files with the version appended are expected by the release # upgrade script `install_upgrade.escript` Enum.each( [escript_name, escript_name <> "-" <> release.version], fn name -> path = Path.join([release.path, "bin", name]) File.write!(path, [shebang, "\n", boot_var, "\n", rest]) end ) release end defp template_vars(release, release_type, :bin = _package_type, edition_type) do [ platform_bin_dir: "bin", platform_data_dir: "data", platform_etc_dir: "etc", platform_lib_dir: "lib", platform_log_dir: "log", platform_plugins_dir: "plugins", runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)", runner_bin_dir: "$RUNNER_ROOT_DIR/bin", runner_etc_dir: "$RUNNER_ROOT_DIR/etc", runner_lib_dir: "$RUNNER_ROOT_DIR/lib", runner_log_dir: "$RUNNER_ROOT_DIR/log", runner_data_dir: "$RUNNER_ROOT_DIR/data", runner_user: "", release_version: release.version, erts_vsn: release.erts_version, # FIXME: this is empty in `make emqx` ??? erl_opts: "", emqx_description: emqx_description(release_type, edition_type), built_on_arch: built_on(), is_elixir: "yes" ] end defp template_vars(release, release_type, :pkg = _package_type, edition_type) do [ platform_bin_dir: "", platform_data_dir: "/var/lib/emqx", platform_etc_dir: "/etc/emqx", platform_lib_dir: "", platform_log_dir: "/var/log/emqx", platform_plugins_dir: "/var/lib/emqx/plugins", runner_root_dir: "/usr/lib/emqx", runner_bin_dir: "/usr/bin", runner_etc_dir: "/etc/emqx", runner_lib_dir: "$RUNNER_ROOT_DIR/lib", runner_log_dir: "/var/log/emqx", runner_data_dir: "/var/lib/emqx", runner_user: "emqx", release_version: release.version, erts_vsn: release.erts_version, # FIXME: this is empty in `make emqx` ??? erl_opts: "", emqx_description: emqx_description(release_type, edition_type), built_on: built_on(), is_elixir: "yes" ] end defp read_enum_env_var(env_var, allowed_values, default_value) do case System.fetch_env(env_var) do :error -> default_value {:ok, raw_value} -> value = raw_value |> String.downcase() |> String.to_atom() if value not in allowed_values do Mix.raise(""" Invalid value #{raw_value} for variable #{env_var}. Allowed values are: #{inspect(allowed_values)} """) end value end end defp emqx_description(release_type, edition_type) do case {release_type, edition_type} do {:cloud, :enterprise} -> "EMQ X Enterprise Edition" {:cloud, :community} -> "EMQ X Community Edition" {:edge, :community} -> "EMQ X Edge Edition" end end defp bcrypt_dep() do if enable_bcrypt?(), do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}], else: [] end defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}], else: [] end defp enable_bcrypt?() do not win32?() end defp enable_quicer?() do not Enum.any?([ build_without_quic?(), win32?(), centos6?() ]) end defp pkg_vsn() do basedir = Path.dirname(__ENV__.file) script = Path.join(basedir, "pkg-vsn.sh") {str_vsn, 0} = System.cmd(script, []) String.trim(str_vsn) end defp win32?(), do: match?({:win_32, _}, :os.type()) defp centos6?() do case File.read("/etc/centos-release") do {:ok, "CentOS release 6" <> _} -> true _ -> false end end defp build_without_quic?() do opt = System.get_env("BUILD_WITHOUT_QUIC", "false") String.downcase(opt) != "false" end defp from_rebar_to_eex_template(str) do # we must not consider surrounding space in the template var name # because some help strings contain informative variables that # should not be interpolated, and those have no spaces. Regex.replace( ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/, str, "<%= \\g{1} %>" ) end defp built_on() do system_architecture = to_string(:erlang.system_info(:system_architecture)) elixir_version = System.version() words = wordsize() "#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}" end # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142 defp wordsize() do size = try do :erlang.system_info({:wordsize, :external}) rescue ErlangError -> :erlang.system_info(:wordsize) end to_string(8 * size) end # As from Erlang/OTP 17, the OTP release number corresponds to the # major OTP version number. No erlang:system_info() argument gives # the exact OTP version. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577 defp otp_release() do major_version = System.otp_release() root_dir = to_string(:code.root_dir()) [root_dir, "releases", major_version, "OTP_VERSION"] |> Path.join() |> File.read() |> case do {:error, _} -> major_version {:ok, version} -> version |> String.trim() |> String.split("**") |> List.first() end end end