01092982c1
Despite Hocon generating the same config in both Elixir and Erlang releases, apparently starting Logger before other applications has the subtle effect of redirecting all logs to `log/erlang.log.N`, even those lines that should appear in `log/emqx.log.N`. Simply not adding it to the application list restores the Erlang release behavior.
631 lines
18 KiB
Elixir
631 lines
18 KiB
Elixir
defmodule EMQXUmbrella.MixProject do
|
|
use Mix.Project
|
|
|
|
@moduledoc """
|
|
|
|
The purpose of this file is to configure the release of EMQX under
|
|
Mix. Since EMQX uses its own configuration conventions and startup
|
|
procedures, one cannot simply use `iex -S mix`. Instead, it's
|
|
recommendd to build and use the release.
|
|
|
|
## Release Environment Variables
|
|
|
|
The release build is controlled by a few environment variables.
|
|
|
|
* `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
|
|
tarball along with the release.
|
|
* `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`. Controls a
|
|
few dependencies and the `vm.args` to be used. Defaults to
|
|
`cloud`.
|
|
* `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`. Controls
|
|
whether the build is intended for direct usage or for packaging.
|
|
Defaults to `bin`.
|
|
* `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`.
|
|
Defaults to `community`.
|
|
"""
|
|
|
|
# Temporary hack while 1.13.2 is not released
|
|
System.version()
|
|
|> Version.parse!()
|
|
|> Version.compare(Version.parse!("1.13.2"))
|
|
|> Kernel.==(:lt)
|
|
|> if(do: Code.require_file("lib/mix/release.exs"))
|
|
|
|
def project() do
|
|
[
|
|
app: :emqx_mix,
|
|
version: pkg_vsn(),
|
|
deps: deps(),
|
|
releases: releases()
|
|
]
|
|
end
|
|
|
|
defp deps() do
|
|
# we need several overrides here because dependencies specify
|
|
# other exact versions, and not ranges.
|
|
[
|
|
{:lc, github: "qzhuyan/lc", tag: "0.1.2"},
|
|
{:redbug, "2.0.7"},
|
|
{:typerefl, github: "k32/typerefl", tag: "0.8.6", override: true},
|
|
{:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"},
|
|
{:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
|
|
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
|
|
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
|
|
{:esockd, github: "emqx/esockd", tag: "5.9.0", override: true},
|
|
{:mria, github: "emqx/mria", tag: "0.1.5", override: true},
|
|
{:ekka, github: "emqx/ekka", tag: "0.11.2", override: true},
|
|
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.0", override: true},
|
|
{:minirest, github: "emqx/minirest", tag: "1.2.10", override: true},
|
|
{:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
|
|
{:replayq, "0.3.3", override: true},
|
|
{:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
|
|
{:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true},
|
|
{:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
|
|
{:observer_cli, "1.7.1"},
|
|
{:system_monitor, github: "k32/system_monitor", tag: "2.2.1"},
|
|
# in conflict by emqtt and hocon
|
|
{:getopt, "1.0.2", override: true},
|
|
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true},
|
|
{:hocon, github: "emqx/hocon", tag: "0.22.2", override: true},
|
|
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true},
|
|
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
|
|
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
|
|
# in conflict by ehttpc and emqtt
|
|
{:gun, github: "emqx/gun", tag: "1.3.6", override: true},
|
|
# in conflict by emqx_connectior and system_monitor
|
|
{:epgsql, github: "emqx/epgsql", tag: "4.7-emqx.2", override: true},
|
|
# in conflict by mongodb and eredis_cluster
|
|
{:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true},
|
|
# in conflict by emqx and observer_cli
|
|
{:recon, github: "ferd/recon", tag: "2.5.1", override: true},
|
|
{:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
|
|
# dependencies of dependencies; we choose specific refs to match
|
|
# what rebar3 chooses.
|
|
# in conflict by gun and emqtt
|
|
{:cowlib,
|
|
github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
|
|
# in conflict by cowboy_swagger and cowboy
|
|
{:ranch,
|
|
github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
|
|
# in conflict by grpc and eetcd
|
|
{:gpb, "4.11.2", override: true}
|
|
] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
|
|
end
|
|
|
|
defp umbrella_apps() do
|
|
"apps/*"
|
|
|> Path.wildcard()
|
|
|> Enum.map(fn path ->
|
|
app =
|
|
path
|
|
|> String.trim_leading("apps/")
|
|
|> String.to_atom()
|
|
|
|
{app, path: path, manager: :rebar3, override: true}
|
|
end)
|
|
end
|
|
|
|
defp releases() do
|
|
[
|
|
emqx: fn ->
|
|
%{
|
|
release_type: release_type,
|
|
package_type: package_type,
|
|
edition_type: edition_type
|
|
} = read_inputs()
|
|
|
|
base_steps = [
|
|
:assemble,
|
|
&create_RELEASES/1,
|
|
©_files(&1, release_type, package_type, edition_type),
|
|
©_escript(&1, "nodetool"),
|
|
©_escript(&1, "install_upgrade.escript")
|
|
]
|
|
|
|
steps =
|
|
if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
|
|
base_steps ++ [&prepare_tar_overlays/1, :tar]
|
|
else
|
|
base_steps
|
|
end
|
|
|
|
[
|
|
applications: applications(release_type),
|
|
skip_mode_validation_for: [
|
|
:emqx_gateway,
|
|
:emqx_dashboard,
|
|
:emqx_resource,
|
|
:emqx_connector,
|
|
:emqx_exhook,
|
|
:emqx_bridge,
|
|
:emqx_modules,
|
|
:emqx_management,
|
|
:emqx_statsd,
|
|
:emqx_retainer,
|
|
:emqx_prometheus,
|
|
:emqx_plugins
|
|
],
|
|
steps: steps,
|
|
strip_beams: false
|
|
]
|
|
end
|
|
]
|
|
end
|
|
|
|
def applications(release_type) do
|
|
[
|
|
crypto: :permanent,
|
|
public_key: :permanent,
|
|
asn1: :permanent,
|
|
syntax_tools: :permanent,
|
|
ssl: :permanent,
|
|
os_mon: :permanent,
|
|
inets: :permanent,
|
|
compiler: :permanent,
|
|
runtime_tools: :permanent,
|
|
redbug: :permanent,
|
|
hocon: :load,
|
|
emqx: :load,
|
|
emqx_conf: :load,
|
|
emqx_machine: :permanent,
|
|
mria: :load,
|
|
mnesia: :load,
|
|
ekka: :load,
|
|
emqx_plugin_libs: :load,
|
|
esasl: :load,
|
|
observer_cli: :permanent,
|
|
system_monitor: :permanent,
|
|
emqx_http_lib: :permanent,
|
|
emqx_resource: :permanent,
|
|
emqx_connector: :permanent,
|
|
emqx_authn: :permanent,
|
|
emqx_authz: :permanent,
|
|
emqx_auto_subscribe: :permanent,
|
|
emqx_gateway: :permanent,
|
|
emqx_exhook: :permanent,
|
|
emqx_bridge: :permanent,
|
|
emqx_rule_engine: :permanent,
|
|
emqx_modules: :permanent,
|
|
emqx_management: :permanent,
|
|
emqx_dashboard: :permanent,
|
|
emqx_retainer: :permanent,
|
|
emqx_statsd: :permanent,
|
|
emqx_prometheus: :permanent,
|
|
emqx_psk: :permanent,
|
|
emqx_slow_subs: :permanent,
|
|
emqx_plugins: :permanent,
|
|
emqx_mix: :none
|
|
] ++
|
|
if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
|
|
if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
|
|
if(release_type == :cloud,
|
|
do: [xmerl: :permanent, observer: :load],
|
|
else: []
|
|
)
|
|
end
|
|
|
|
defp read_inputs() do
|
|
release_type =
|
|
read_enum_env_var(
|
|
"EMQX_RELEASE_TYPE",
|
|
[:cloud, :edge],
|
|
:cloud
|
|
)
|
|
|
|
package_type =
|
|
read_enum_env_var(
|
|
"EMQX_PACKAGE_TYPE",
|
|
[:bin, :pkg],
|
|
:bin
|
|
)
|
|
|
|
edition_type =
|
|
read_enum_env_var(
|
|
"EMQX_EDITION_TYPE",
|
|
[:community, :enterprise],
|
|
:community
|
|
)
|
|
|
|
%{
|
|
release_type: release_type,
|
|
package_type: package_type,
|
|
edition_type: edition_type
|
|
}
|
|
end
|
|
|
|
#############################################################################
|
|
# Custom Steps
|
|
#############################################################################
|
|
|
|
defp copy_files(release, release_type, package_type, edition_type) do
|
|
overwrite? = Keyword.get(release.options, :overwrite, false)
|
|
|
|
bin = Path.join(release.path, "bin")
|
|
etc = Path.join(release.path, "etc")
|
|
log = Path.join(release.path, "log")
|
|
|
|
Mix.Generator.create_directory(bin)
|
|
Mix.Generator.create_directory(etc)
|
|
Mix.Generator.create_directory(log)
|
|
Mix.Generator.create_directory(Path.join(etc, "certs"))
|
|
|
|
Enum.each(
|
|
["mnesia", "configs", "patches", "scripts"],
|
|
fn dir ->
|
|
path = Path.join([release.path, "data", dir])
|
|
Mix.Generator.create_directory(path)
|
|
end
|
|
)
|
|
|
|
Mix.Generator.copy_file(
|
|
"apps/emqx_authz/etc/acl.conf",
|
|
Path.join(etc, "acl.conf"),
|
|
force: overwrite?
|
|
)
|
|
|
|
# required by emqx_authz
|
|
File.cp_r!(
|
|
"apps/emqx/etc/certs",
|
|
Path.join(etc, "certs")
|
|
)
|
|
|
|
# this is required by the produced escript / nodetool
|
|
Mix.Generator.copy_file(
|
|
Path.join(release.version_path, "start_clean.boot"),
|
|
Path.join(bin, "no_dot_erlang.boot"),
|
|
force: overwrite?
|
|
)
|
|
|
|
assigns = template_vars(release, release_type, package_type, edition_type)
|
|
|
|
# This is generated by `scripts/merge-config.escript` or `make
|
|
# conf-segs`. So, this should be run before the release.
|
|
# TODO: run as a "compiler" step???
|
|
conf_rendered =
|
|
File.read!("apps/emqx_conf/etc/emqx.conf.all")
|
|
|> from_rebar_to_eex_template()
|
|
|> EEx.eval_string(assigns)
|
|
|
|
File.write!(
|
|
Path.join(etc, "emqx.conf"),
|
|
conf_rendered
|
|
)
|
|
|
|
vars_rendered =
|
|
File.read!("rel/emqx_vars")
|
|
|> from_rebar_to_eex_template()
|
|
|> EEx.eval_string(assigns)
|
|
|
|
File.write!(
|
|
Path.join([release.path, "releases", "emqx_vars"]),
|
|
vars_rendered
|
|
)
|
|
|
|
vm_args_template_path =
|
|
case release_type do
|
|
:cloud ->
|
|
"apps/emqx/etc/emqx_cloud/vm.args"
|
|
|
|
:edge ->
|
|
"apps/emqx/etc/emqx_edge/vm.args"
|
|
end
|
|
|
|
vm_args_rendered =
|
|
File.read!(vm_args_template_path)
|
|
|> from_rebar_to_eex_template()
|
|
|> EEx.eval_string(assigns)
|
|
|
|
File.write!(
|
|
Path.join(etc, "vm.args"),
|
|
vm_args_rendered
|
|
)
|
|
|
|
File.write!(
|
|
Path.join(release.version_path, "vm.args"),
|
|
vm_args_rendered
|
|
)
|
|
|
|
for name <- [
|
|
"emqx",
|
|
"emqx_ctl"
|
|
] do
|
|
Mix.Generator.copy_file(
|
|
"bin/#{name}",
|
|
Path.join(bin, name),
|
|
force: overwrite?
|
|
)
|
|
|
|
# Files with the version appended are expected by the release
|
|
# upgrade script `install_upgrade.escript`
|
|
Mix.Generator.copy_file(
|
|
Path.join(bin, name),
|
|
Path.join(bin, name <> "-#{release.version}"),
|
|
force: overwrite?
|
|
)
|
|
end
|
|
|
|
for base_name <- ["emqx", "emqx_ctl"],
|
|
suffix <- ["", "-#{release.version}"] do
|
|
name = base_name <> suffix
|
|
File.chmod!(Path.join(bin, name), 0o755)
|
|
end
|
|
|
|
built_on_rendered =
|
|
File.read!("rel/BUILT_ON")
|
|
|> from_rebar_to_eex_template()
|
|
|> EEx.eval_string(assigns)
|
|
|
|
File.write!(
|
|
Path.join([release.version_path, "BUILT_ON"]),
|
|
built_on_rendered
|
|
)
|
|
|
|
release
|
|
end
|
|
|
|
# needed by nodetool and by release_handler
|
|
defp create_RELEASES(release) do
|
|
apps =
|
|
Enum.map(release.applications, fn {app_name, app_props} ->
|
|
app_vsn = Keyword.fetch!(app_props, :vsn)
|
|
|
|
app_path =
|
|
"./lib"
|
|
|> Path.join("#{app_name}-#{app_vsn}")
|
|
|> to_charlist()
|
|
|
|
{app_name, app_vsn, app_path}
|
|
end)
|
|
|
|
release_entry = [
|
|
{
|
|
:release,
|
|
to_charlist(release.name),
|
|
to_charlist(release.version),
|
|
release.erts_version,
|
|
apps,
|
|
:permanent
|
|
}
|
|
]
|
|
|
|
release.path
|
|
|> Path.join("releases")
|
|
|> Path.join("RELEASES")
|
|
|> File.open!([:write, :utf8], fn handle ->
|
|
IO.puts(handle, "%% coding: utf-8")
|
|
:io.format(handle, '~tp.~n', [release_entry])
|
|
end)
|
|
|
|
release
|
|
end
|
|
|
|
defp copy_escript(release, escript_name) do
|
|
[shebang, rest] =
|
|
"bin/#{escript_name}"
|
|
|> File.read!()
|
|
|> String.split("\n", parts: 2)
|
|
|
|
# the elixir version of escript + start.boot required the boot_var
|
|
# RELEASE_LIB to be defined.
|
|
boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"
|
|
|
|
# Files with the version appended are expected by the release
|
|
# upgrade script `install_upgrade.escript`
|
|
Enum.each(
|
|
[escript_name, escript_name <> "-" <> release.version],
|
|
fn name ->
|
|
path = Path.join([release.path, "bin", name])
|
|
File.write!(path, [shebang, "\n", boot_var, "\n", rest])
|
|
end
|
|
)
|
|
|
|
release
|
|
end
|
|
|
|
# The `:tar` built-in step in Mix Release does not currently add the
|
|
# `etc` directory into the resulting tarball. The workaround is to
|
|
# add those to the `:overlays` key before running `:tar`.
|
|
# See: https://hexdocs.pm/mix/1.13.1/Mix.Release.html#__struct__/0
|
|
defp prepare_tar_overlays(release) do
|
|
Map.update!(release, :overlays, &["etc", "data" | &1])
|
|
end
|
|
|
|
#############################################################################
|
|
# Helper functions
|
|
#############################################################################
|
|
|
|
defp template_vars(release, release_type, :bin = _package_type, edition_type) do
|
|
[
|
|
platform_bin_dir: "bin",
|
|
platform_data_dir: "data",
|
|
platform_etc_dir: "etc",
|
|
platform_lib_dir: "lib",
|
|
platform_log_dir: "log",
|
|
platform_plugins_dir: "plugins",
|
|
runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)",
|
|
runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
|
|
runner_etc_dir: "$RUNNER_ROOT_DIR/etc",
|
|
runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
|
|
runner_log_dir: "$RUNNER_ROOT_DIR/log",
|
|
runner_data_dir: "$RUNNER_ROOT_DIR/data",
|
|
runner_user: "",
|
|
release_version: release.version,
|
|
erts_vsn: release.erts_version,
|
|
# FIXME: this is empty in `make emqx` ???
|
|
erl_opts: "",
|
|
emqx_description: emqx_description(release_type, edition_type),
|
|
built_on_arch: built_on(),
|
|
is_elixir: "yes"
|
|
]
|
|
end
|
|
|
|
defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
|
|
[
|
|
platform_bin_dir: "",
|
|
platform_data_dir: "/var/lib/emqx",
|
|
platform_etc_dir: "/etc/emqx",
|
|
platform_lib_dir: "",
|
|
platform_log_dir: "/var/log/emqx",
|
|
platform_plugins_dir: "/var/lib/emqx/plugins",
|
|
runner_root_dir: "/usr/lib/emqx",
|
|
runner_bin_dir: "/usr/bin",
|
|
runner_etc_dir: "/etc/emqx",
|
|
runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
|
|
runner_log_dir: "/var/log/emqx",
|
|
runner_data_dir: "/var/lib/emqx",
|
|
runner_user: "emqx",
|
|
release_version: release.version,
|
|
erts_vsn: release.erts_version,
|
|
# FIXME: this is empty in `make emqx` ???
|
|
erl_opts: "",
|
|
emqx_description: emqx_description(release_type, edition_type),
|
|
built_on_arch: built_on(),
|
|
is_elixir: "yes"
|
|
]
|
|
end
|
|
|
|
defp read_enum_env_var(env_var, allowed_values, default_value) do
|
|
case System.fetch_env(env_var) do
|
|
:error ->
|
|
default_value
|
|
|
|
{:ok, raw_value} ->
|
|
value =
|
|
raw_value
|
|
|> String.downcase()
|
|
|> String.to_atom()
|
|
|
|
if value not in allowed_values do
|
|
Mix.raise("""
|
|
Invalid value #{raw_value} for variable #{env_var}.
|
|
Allowed values are: #{inspect(allowed_values)}
|
|
""")
|
|
end
|
|
|
|
value
|
|
end
|
|
end
|
|
|
|
defp emqx_description(release_type, edition_type) do
|
|
case {release_type, edition_type} do
|
|
{:cloud, :enterprise} ->
|
|
"EMQ X Enterprise Edition"
|
|
|
|
{:cloud, :community} ->
|
|
"EMQ X Community Edition"
|
|
|
|
{:edge, :community} ->
|
|
"EMQ X Edge Edition"
|
|
end
|
|
end
|
|
|
|
defp bcrypt_dep() do
|
|
if enable_bcrypt?(),
|
|
do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}],
|
|
else: []
|
|
end
|
|
|
|
defp quicer_dep() do
|
|
if enable_quicer?(),
|
|
# in conflict with emqx and emqtt
|
|
do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}],
|
|
else: []
|
|
end
|
|
|
|
defp enable_bcrypt?() do
|
|
not win32?()
|
|
end
|
|
|
|
defp enable_quicer?() do
|
|
not Enum.any?([
|
|
build_without_quic?(),
|
|
win32?(),
|
|
centos6?()
|
|
])
|
|
end
|
|
|
|
defp pkg_vsn() do
|
|
%{edition_type: edition_type} = read_inputs()
|
|
basedir = Path.dirname(__ENV__.file)
|
|
script = Path.join(basedir, "pkg-vsn.sh")
|
|
{str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)])
|
|
|
|
String.trim(str_vsn)
|
|
end
|
|
|
|
defp win32?(),
|
|
do: match?({:win_32, _}, :os.type())
|
|
|
|
defp centos6?() do
|
|
case File.read("/etc/centos-release") do
|
|
{:ok, "CentOS release 6" <> _} ->
|
|
true
|
|
|
|
_ ->
|
|
false
|
|
end
|
|
end
|
|
|
|
defp build_without_quic?() do
|
|
opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
|
|
|
|
String.downcase(opt) != "false"
|
|
end
|
|
|
|
defp from_rebar_to_eex_template(str) do
|
|
# we must not consider surrounding space in the template var name
|
|
# because some help strings contain informative variables that
|
|
# should not be interpolated, and those have no spaces.
|
|
Regex.replace(
|
|
~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
|
|
str,
|
|
"<%= \\g{1} %>"
|
|
)
|
|
end
|
|
|
|
defp built_on() do
|
|
system_architecture = to_string(:erlang.system_info(:system_architecture))
|
|
elixir_version = System.version()
|
|
words = wordsize()
|
|
|
|
"#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}"
|
|
end
|
|
|
|
# https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
|
|
defp wordsize() do
|
|
size =
|
|
try do
|
|
:erlang.system_info({:wordsize, :external})
|
|
rescue
|
|
ErlangError ->
|
|
:erlang.system_info(:wordsize)
|
|
end
|
|
|
|
to_string(8 * size)
|
|
end
|
|
|
|
# As from Erlang/OTP 17, the OTP release number corresponds to the
|
|
# major OTP version number. No erlang:system_info() argument gives
|
|
# the exact OTP version.
|
|
# https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
|
|
# https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
|
|
defp otp_release() do
|
|
major_version = System.otp_release()
|
|
root_dir = to_string(:code.root_dir())
|
|
|
|
[root_dir, "releases", major_version, "OTP_VERSION"]
|
|
|> Path.join()
|
|
|> File.read()
|
|
|> case do
|
|
{:error, _} ->
|
|
major_version
|
|
|
|
{:ok, version} ->
|
|
version
|
|
|> String.trim()
|
|
|> String.split("**")
|
|
|> List.first()
|
|
end
|
|
end
|
|
end
|