yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #13455 from thalesmg/20240711-m-mix-umbrella-part-III-no-ci 

sync new mix build work to master
This commit is contained in:
Thales Macedo Garitezi 2024-07-16 14:41:10 -03:00 committed by GitHub
parent 706cab3c86 4a08bfc93f
commit 1ad02a11e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 509 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/.zipignore2 vendored Normal file
View File

@ -0,0 +1 @@
*/.github/*

8
Makefile
View File

@ -28,6 +28,8 @@ CT_COVER_EXPORT_PREFIX ?= $(PROFILE)
export REBAR_GIT_CLONE_OPTIONS += --depth=1
ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
.PHONY: default
default: $(REBAR) $(PROFILE)
@ -58,8 +60,12 @@ ensure-mix-rebar3: $(REBAR)
ensure-mix-rebar: $(REBAR)
@mix local.rebar --if-missing --force
.PHONY: elixir-common-deps
elixir-common-deps: $(ELIXIR_COMMON_DEPS)
.PHONY: mix-deps-get
mix-deps-get: $(ELIXIR_COMMON_DEPS)
mix-deps-get: elixir-common-deps
@mix deps.get
.PHONY: eunit

14
apps/emqx/mix.exs
View File

@ -8,7 +8,7 @@ defmodule EMQX.MixProject do
app: :emqx,
version: "0.1.0",
build_path: "../../_build",
erlc_paths: UMP.erlc_paths(),
erlc_paths: erlc_paths(),
erlc_options: [
{:i, "src"}
| UMP.erlc_options()
@ -36,8 +36,9 @@ defmodule EMQX.MixProject do
def deps() do
## FIXME!!! go though emqx.app.src and add missing stuff...
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx_utils, in_umbrella: true},
# {:emqx_ds_backends, in_umbrella: true},
{:emqx_ds_backends, in_umbrella: true},
UMP.common_dep(:gproc),
UMP.common_dep(:gen_rpc),
@ -53,6 +54,15 @@ defmodule EMQX.MixProject do
] ++ UMP.quicer_dep()
end
defp erlc_paths() do
paths = UMP.erlc_paths()
if UMP.test_env?() do
["integration_test" | paths]
else
paths
end
end
defp extra_dirs() do
dirs = ["src", "etc"]
if UMP.test_env?() do

1
apps/emqx_auth/mix.exs
View File

@ -28,6 +28,7 @@ defmodule EMQXAuth.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true}
]

1
apps/emqx_bridge/mix.exs
View File

@ -28,6 +28,7 @@ defmodule EMQXBridge.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_resource, in_umbrella: true},
{:emqx_connector, in_umbrella: true},

3
apps/emqx_bridge_hstreamdb/mix.exs
View File

@ -24,7 +24,8 @@ defmodule EMQXBridgeHstreamdb.MixProject do
def deps() do
[
{:hstreamdb_erl,
github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1",
system_env: UMP.emqx_app_system_env()},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_connector, in_umbrella: true, runtime: false},

3
apps/emqx_dashboard_sso/mix.exs
View File

@ -26,7 +26,8 @@ defmodule EMQXDashboardSso.MixProject do
{:emqx_ctl, in_umbrella: true},
{:emqx_ldap, in_umbrella: true},
{:emqx_dashboard, in_umbrella: true},
{:esaml, github: "emqx/esaml", tag: "v1.1.3"}
{:esaml, github: "emqx/esaml", tag: "v1.1.3"},
{:oidcc, github: "emqx/oidcc", tag: "v3.2.0-1"},
]
end
end

10
apps/emqx_durable_storage/mix.exs
View File

@ -7,9 +7,14 @@ defmodule EMQXDurableStorage.MixProject do
app: :emqx_durable_storage,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
compilers: [:yecc, :leex, :elixir, :asn1, :erlang, :app],
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
erlc_paths: ["gen_src" | UMP.erlc_paths()],
# used by our `compile.asn1` compiler
asn1_srcs: [
%{src: "./asn.1/DurableMessage.asn",
compile_opts: [:per, :noobj, outdir: ~c"gen_src"]}
],
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
@ -28,6 +33,7 @@ defmodule EMQXDurableStorage.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx_utils, in_umbrella: true},
UMP.common_dep(:rocksdb),
UMP.common_dep(:gproc),

1
apps/emqx_exhook/mix.exs
View File

@ -36,6 +36,7 @@ defmodule EMQXExhook.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
UMP.common_dep(:grpc)

1
apps/emqx_gateway/mix.exs
View File

@ -26,6 +26,7 @@ defmodule EMQXGateway.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_ctl, in_umbrella: true},

2
apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
View File

@ -48,6 +48,8 @@ init_per_suite(Conf) ->
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_auth,
emqx_auth_mnesia,
emqx_management,
{emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"},
{emqx_gateway, ?CONF_DEFAULT}

1
apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl
View File

@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) ->
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_auth_http,
emqx_management,
{emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"},
{emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()}

1
apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl
View File

@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) ->
Apps = emqx_cth_suite:start(
[
{emqx_conf, "authorization { no_match = deny, cache { enable = false } }"},
emqx_auth_http,
{emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()}
| emqx_gateway_test_utils:all_gateway_apps()
],

1
apps/emqx_gateway_exproto/mix.exs
View File

@ -34,6 +34,7 @@ defmodule EMQXGatewayExproto.MixProject do
def deps() do
test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: []
test_deps ++ [
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_gateway, in_umbrella: true},

1
apps/emqx_gcp_device/mix.exs
View File

@ -27,6 +27,7 @@ defmodule EMQXGCPDevice.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
UMP.common_dep(:jose),

1
apps/emqx_license/mix.exs
View File

@ -26,6 +26,7 @@ defmodule EMQXLicense.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_ctl, in_umbrella: true},

2
apps/emqx_machine/mix.exs
View File

@ -30,6 +30,8 @@ defmodule EMQXMachine.MixProject do
{:emqx_dashboard, in_umbrella: true, runtime: false},
{:emqx_management, in_umbrella: true, runtime: false},
UMP.common_dep(:covertool),
UMP.common_dep(:system_monitor),
UMP.common_dep(:redbug),
]
end
end

0
apps/emqx_exhook/lib/emqx/grpc/template/client.eex → apps/emqx_mix_utils/lib/emqx/grpc/template/client.eex
View File

0
apps/emqx_exhook/lib/emqx/grpc/template/service.eex → apps/emqx_mix_utils/lib/emqx/grpc/template/service.eex
View File

82
apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex Normal file
View File

@ -0,0 +1,82 @@
defmodule Mix.Tasks.Compile.Asn1 do
use Mix.Task.Compiler
@recursive true
@manifest_vsn 1
@manifest "compile.asn1"
# TODO: use manifest to track generated files?
@impl true
def manifests(), do: [manifest()]
defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest)
@impl true
def run(_args) do
add_to_path_and_cache(:asn1)
Mix.Project.get!()
config = Mix.Project.config()
app_root = File.cwd!()
asn1_srcs = config[:asn1_srcs] || []
manifest_data = read_manifest(manifest())
manifest_modified_time = Mix.Utils.last_modified(manifest())
Enum.each(asn1_srcs, &compile(&1, app_root, manifest_modified_time))
write_manifest(manifest(), manifest_data)
{:noop, []}
end
defp compile(src, app_root, manifest_modified_time) do
%{
src: src_path,
compile_opts: compile_opts
} = src
src_path =
app_root
|> Path.join(src_path)
|> Path.expand()
if stale?(src_path, manifest_modified_time) do
Mix.shell().info("compiling asn1 file: #{src_path}")
:ok = :asn1ct.compile(to_charlist(src_path), compile_opts)
else
Mix.shell().info("file is up to date, not compiling: #{src_path}")
end
end
defp stale?(file, manifest_modified_time) do
with true <- File.exists?(file),
false <- Mix.Utils.stale?([file], [manifest_modified_time]) do
false
else
_ -> true
end
end
defp read_manifest(file) do
try do
file |> File.read!() |> :erlang.binary_to_term()
rescue
_ -> %{}
else
{@manifest_vsn, data} when is_map(data) -> data
_ -> %{}
end
end
defp write_manifest(file, data) do
Mix.shell().info("writing manifest #{file}")
File.mkdir_p!(Path.dirname(file))
File.write!(file, :erlang.term_to_binary({@manifest_vsn, data}))
end
def add_to_path_and_cache(lib_name) do
:code.lib_dir()
|> Path.join("#{lib_name}-*")
|> Path.wildcard()
|> hd()
|> Path.join("ebin")
|> to_charlist()
|> :code.add_path(:cache)
end
end

0
apps/emqx/lib/mix/tasks/compile.copy_srcs.ex → apps/emqx_mix_utils/lib/mix/tasks/compile.copy_srcs.ex
View File

7
apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex → apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex
View File

@ -1,6 +1,8 @@
defmodule Mix.Tasks.Compile.Grpc do
use Mix.Task.Compiler
alias EMQXUmbrella.MixProject, as: UMP
@recursive true
@manifest_vsn 1
@manifest "compile.grpc"
@ -46,6 +48,9 @@ defmodule Mix.Tasks.Compile.Grpc do
write_manifest(manifest(), manifest_data)
{:noop, []}
after
Application.unload(:gpb)
Application.unload(:syntax_tools)
end
defp compile_pb(proto_src, context) do
@ -95,7 +100,7 @@ defmodule Mix.Tasks.Compile.Grpc do
:return_errors,
i: to_charlist(gpb_include_dir),
outdir: to_charlist(ebin_path)
]
] ++ UMP.erlc_options()
)
# todo: error handling & logging
case compile_res do

13
lib/mix/tasks/emqx.ct.ex → apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex
View File

@ -48,10 +48,11 @@ defmodule Mix.Tasks.Emqx.Ct do
abort_if_missing_suites: true,
auto_compile: false,
suite: opts |> Map.fetch!(:suites) |> Enum.map(&to_charlist/1),
group: opts |> Map.fetch!(:group_paths) |> Enum.map(fn gp -> Enum.map(gp, &String.to_atom/1) end),
testcase: opts |> Map.fetch!(:cases) |> Enum.map(&to_charlist/1),
readable: 'true',
name: node_name,
ct_hooks: [:cth_readable_shell],
ct_hooks: [:cth_readable_shell, :cth_readable_failonly],
logdir: to_charlist(logdir)
)
@ -198,11 +199,13 @@ defmodule Mix.Tasks.Emqx.Ct do
args,
strict: [
suites: :string,
groups: :string,
group_paths: :string,
cases: :string])
|> IO.inspect(label: :opts)
suites = get_name_list(opts, :suites)
groups = get_name_list(opts, :groups)
group_paths =
opts
|> get_name_list(:group_paths)
|> Enum.map(& String.split(&1, ".", trim: true))
cases = get_name_list(opts, :cases)
if suites == [] do
@ -211,7 +214,7 @@ defmodule Mix.Tasks.Emqx.Ct do
%{
suites: suites,
groups: groups,
group_paths: group_paths,
cases: cases
}
end

158
apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex Normal file
View File

@ -0,0 +1,158 @@
defmodule Mix.Tasks.Emqx.Dialyzer do
use Mix.Task
alias Mix.Tasks.Emqx.Ct, as: ECt
@requirements ["compile", "loadpaths"]
@excluded_mods (
[
:emqx_exproto_v_1_connection_unary_handler_bhvr,
:emqx_exproto_v_1_connection_handler_client,
:emqx_exproto_v_1_connection_handler_bhvr,
:emqx_exproto_v_1_connection_adapter_client,
:emqx_exproto_v_1_connection_adapter_bhvr,
:emqx_exproto_v_1_connection_unary_handler_client,
:emqx_exhook_v_2_hook_provider_client,
:emqx_exhook_v_2_hook_provider_bhvr,
Mix.Tasks.Compile.Grpc,
Mix.Tasks.Compile.CopySrcs,
]
|> MapSet.new(&to_string/1)
)
@impl true
def run(_args) do
ECt.add_to_path_and_cache(:dialyzer)
%{
umbrella_apps: umbrella_apps,
dep_apps: dep_apps
} = resolve_apps()
umbrella_files = Enum.flat_map(umbrella_apps, & resolve_files/1)
dep_files = Enum.flat_map(dep_apps, & resolve_files/1)
files =
(umbrella_files ++ dep_files)
|> Enum.reject(fn path ->
name = Path.basename(path, ".beam")
MapSet.member?(@excluded_mods, name)
end)
|> Enum.map(&to_charlist/1)
warning_files =
umbrella_files
|> Enum.reject(fn path ->
name = Path.basename(path, ".beam")
MapSet.member?(@excluded_mods, name)
end)
|> Enum.map(&to_charlist/1)
warning_apps = Enum.sort(umbrella_apps)
try do
:dialyzer.run(
analysis_type: :incremental,
warnings: [
:unmatched_returns,
:error_handling
],
# plt_location: ~c".",
# plt_prefix: ~c"emqx_dialyzer",
warning_files: warning_files,
warning_files_rec: warning_files,
# apps: umbrella_apps ++ dep_apps,
# warning_apps: warning_apps,
get_warnings: false,
files: files,
files_rec: files
)
catch
{:dialyzer_error, msg} ->
{:dialyzer_error, to_string(msg)}
err ->
{:throw, err}
end
|> IO.inspect(limit: :infinity)
end
defp resolve_apps() do
base_apps = MapSet.new([:erts, :crypto])
# excluded_apps = MapSet.new([:elixir])
excluded_apps = MapSet.new()
acc = %{
umbrella_apps: [],
dep_apps: base_apps
}
Mix.Dep.Umbrella.loaded()
|> Enum.reduce(acc, fn dep, acc ->
# IO.inspect(dep)
props = dep.opts[:app_properties]
optional_apps = Keyword.get(props, :optional_applications, [])
apps = Keyword.get(props, :applications, [])
included_apps = Keyword.get(props, :included_applications, [])
dep_apps = MapSet.new(optional_apps ++ apps ++ included_apps)
acc
|> Map.update!(:umbrella_apps, & [dep.app | &1])
|> Map.update!(:dep_apps, & MapSet.union(&1, dep_apps))
end)
|> then(fn acc ->
dep_apps =
acc.dep_apps
|> MapSet.difference(MapSet.new(acc.umbrella_apps))
|> MapSet.difference(excluded_apps)
|> Enum.reduce(MapSet.new(), &find_nested_apps/2)
|> MapSet.difference(excluded_apps)
|> Enum.filter(&app_present?/1)
%{acc | dep_apps: dep_apps}
end)
end
defp app_present?(app) do
match?({:ok, _}, ebin_dir(app))
end
defp find_nested_apps(app, seen) do
if MapSet.member?(seen, app) do
seen
else
seen = MapSet.put(seen, app)
apps = case :application.get_key(app, :applications) do
{:ok, apps} -> apps
:undefined -> []
end
included_apps = case :application.get_key(app, :included_applications) do
{:ok, apps} -> apps
:undefined -> []
end
optional_apps = case :application.get_key(app, :optional_applications) do
{:ok, apps} -> apps
:undefined -> []
end
Enum.reduce(apps ++ included_apps, seen, &find_nested_apps/2)
end
end
defp resolve_files(app) do
with {:ok, dir} <- ebin_dir(app) do
Mix.Utils.extract_files([dir], [:beam])
else
_ -> []
end
end
defp ebin_dir(app) do
with dir when is_list(dir) <- :code.lib_dir(app, :ebin),
dir = to_string(dir),
true <- File.dir?(dir) || {:error, :not_a_dir} do
{:ok, to_string(dir)}
else
error ->
Mix.shell().info(IO.ANSI.format([
[:yellow,
"Unknown application: #{app}; error: #{inspect(error)}",
"; if this is is an optional application, ignore."
],
]))
:error
end
end
end

122
apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex Normal file
View File

@ -0,0 +1,122 @@
defmodule Mix.Tasks.Emqx.Eunit do
use Mix.Task
alias Mix.Tasks.Emqx.Ct, as: ECt
# todo: invoke the equivalent of `make merge-config` as a requirement...
@requirements ["compile", "loadpaths"]
@impl true
def run(args) do
Mix.debug(true)
IO.inspect(args)
Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1)
ECt.ensure_whole_emqx_project_is_loaded!()
ECt.unload_emqx_applications!()
{_, 0} = System.cmd("epmd", ["-daemon"])
node_name = :"test@127.0.0.1"
:net_kernel.start([node_name, :longnames])
# unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect
# the version without the `-test` suffix.
System.fetch_env!("PROFILE")
|> String.replace_suffix("-test", "")
|> then(& System.put_env("PROFILE", &1))
args
|> parse_args!()
|> discover_tests()
|> :eunit.test(
verbose: true,
print_depth: 100
)
|> case do
:ok -> :ok
:error -> Mix.raise("errors found in tests")
end
end
defp add_to_path_and_cache(lib_name) do
:code.lib_dir()
|> Path.join("#{lib_name}-*")
|> Path.wildcard()
|> hd()
|> Path.join("ebin")
|> to_charlist()
|> :code.add_path(:cache)
end
defp parse_args!(args) do
{opts, _rest} = OptionParser.parse!(
args,
strict: [
cases: :string,
modules: :string,
]
)
cases =
opts
|> get_name_list(:cases)
|> Enum.flat_map(&resolve_test_fns!/1)
modules =
opts
|> get_name_list(:modules)
|> Enum.map(&String.to_atom/1)
%{
cases: cases,
modules: modules,
}
end
defp get_name_list(opts, key) do
opts
|> Keyword.get(key, "")
|> String.split(",", trim: true)
end
defp resolve_test_fns!(mod_fn_str) do
{mod, fun} = case String.split(mod_fn_str, ":") do
[mod, fun] ->
{String.to_atom(mod), String.to_atom(fun)}
_ ->
Mix.raise("Bad test case spec; must of `MOD:FUN` form. Got: #{mod_fn_str}`")
end
if not has_test_case?(mod, fun) do
Mix.raise("Module #{mod} does not export test case #{fun}")
end
if to_string(fun) =~ ~r/_test_$/ do
apply(mod, fun, [])
else
[Function.capture(mod, fun, 0)]
end
end
defp has_test_case?(mod, fun) do
try do
mod.module_info(:functions)
|> Enum.find(& &1 == {fun, 0})
|> then(& !! &1)
rescue
UndefinedFunctionError -> false
end
end
defp discover_tests(%{cases: [], modules: []} = _opts) do
Mix.Dep.Umbrella.cached()
|> Enum.map(& {:application, &1.app})
end
defp discover_tests(%{cases: cases, modules: modules}) do
Enum.concat(
[
cases,
Enum.map(modules, & {:module, &1})
]
)
end
end

2
lib/mix/tasks/emqx.proper.ex → apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex
View File

@ -1,8 +1,6 @@
defmodule Mix.Tasks.Emqx.Proper do
use Mix.Task
# Code.require_file("emqx.ct.ex", __DIR__)
alias Mix.Tasks.Emqx.Ct, as: ECt
# todo: invoke the equivalent of `make merge-config` as a requirement...

26
apps/emqx_mix_utils/mix.exs Normal file
View File

@ -0,0 +1,26 @@
defmodule EMQXMixUtils.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_mix_utils,
version: "0.1.0",
build_path: "../../_build",
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[]
end
end

3
apps/emqx_modules/mix.exs
View File

@ -26,7 +26,8 @@ defmodule EMQXModules.MixProject do
{:emqx, in_umbrella: true},
{:emqx_ctl, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_conf, in_umbrella: true}
{:emqx_conf, in_umbrella: true},
UMP.common_dep(:observer_cli)
]
end
end

3
apps/emqx_rule_engine/mix.exs
View File

@ -22,7 +22,7 @@ defmodule EMQXRuleEngine.MixProject do
end
def deps() do
[
UMP.jq_dep() ++ [
{:emqx, in_umbrella: true},
{:emqx_ctl, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
@ -31,6 +31,7 @@ defmodule EMQXRuleEngine.MixProject do
{:emqx_bridge, in_umbrella: true},
UMP.common_dep(:rulesql),
UMP.common_dep(:emqtt),
UMP.common_dep(:uuid),
]
end
end

1
apps/emqx_s3/mix.exs
View File

@ -26,6 +26,7 @@ defmodule EMQXS3.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
UMP.common_dep(:gproc),
UMP.common_dep(:ehttpc),

2
apps/emqx_schema_registry/mix.exs
View File

@ -28,7 +28,7 @@ defmodule EMQXSchemaRegistry.MixProject do
{:emqx_rule_engine, in_umbrella: true},
{:erlavro, github: "emqx/erlavro", tag: "2.10.0"},
{:jesse, github: "emqx/jesse", tag: "1.8.0"},
UMP.common_dep(:gpb),
UMP.common_dep(:gpb, runtime: true),
]
end
end

1
apps/emqx_telemetry/mix.exs
View File

@ -26,6 +26,7 @@ defmodule EMQXTelemetry.MixProject do
def deps() do
[
{:emqx_mix_utils, in_umbrella: true, runtime: false},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_conf, in_umbrella: true}

58
lib/mix/tasks/emqx.eunit.ex
View File

@ -1,58 +0,0 @@
defmodule Mix.Tasks.Emqx.Eunit do
use Mix.Task
# Code.require_file("emqx.ct.ex", __DIR__)
alias Mix.Tasks.Emqx.Ct, as: ECt
# todo: invoke the equivalent of `make merge-config` as a requirement...
@requirements ["compile", "loadpaths"]
@impl true
def run(args) do
Mix.debug(true)
IO.inspect(args)
Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1)
ECt.ensure_whole_emqx_project_is_loaded!()
ECt.unload_emqx_applications!()
{_, 0} = System.cmd("epmd", ["-daemon"])
node_name = :"test@127.0.0.1"
:net_kernel.start([node_name, :longnames])
# unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect
# the version without the `-test` suffix.
System.fetch_env!("PROFILE")
|> String.replace_suffix("-test", "")
|> then(& System.put_env("PROFILE", &1))
discover_tests()
|> :eunit.test(
verbose: true,
print_depth: 100
)
|> case do
:ok -> :ok
:error -> Mix.raise("errors found in tests")
end
end
defp add_to_path_and_cache(lib_name) do
:code.lib_dir()
|> Path.join("#{lib_name}-*")
|> Path.wildcard()
|> hd()
|> Path.join("ebin")
|> to_charlist()
|> :code.add_path(:cache)
end
## TODO: allow filtering modules and test names
defp discover_tests() do
Mix.Dep.Umbrella.cached()
|> Enum.map(& {:application, &1.app})
end
end

74
mix.exs
View File

@ -40,11 +40,7 @@ defmodule EMQXUmbrella.MixProject do
if new_mix_build?() do
[
# TODO: these lines will be uncommented when we switch to using mix as the manager
# for all umbrella apps.
apps_path: "apps",
apps:
applications(profile_info.release_type, profile_info.edition_type) |> Keyword.keys(),
erlc_options: erlc_options(profile_info, version),
version: version,
deps: deps(profile_info, version),
@ -160,7 +156,7 @@ defmodule EMQXUmbrella.MixProject do
{:ssl_verify_fun, "1.1.7", override: true},
common_dep(:rfc3339),
common_dep(:bcrypt),
{:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
common_dep(:uuid),
{:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
common_dep(:ra),
{:mimerl, "1.2.0", override: true}
@ -169,12 +165,22 @@ defmodule EMQXUmbrella.MixProject do
def extra_release_apps() do
[
{:redbug, github: "emqx/redbug", tag: "2.0.10"},
{:observer_cli, "1.7.1"},
{:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"}
common_dep(:redbug),
common_dep(:observer_cli),
common_dep(:system_monitor)
]
end
def common_dep(dep_name, overrides) do
case common_dep(dep_name) do
{^dep_name, opts} ->
{dep_name, Keyword.merge(opts, overrides)}
{^dep_name, tag, opts} when is_binary(tag) ->
{dep_name, tag, Keyword.merge(opts, overrides)}
end
end
def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true}
def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true}
def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}
@ -186,7 +192,12 @@ defmodule EMQXUmbrella.MixProject do
def common_dep(:ranch), do: {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true}
def common_dep(:ehttpc), do: {:ehttpc, github: "emqx/ehttpc", tag: "0.4.14", override: true}
def common_dep(:jiffy), do: {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true}
def common_dep(:grpc), do: {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}
def common_dep(:grpc),
do:
{:grpc,
github: "emqx/grpc-erl", tag: "0.6.12", override: true, system_env: emqx_app_system_env()}
def common_dep(:cowboy), do: {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}
def common_dep(:jsone), do: {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true}
def common_dep(:ecpool), do: {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true}
@ -207,6 +218,13 @@ defmodule EMQXUmbrella.MixProject do
def common_dep(:esasl), do: {:esasl, github: "emqx/esasl", tag: "0.2.1"}
def common_dep(:gen_rpc), do: {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true}
def common_dep(:system_monitor),
do: {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"}
def common_dep(:uuid), do: {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}
def common_dep(:redbug), do: {:redbug, github: "emqx/redbug", tag: "2.0.10"}
def common_dep(:observer_cli), do: {:observer_cli, "1.7.1"}
def common_dep(:jose),
do: {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2", override: true}
@ -252,7 +270,7 @@ defmodule EMQXUmbrella.MixProject do
github: "kafka4beam/snabbkaffe",
tag: "1.0.10",
override: true,
system_env: emqx_app_system_env(profile_info(), pkg_vsn())
system_env: emqx_app_system_env()
}
###############################################################################################
@ -293,6 +311,7 @@ defmodule EMQXUmbrella.MixProject do
false
end
end)
|> Enum.reject(fn {app, _} -> app == :emqx_mix_utils end)
|> Enum.reject(fn {app, _} -> app in excluded_apps end)
end
@ -416,16 +435,24 @@ defmodule EMQXUmbrella.MixProject do
)
end
###############################################################################################
# END DEPRECATED FOR MIX BLOCK
###############################################################################################
def emqx_app_system_env(profile_info, version) do
erlc_options(profile_info, version)
|> dump_as_erl()
|> then(&[{"ERL_COMPILER_OPTIONS", &1}])
end
def emqx_app_system_env() do
k = {__MODULE__, :emqx_app_system_env}
get_memoized(k, fn ->
emqx_app_system_env(profile_info(), pkg_vsn())
end)
end
###############################################################################################
# END DEPRECATED FOR MIX BLOCK
###############################################################################################
defp erlc_options(%{edition_type: edition_type}, version) do
[
:debug_info,
@ -542,6 +569,7 @@ defmodule EMQXUmbrella.MixProject do
} = check_profile!()
base_steps = [
&merge_config/1,
&make_docs/1,
:assemble,
&create_RELEASES/1,
@ -780,6 +808,12 @@ defmodule EMQXUmbrella.MixProject do
# Custom Steps
#############################################################################
# Gathers i18n files and merge them before producing docs and schemas.
defp merge_config(release) do
{_, 0} = System.cmd("bash", ["-c", "./scripts/merge-config.escript"])
release
end
defp make_docs(release) do
profile = System.get_env("MIX_ENV")
os_cmd("build", [profile, "docs"])
@ -1104,7 +1138,7 @@ defmodule EMQXUmbrella.MixProject do
defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
defp emqx_schema_mod(:community), do: :emqx_conf_schema
defp jq_dep() do
def jq_dep() do
if enable_jq?(),
do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}],
else: []
@ -1267,7 +1301,8 @@ defmodule EMQXUmbrella.MixProject do
[
ct: &do_ct/1,
eunit: &do_eunit/1,
proper: &do_proper/1
proper: &do_proper/1,
dialyzer: &do_dialyzer/1
]
end
@ -1278,24 +1313,25 @@ defmodule EMQXUmbrella.MixProject do
ensure_test_mix_env!()
set_test_env!(true)
Code.require_file("lib/mix/tasks/emqx.ct.ex")
Mix.Task.run("emqx.ct", args)
end
defp do_eunit(args) do
ensure_test_mix_env!()
set_test_env!(true)
Code.require_file("lib/mix/tasks/emqx.eunit.ex")
Mix.Task.run("emqx.eunit", args)
end
defp do_proper(args) do
ensure_test_mix_env!()
set_test_env!(true)
Code.require_file("lib/mix/tasks/emqx.proper.ex")
Mix.Task.run("emqx.proper", args)
end
defp do_dialyzer(args) do
Mix.Task.run("emqx.dialyzer", args)
end
defp ensure_test_mix_env!() do
Mix.env()
|> to_string()