From b91515b1311d14156a62044c264b8e477c78a8db Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 5 Jul 2024 11:24:48 -0300 Subject: [PATCH 01/14] fix(schema registry mix): gpb is a runtime dep --- apps/emqx_schema_registry/mix.exs | 2 +- mix.exs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_schema_registry/mix.exs b/apps/emqx_schema_registry/mix.exs index 0abffad6f..d286b3292 100644 --- a/apps/emqx_schema_registry/mix.exs +++ b/apps/emqx_schema_registry/mix.exs @@ -28,7 +28,7 @@ defmodule EMQXSchemaRegistry.MixProject do {:emqx_rule_engine, in_umbrella: true}, {:erlavro, github: "emqx/erlavro", tag: "2.10.0"}, {:jesse, github: "emqx/jesse", tag: "1.8.0"}, - UMP.common_dep(:gpb), + UMP.common_dep(:gpb, runtime: true), ] end end diff --git a/mix.exs b/mix.exs index 9e9e778b0..43a1923c0 100644 --- a/mix.exs +++ b/mix.exs @@ -175,6 +175,16 @@ defmodule EMQXUmbrella.MixProject do ] end + def common_dep(dep_name, overrides) do + case common_dep(dep_name) do + {^dep_name, opts} -> + {dep_name, Keyword.merge(opts, overrides)} + + {^dep_name, tag, opts} when is_binary(tag) -> + {dep_name, tag, Keyword.merge(opts, overrides)} + end + end + def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true} def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true} def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true} From 5279ad76be09711d0f1143335a9c4ca531508319 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 5 Jul 2024 11:32:52 -0300 Subject: [PATCH 02/14] fix(grpc compiler): unload apps to avoid side effects --- apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex index 615f98c29..333c1ae29 100644 --- a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex +++ b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex @@ -46,6 +46,9 @@ defmodule Mix.Tasks.Compile.Grpc do write_manifest(manifest(), manifest_data) {:noop, []} + after + Application.unload(:gpb) + Application.unload(:syntax_tools) end defp compile_pb(proto_src, context) do From 818070ad44f6605fbba55ea4bfbb87f7eec111b3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 5 Jul 2024 17:07:16 -0300 Subject: [PATCH 03/14] test(mix): add integration test path --- apps/emqx/mix.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs index 279332bfe..2759326bc 100644 --- a/apps/emqx/mix.exs +++ b/apps/emqx/mix.exs @@ -53,6 +53,15 @@ defmodule EMQX.MixProject do ] ++ UMP.quicer_dep() end + defp erlc_paths() do + paths = UMP.erlc_paths() + if UMP.test_env?() do + ["integration_test" | paths] + else + paths + end + end + defp extra_dirs() do dirs = ["src", "etc"] if UMP.test_env?() do From 48e604bda84084e627f9cf5fb03947b835dc1105 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 5 Jul 2024 17:47:26 -0300 Subject: [PATCH 04/14] fix(mix grpc): include default erlc options --- apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex index 333c1ae29..0a33f62b7 100644 --- a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex +++ b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex @@ -1,6 +1,8 @@ defmodule Mix.Tasks.Compile.Grpc do use Mix.Task.Compiler + alias EMQXUmbrella.MixProject, as: UMP + @recursive true @manifest_vsn 1 @manifest "compile.grpc" @@ -98,7 +100,7 @@ defmodule Mix.Tasks.Compile.Grpc do :return_errors, i: to_charlist(gpb_include_dir), outdir: to_charlist(ebin_path) - ] + ] ++ UMP.erlc_options() ) # todo: error handling & logging case compile_res do From f3c6d10f767f092b15558c639e8a03de5ef95fae Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 9 Jul 2024 10:04:11 -0300 Subject: [PATCH 05/14] fix(mix): fix compile paths and deps --- apps/emqx/mix.exs | 4 +-- apps/emqx_bridge_hstreamdb/mix.exs | 3 ++- apps/emqx_dashboard_sso/mix.exs | 3 ++- apps/emqx_machine/mix.exs | 2 ++ apps/emqx_modules/mix.exs | 3 ++- apps/emqx_rule_engine/mix.exs | 3 ++- mix.exs | 42 ++++++++++++++++++++++-------- 7 files changed, 43 insertions(+), 17 deletions(-) diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs index 2759326bc..fd9aca57e 100644 --- a/apps/emqx/mix.exs +++ b/apps/emqx/mix.exs @@ -8,7 +8,7 @@ defmodule EMQX.MixProject do app: :emqx, version: "0.1.0", build_path: "../../_build", - erlc_paths: UMP.erlc_paths(), + erlc_paths: erlc_paths(), erlc_options: [ {:i, "src"} | UMP.erlc_options() @@ -37,7 +37,7 @@ defmodule EMQX.MixProject do ## FIXME!!! go though emqx.app.src and add missing stuff... [ {:emqx_utils, in_umbrella: true}, - # {:emqx_ds_backends, in_umbrella: true}, + {:emqx_ds_backends, in_umbrella: true}, UMP.common_dep(:gproc), UMP.common_dep(:gen_rpc), diff --git a/apps/emqx_bridge_hstreamdb/mix.exs b/apps/emqx_bridge_hstreamdb/mix.exs index 87f8b80c2..8c21da7aa 100644 --- a/apps/emqx_bridge_hstreamdb/mix.exs +++ b/apps/emqx_bridge_hstreamdb/mix.exs @@ -24,7 +24,8 @@ defmodule EMQXBridgeHstreamdb.MixProject do def deps() do [ {:hstreamdb_erl, - github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"}, + github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1", + system_env: UMP.emqx_app_system_env()}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_connector, in_umbrella: true, runtime: false}, diff --git a/apps/emqx_dashboard_sso/mix.exs b/apps/emqx_dashboard_sso/mix.exs index 13a44f61f..94b99aceb 100644 --- a/apps/emqx_dashboard_sso/mix.exs +++ b/apps/emqx_dashboard_sso/mix.exs @@ -26,7 +26,8 @@ defmodule EMQXDashboardSso.MixProject do {:emqx_ctl, in_umbrella: true}, {:emqx_ldap, in_umbrella: true}, {:emqx_dashboard, in_umbrella: true}, - {:esaml, github: "emqx/esaml", tag: "v1.1.3"} + {:esaml, github: "emqx/esaml", tag: "v1.1.3"}, + {:oidcc, github: "emqx/oidcc", tag: "v3.2.0-1"}, ] end end diff --git a/apps/emqx_machine/mix.exs b/apps/emqx_machine/mix.exs index 37fb59d5d..4ea0eef79 100644 --- a/apps/emqx_machine/mix.exs +++ b/apps/emqx_machine/mix.exs @@ -30,6 +30,8 @@ defmodule EMQXMachine.MixProject do {:emqx_dashboard, in_umbrella: true, runtime: false}, {:emqx_management, in_umbrella: true, runtime: false}, UMP.common_dep(:covertool), + UMP.common_dep(:system_monitor), + UMP.common_dep(:redbug), ] end end diff --git a/apps/emqx_modules/mix.exs b/apps/emqx_modules/mix.exs index f271d7729..b88171536 100644 --- a/apps/emqx_modules/mix.exs +++ b/apps/emqx_modules/mix.exs @@ -26,7 +26,8 @@ defmodule EMQXModules.MixProject do {:emqx, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, - {:emqx_conf, in_umbrella: true} + {:emqx_conf, in_umbrella: true}, + UMP.common_dep(:observer_cli) ] end end diff --git a/apps/emqx_rule_engine/mix.exs b/apps/emqx_rule_engine/mix.exs index 0db98f21a..6463a8532 100644 --- a/apps/emqx_rule_engine/mix.exs +++ b/apps/emqx_rule_engine/mix.exs @@ -22,7 +22,7 @@ defmodule EMQXRuleEngine.MixProject do end def deps() do - [ + UMP.jq_dep() ++ [ {:emqx, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, @@ -31,6 +31,7 @@ defmodule EMQXRuleEngine.MixProject do {:emqx_bridge, in_umbrella: true}, UMP.common_dep(:rulesql), UMP.common_dep(:emqtt), + UMP.common_dep(:uuid), ] end end diff --git a/mix.exs b/mix.exs index 43a1923c0..9c1a8a5f5 100644 --- a/mix.exs +++ b/mix.exs @@ -160,7 +160,7 @@ defmodule EMQXUmbrella.MixProject do {:ssl_verify_fun, "1.1.7", override: true}, common_dep(:rfc3339), common_dep(:bcrypt), - {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, + common_dep(:uuid), {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, common_dep(:ra), {:mimerl, "1.2.0", override: true} @@ -169,9 +169,9 @@ defmodule EMQXUmbrella.MixProject do def extra_release_apps() do [ - {:redbug, github: "emqx/redbug", tag: "2.0.10"}, - {:observer_cli, "1.7.1"}, - {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"} + common_dep(:redbug), + common_dep(:observer_cli), + common_dep(:system_monitor) ] end @@ -196,7 +196,12 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:ranch), do: {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true} def common_dep(:ehttpc), do: {:ehttpc, github: "emqx/ehttpc", tag: "0.4.14", override: true} def common_dep(:jiffy), do: {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true} - def common_dep(:grpc), do: {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true} + + def common_dep(:grpc), + do: + {:grpc, + github: "emqx/grpc-erl", tag: "0.6.12", override: true, system_env: emqx_app_system_env()} + def common_dep(:cowboy), do: {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true} def common_dep(:jsone), do: {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true} def common_dep(:ecpool), do: {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true} @@ -217,6 +222,13 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:esasl), do: {:esasl, github: "emqx/esasl", tag: "0.2.1"} def common_dep(:gen_rpc), do: {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true} + def common_dep(:system_monitor), + do: {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"} + + def common_dep(:uuid), do: {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true} + def common_dep(:redbug), do: {:redbug, github: "emqx/redbug", tag: "2.0.10"} + def common_dep(:observer_cli), do: {:observer_cli, "1.7.1"} + def common_dep(:jose), do: {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2", override: true} @@ -262,7 +274,7 @@ defmodule EMQXUmbrella.MixProject do github: "kafka4beam/snabbkaffe", tag: "1.0.10", override: true, - system_env: emqx_app_system_env(profile_info(), pkg_vsn()) + system_env: emqx_app_system_env() } ############################################################################################### @@ -426,16 +438,24 @@ defmodule EMQXUmbrella.MixProject do ) end - ############################################################################################### - # END DEPRECATED FOR MIX BLOCK - ############################################################################################### - def emqx_app_system_env(profile_info, version) do erlc_options(profile_info, version) |> dump_as_erl() |> then(&[{"ERL_COMPILER_OPTIONS", &1}]) end + def emqx_app_system_env() do + k = {__MODULE__, :emqx_app_system_env} + + get_memoized(k, fn -> + emqx_app_system_env(profile_info(), pkg_vsn()) + end) + end + + ############################################################################################### + # END DEPRECATED FOR MIX BLOCK + ############################################################################################### + defp erlc_options(%{edition_type: edition_type}, version) do [ :debug_info, @@ -1114,7 +1134,7 @@ defmodule EMQXUmbrella.MixProject do defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema defp emqx_schema_mod(:community), do: :emqx_conf_schema - defp jq_dep() do + def jq_dep() do if enable_jq?(), do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}], else: [] From 21313c766da91ae92481b0c546d10bc26871d594 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 9 Jul 2024 10:04:38 -0300 Subject: [PATCH 06/14] ci: add dialyzer mix task --- lib/mix/tasks/emqx.dialyzer.ex | 164 +++++++++++++++++++++++++++++++++ mix.exs | 8 +- 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/emqx.dialyzer.ex diff --git a/lib/mix/tasks/emqx.dialyzer.ex b/lib/mix/tasks/emqx.dialyzer.ex new file mode 100644 index 000000000..9a2a5ad16 --- /dev/null +++ b/lib/mix/tasks/emqx.dialyzer.ex @@ -0,0 +1,164 @@ +defmodule Mix.Tasks.Emqx.Dialyzer do + use Mix.Task + + alias EMQXUmbrella.MixProject, as: UMP + + if UMP.new_mix_build?() do + Code.require_file("emqx.ct.ex", __DIR__) + end + + alias Mix.Tasks.Emqx.Ct, as: ECt + + @requirements ["compile", "loadpaths"] + + @excluded_mods ( + [ + :emqx_exproto_v_1_connection_unary_handler_bhvr, + :emqx_exproto_v_1_connection_handler_client, + :emqx_exproto_v_1_connection_handler_bhvr, + :emqx_exproto_v_1_connection_adapter_client, + :emqx_exproto_v_1_connection_adapter_bhvr, + :emqx_exproto_v_1_connection_unary_handler_client, + :emqx_exhook_v_2_hook_provider_client, + :emqx_exhook_v_2_hook_provider_bhvr, + Mix.Tasks.Compile.Grpc, + Mix.Tasks.Compile.CopySrcs, + ] + |> MapSet.new(&to_string/1) + ) + + @impl true + def run(_args) do + ECt.add_to_path_and_cache(:dialyzer) + + %{ + umbrella_apps: umbrella_apps, + dep_apps: dep_apps + } = resolve_apps() + umbrella_files = Enum.flat_map(umbrella_apps, & resolve_files/1) + dep_files = Enum.flat_map(dep_apps, & resolve_files/1) + files = + (umbrella_files ++ dep_files) + |> Enum.reject(fn path -> + name = Path.basename(path, ".beam") + MapSet.member?(@excluded_mods, name) + end) + |> Enum.map(&to_charlist/1) + warning_files = + umbrella_files + |> Enum.reject(fn path -> + name = Path.basename(path, ".beam") + MapSet.member?(@excluded_mods, name) + end) + |> Enum.map(&to_charlist/1) + warning_apps = Enum.sort(umbrella_apps) + + try do + :dialyzer.run( + analysis_type: :incremental, + warnings: [ + :unmatched_returns, + :error_handling + ], + # plt_location: ~c".", + # plt_prefix: ~c"emqx_dialyzer", + warning_files: warning_files, + warning_files_rec: warning_files, + # apps: umbrella_apps ++ dep_apps, + # warning_apps: warning_apps, + get_warnings: false, + files: files, + files_rec: files + ) + catch + {:dialyzer_error, msg} -> + {:dialyzer_error, to_string(msg)} + err -> + {:throw, err} + end + |> IO.inspect(limit: :infinity) + end + + defp resolve_apps() do + base_apps = MapSet.new([:erts, :crypto]) + # excluded_apps = MapSet.new([:elixir]) + excluded_apps = MapSet.new() + acc = %{ + umbrella_apps: [], + dep_apps: base_apps + } + + Mix.Dep.Umbrella.loaded() + |> Enum.reduce(acc, fn dep, acc -> + # IO.inspect(dep) + props = dep.opts[:app_properties] + optional_apps = Keyword.get(props, :optional_applications, []) + apps = Keyword.get(props, :applications, []) + included_apps = Keyword.get(props, :included_applications, []) + dep_apps = MapSet.new(optional_apps ++ apps ++ included_apps) + acc + |> Map.update!(:umbrella_apps, & [dep.app | &1]) + |> Map.update!(:dep_apps, & MapSet.union(&1, dep_apps)) + end) + |> then(fn acc -> + dep_apps = + acc.dep_apps + |> MapSet.difference(MapSet.new(acc.umbrella_apps)) + |> MapSet.difference(excluded_apps) + |> Enum.reduce(MapSet.new(), &find_nested_apps/2) + |> MapSet.difference(excluded_apps) + |> Enum.filter(&app_present?/1) + %{acc | dep_apps: dep_apps} + end) + end + + defp app_present?(app) do + match?({:ok, _}, ebin_dir(app)) + end + + defp find_nested_apps(app, seen) do + if MapSet.member?(seen, app) do + seen + else + seen = MapSet.put(seen, app) + apps = case :application.get_key(app, :applications) do + {:ok, apps} -> apps + :undefined -> [] + end + included_apps = case :application.get_key(app, :included_applications) do + {:ok, apps} -> apps + :undefined -> [] + end + optional_apps = case :application.get_key(app, :optional_applications) do + {:ok, apps} -> apps + :undefined -> [] + end + Enum.reduce(apps ++ included_apps, seen, &find_nested_apps/2) + end + end + + defp resolve_files(app) do + with {:ok, dir} <- ebin_dir(app) do + Mix.Utils.extract_files([dir], [:beam]) + else + _ -> [] + end + end + + defp ebin_dir(app) do + with dir when is_list(dir) <- :code.lib_dir(app, :ebin), + dir = to_string(dir), + true <- File.dir?(dir) || {:error, :not_a_dir} do + {:ok, to_string(dir)} + else + error -> + Mix.shell().info(IO.ANSI.format([ + [:yellow, + "Unknown application: #{app}; error: #{inspect(error)}", + "; if this is is an optional application, ignore." + ], + ])) + :error + end + end +end diff --git a/mix.exs b/mix.exs index 9c1a8a5f5..cb508643e 100644 --- a/mix.exs +++ b/mix.exs @@ -1297,7 +1297,8 @@ defmodule EMQXUmbrella.MixProject do [ ct: &do_ct/1, eunit: &do_eunit/1, - proper: &do_proper/1 + proper: &do_proper/1, + dialyzer: &do_dialyzer/1 ] end @@ -1326,6 +1327,11 @@ defmodule EMQXUmbrella.MixProject do Mix.Task.run("emqx.proper", args) end + defp do_dialyzer(args) do + Code.require_file("lib/mix/tasks/emqx.dialyzer.ex") + Mix.Task.run("emqx.dialyzer", args) + end + defp ensure_test_mix_env!() do Mix.env() |> to_string() From 066fd0481b1fc0d237eae9dc5c25c71e11701a4d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 9 Jul 2024 16:00:49 -0300 Subject: [PATCH 07/14] feat(mix): compile asn1 files --- .../lib/mix/tasks/compile.asn1.ex | 82 +++++++++++++++++++ apps/emqx_durable_storage/mix.exs | 9 +- 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex diff --git a/apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex b/apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex new file mode 100644 index 000000000..afec3f62c --- /dev/null +++ b/apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex @@ -0,0 +1,82 @@ +defmodule Mix.Tasks.Compile.Asn1 do + use Mix.Task.Compiler + + @recursive true + @manifest_vsn 1 + @manifest "compile.asn1" + # TODO: use manifest to track generated files? + + @impl true + def manifests(), do: [manifest()] + defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest) + + @impl true + def run(_args) do + add_to_path_and_cache(:asn1) + + Mix.Project.get!() + config = Mix.Project.config() + app_root = File.cwd!() + + asn1_srcs = config[:asn1_srcs] || [] + manifest_data = read_manifest(manifest()) + manifest_modified_time = Mix.Utils.last_modified(manifest()) + Enum.each(asn1_srcs, &compile(&1, app_root, manifest_modified_time)) + write_manifest(manifest(), manifest_data) + + {:noop, []} + end + + defp compile(src, app_root, manifest_modified_time) do + %{ + src: src_path, + compile_opts: compile_opts + } = src + src_path = + app_root + |> Path.join(src_path) + |> Path.expand() + if stale?(src_path, manifest_modified_time) do + Mix.shell().info("compiling asn1 file: #{src_path}") + :ok = :asn1ct.compile(to_charlist(src_path), compile_opts) + else + Mix.shell().info("file is up to date, not compiling: #{src_path}") + end + end + + defp stale?(file, manifest_modified_time) do + with true <- File.exists?(file), + false <- Mix.Utils.stale?([file], [manifest_modified_time]) do + false + else + _ -> true + end + end + + defp read_manifest(file) do + try do + file |> File.read!() |> :erlang.binary_to_term() + rescue + _ -> %{} + else + {@manifest_vsn, data} when is_map(data) -> data + _ -> %{} + end + end + + defp write_manifest(file, data) do + Mix.shell().info("writing manifest #{file}") + File.mkdir_p!(Path.dirname(file)) + File.write!(file, :erlang.term_to_binary({@manifest_vsn, data})) + end + + def add_to_path_and_cache(lib_name) do + :code.lib_dir() + |> Path.join("#{lib_name}-*") + |> Path.wildcard() + |> hd() + |> Path.join("ebin") + |> to_charlist() + |> :code.add_path(:cache) + end +end diff --git a/apps/emqx_durable_storage/mix.exs b/apps/emqx_durable_storage/mix.exs index fd93f3bf5..4cb1a9b35 100644 --- a/apps/emqx_durable_storage/mix.exs +++ b/apps/emqx_durable_storage/mix.exs @@ -7,9 +7,14 @@ defmodule EMQXDurableStorage.MixProject do app: :emqx_durable_storage, version: "0.1.0", build_path: "../../_build", - # config_path: "../../config/config.exs", + compilers: [:yecc, :leex, :elixir, :asn1, :erlang, :app], erlc_options: UMP.erlc_options(), - erlc_paths: UMP.erlc_paths(), + erlc_paths: ["gen_src" | UMP.erlc_paths()], + # used by our `compile.asn1` compiler + asn1_srcs: [ + %{src: "./asn.1/DurableMessage.asn", + compile_opts: [:per, :noobj, outdir: ~c"gen_src"]} + ], deps_path: "../../deps", lockfile: "../../mix.lock", elixir: "~> 1.14", From 70786d6aca85c9828ad8ded32bf1c395bed5f32f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Jul 2024 16:49:25 -0300 Subject: [PATCH 08/14] test: fix suite apps --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 2 ++ apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl | 1 + 3 files changed, 4 insertions(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 11271b329..3183564df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -48,6 +48,8 @@ init_per_suite(Conf) -> Apps = emqx_cth_suite:start( [ emqx_conf, + emqx_auth, + emqx_auth_mnesia, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, {emqx_gateway, ?CONF_DEFAULT} diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 936856c2e..e09d9356e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) -> Apps = emqx_cth_suite:start( [ emqx_conf, + emqx_auth_http, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()} diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 5b262158f..923b71ced 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) -> Apps = emqx_cth_suite:start( [ {emqx_conf, "authorization { no_match = deny, cache { enable = false } }"}, + emqx_auth_http, {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()} | emqx_gateway_test_utils:all_gateway_apps() ], From 39c82fbe89e2182563e5874ed0fa10b855058c49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Jul 2024 14:17:21 -0300 Subject: [PATCH 09/14] feat(mix): always run merge-config before release --- mix.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mix.exs b/mix.exs index cb508643e..24a9ec820 100644 --- a/mix.exs +++ b/mix.exs @@ -572,6 +572,7 @@ defmodule EMQXUmbrella.MixProject do } = check_profile!() base_steps = [ + &merge_config/1, &make_docs/1, :assemble, &create_RELEASES/1, @@ -810,6 +811,12 @@ defmodule EMQXUmbrella.MixProject do # Custom Steps ############################################################################# + # Gathers i18n files and merge them before producing docs and schemas. + defp merge_config(release) do + {_, 0} = System.cmd("bash", ["-c", "./scripts/merge-config.escript"]) + release + end + defp make_docs(release) do profile = System.get_env("MIX_ENV") os_cmd("build", [profile, "docs"]) From bbd51bdf18d6e4d62579bd7474c80e5184750c8d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Jul 2024 15:08:45 -0300 Subject: [PATCH 10/14] feat(mix ct): add support for specifying group paths --- lib/mix/tasks/emqx.ct.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/emqx.ct.ex b/lib/mix/tasks/emqx.ct.ex index ed75f3adc..d03ddb414 100644 --- a/lib/mix/tasks/emqx.ct.ex +++ b/lib/mix/tasks/emqx.ct.ex @@ -48,6 +48,7 @@ defmodule Mix.Tasks.Emqx.Ct do abort_if_missing_suites: true, auto_compile: false, suite: opts |> Map.fetch!(:suites) |> Enum.map(&to_charlist/1), + group: opts |> Map.fetch!(:group_paths) |> Enum.map(fn gp -> Enum.map(gp, &String.to_atom/1) end), testcase: opts |> Map.fetch!(:cases) |> Enum.map(&to_charlist/1), readable: 'true', name: node_name, @@ -198,11 +199,13 @@ defmodule Mix.Tasks.Emqx.Ct do args, strict: [ suites: :string, - groups: :string, + group_paths: :string, cases: :string]) - |> IO.inspect(label: :opts) suites = get_name_list(opts, :suites) - groups = get_name_list(opts, :groups) + group_paths = + opts + |> get_name_list(:group_paths) + |> Enum.map(& String.split(&1, ".", trim: true)) cases = get_name_list(opts, :cases) if suites == [] do @@ -211,7 +214,7 @@ defmodule Mix.Tasks.Emqx.Ct do %{ suites: suites, - groups: groups, + group_paths: group_paths, cases: cases } end From 9a003ee3cf6a644c06fde00b1fa70124cc5ab6f0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Jul 2024 15:28:01 -0300 Subject: [PATCH 11/14] feat(mix eunit): add support for filtering test cases --- lib/mix/tasks/emqx.eunit.ex | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/emqx.eunit.ex b/lib/mix/tasks/emqx.eunit.ex index a2c0dfe39..2ec0e7abc 100644 --- a/lib/mix/tasks/emqx.eunit.ex +++ b/lib/mix/tasks/emqx.eunit.ex @@ -1,7 +1,11 @@ defmodule Mix.Tasks.Emqx.Eunit do use Mix.Task - # Code.require_file("emqx.ct.ex", __DIR__) + alias EMQXUmbrella.MixProject, as: UMP + + if UMP.new_mix_build?() do + Code.require_file("emqx.ct.ex", __DIR__) + end alias Mix.Tasks.Emqx.Ct, as: ECt @@ -29,7 +33,9 @@ defmodule Mix.Tasks.Emqx.Eunit do |> String.replace_suffix("-test", "") |> then(& System.put_env("PROFILE", &1)) - discover_tests() + args + |> parse_args!() + |> discover_tests() |> :eunit.test( verbose: true, print_depth: 100 @@ -50,9 +56,73 @@ defmodule Mix.Tasks.Emqx.Eunit do |> :code.add_path(:cache) end - ## TODO: allow filtering modules and test names - defp discover_tests() do + defp parse_args!(args) do + {opts, _rest} = OptionParser.parse!( + args, + strict: [ + cases: :string, + modules: :string, + ] + ) + cases = + opts + |> get_name_list(:cases) + |> Enum.flat_map(&resolve_test_fns!/1) + modules = + opts + |> get_name_list(:modules) + |> Enum.map(&String.to_atom/1) + + %{ + cases: cases, + modules: modules, + } + end + + defp get_name_list(opts, key) do + opts + |> Keyword.get(key, "") + |> String.split(",", trim: true) + end + + defp resolve_test_fns!(mod_fn_str) do + {mod, fun} = case String.split(mod_fn_str, ":") do + [mod, fun] -> + {String.to_atom(mod), String.to_atom(fun)} + _ -> + Mix.raise("Bad test case spec; must of `MOD:FUN` form. Got: #{mod_fn_str}`") + end + if not has_test_case?(mod, fun) do + Mix.raise("Module #{mod} does not export test case #{fun}") + end + + if to_string(fun) =~ ~r/_test_$/ do + apply(mod, fun, []) + else + [Function.capture(mod, fun, 0)] + end + end + + defp has_test_case?(mod, fun) do + try do + mod.module_info(:functions) + |> Enum.find(& &1 == {fun, 0}) + |> then(& !! &1) + rescue + UndefinedFunctionError -> false + end + end + + defp discover_tests(%{cases: [], modules: []} = _opts) do Mix.Dep.Umbrella.cached() |> Enum.map(& {:application, &1.app}) end + defp discover_tests(%{cases: cases, modules: modules}) do + Enum.concat( + [ + cases, + Enum.map(modules, & {:module, &1}) + ] + ) + end end From 02a0ccfdd16cfcae358ebe5970d5a3a9ae3d1fa9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Jul 2024 15:39:32 -0300 Subject: [PATCH 12/14] ci: preparations for new mix build --- .github/workflows/.zipignore2 | 1 + Makefile | 8 +++++- apps/emqx/mix.exs | 1 + apps/emqx_auth/mix.exs | 1 + apps/emqx_bridge/mix.exs | 1 + apps/emqx_durable_storage/mix.exs | 1 + apps/emqx_exhook/mix.exs | 1 + apps/emqx_gateway/mix.exs | 1 + apps/emqx_gateway_exproto/mix.exs | 1 + apps/emqx_gcp_device/mix.exs | 1 + apps/emqx_license/mix.exs | 1 + .../lib/emqx/grpc/template/client.eex | 0 .../lib/emqx/grpc/template/service.eex | 0 .../lib/mix/tasks/compile.asn1.ex | 0 .../lib/mix/tasks/compile.copy_srcs.ex | 0 .../lib/mix/tasks/compile.grpc.ex | 0 .../emqx_mix_utils/lib}/mix/tasks/emqx.ct.ex | 0 .../lib}/mix/tasks/emqx.dialyzer.ex | 6 ----- .../lib}/mix/tasks/emqx.eunit.ex | 6 ----- .../lib}/mix/tasks/emqx.proper.ex | 2 -- apps/emqx_mix_utils/mix.exs | 26 +++++++++++++++++++ apps/emqx_s3/mix.exs | 1 + apps/emqx_telemetry/mix.exs | 1 + mix.exs | 7 +---- 24 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/.zipignore2 rename apps/{emqx_exhook => emqx_mix_utils}/lib/emqx/grpc/template/client.eex (100%) rename apps/{emqx_exhook => emqx_mix_utils}/lib/emqx/grpc/template/service.eex (100%) rename apps/{emqx_durable_storage => emqx_mix_utils}/lib/mix/tasks/compile.asn1.ex (100%) rename apps/{emqx => emqx_mix_utils}/lib/mix/tasks/compile.copy_srcs.ex (100%) rename apps/{emqx_exhook => emqx_mix_utils}/lib/mix/tasks/compile.grpc.ex (100%) rename {lib => apps/emqx_mix_utils/lib}/mix/tasks/emqx.ct.ex (100%) rename {lib => apps/emqx_mix_utils/lib}/mix/tasks/emqx.dialyzer.ex (97%) rename {lib => apps/emqx_mix_utils/lib}/mix/tasks/emqx.eunit.ex (95%) rename {lib => apps/emqx_mix_utils/lib}/mix/tasks/emqx.proper.ex (97%) create mode 100644 apps/emqx_mix_utils/mix.exs diff --git a/.github/workflows/.zipignore2 b/.github/workflows/.zipignore2 new file mode 100644 index 000000000..f0d0058a7 --- /dev/null +++ b/.github/workflows/.zipignore2 @@ -0,0 +1 @@ +*/.github/* diff --git a/Makefile b/Makefile index ae92feced..6aaaf58d7 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ CT_COVER_EXPORT_PREFIX ?= $(PROFILE) export REBAR_GIT_CLONE_OPTIONS += --depth=1 +ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar + .PHONY: default default: $(REBAR) $(PROFILE) @@ -58,8 +60,12 @@ ensure-mix-rebar3: $(REBAR) ensure-mix-rebar: $(REBAR) @mix local.rebar --if-missing --force + +.PHONY: elixir-common-deps +elixir-common-deps: $(ELIXIR_COMMON_DEPS) + .PHONY: mix-deps-get -mix-deps-get: $(ELIXIR_COMMON_DEPS) +mix-deps-get: elixir-common-deps @mix deps.get .PHONY: eunit diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs index fd9aca57e..c9de2b118 100644 --- a/apps/emqx/mix.exs +++ b/apps/emqx/mix.exs @@ -36,6 +36,7 @@ defmodule EMQX.MixProject do def deps() do ## FIXME!!! go though emqx.app.src and add missing stuff... [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx_utils, in_umbrella: true}, {:emqx_ds_backends, in_umbrella: true}, diff --git a/apps/emqx_auth/mix.exs b/apps/emqx_auth/mix.exs index aacc0bda6..e3438c27b 100644 --- a/apps/emqx_auth/mix.exs +++ b/apps/emqx_auth/mix.exs @@ -28,6 +28,7 @@ defmodule EMQXAuth.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true} ] diff --git a/apps/emqx_bridge/mix.exs b/apps/emqx_bridge/mix.exs index 8322b5606..dd6b91b94 100644 --- a/apps/emqx_bridge/mix.exs +++ b/apps/emqx_bridge/mix.exs @@ -28,6 +28,7 @@ defmodule EMQXBridge.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_resource, in_umbrella: true}, {:emqx_connector, in_umbrella: true}, diff --git a/apps/emqx_durable_storage/mix.exs b/apps/emqx_durable_storage/mix.exs index 4cb1a9b35..59626a679 100644 --- a/apps/emqx_durable_storage/mix.exs +++ b/apps/emqx_durable_storage/mix.exs @@ -33,6 +33,7 @@ defmodule EMQXDurableStorage.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx_utils, in_umbrella: true}, UMP.common_dep(:rocksdb), UMP.common_dep(:gproc), diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs index 4e72cfa24..5baf6dac4 100644 --- a/apps/emqx_exhook/mix.exs +++ b/apps/emqx_exhook/mix.exs @@ -36,6 +36,7 @@ defmodule EMQXExhook.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, UMP.common_dep(:grpc) diff --git a/apps/emqx_gateway/mix.exs b/apps/emqx_gateway/mix.exs index 7c4daeb46..fb1b48d40 100644 --- a/apps/emqx_gateway/mix.exs +++ b/apps/emqx_gateway/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXGateway.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, diff --git a/apps/emqx_gateway_exproto/mix.exs b/apps/emqx_gateway_exproto/mix.exs index e3564694d..558fe9ed7 100644 --- a/apps/emqx_gateway_exproto/mix.exs +++ b/apps/emqx_gateway_exproto/mix.exs @@ -34,6 +34,7 @@ defmodule EMQXGatewayExproto.MixProject do def deps() do test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: [] test_deps ++ [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_gateway, in_umbrella: true}, diff --git a/apps/emqx_gcp_device/mix.exs b/apps/emqx_gcp_device/mix.exs index 94e1dc077..cafdf6a12 100644 --- a/apps/emqx_gcp_device/mix.exs +++ b/apps/emqx_gcp_device/mix.exs @@ -27,6 +27,7 @@ defmodule EMQXGCPDevice.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_auth, in_umbrella: true}, UMP.common_dep(:jose), diff --git a/apps/emqx_license/mix.exs b/apps/emqx_license/mix.exs index e0d2937de..a01256dad 100644 --- a/apps/emqx_license/mix.exs +++ b/apps/emqx_license/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXLicense.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, diff --git a/apps/emqx_exhook/lib/emqx/grpc/template/client.eex b/apps/emqx_mix_utils/lib/emqx/grpc/template/client.eex similarity index 100% rename from apps/emqx_exhook/lib/emqx/grpc/template/client.eex rename to apps/emqx_mix_utils/lib/emqx/grpc/template/client.eex diff --git a/apps/emqx_exhook/lib/emqx/grpc/template/service.eex b/apps/emqx_mix_utils/lib/emqx/grpc/template/service.eex similarity index 100% rename from apps/emqx_exhook/lib/emqx/grpc/template/service.eex rename to apps/emqx_mix_utils/lib/emqx/grpc/template/service.eex diff --git a/apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex similarity index 100% rename from apps/emqx_durable_storage/lib/mix/tasks/compile.asn1.ex rename to apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex diff --git a/apps/emqx/lib/mix/tasks/compile.copy_srcs.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.copy_srcs.ex similarity index 100% rename from apps/emqx/lib/mix/tasks/compile.copy_srcs.ex rename to apps/emqx_mix_utils/lib/mix/tasks/compile.copy_srcs.ex diff --git a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex similarity index 100% rename from apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex rename to apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex diff --git a/lib/mix/tasks/emqx.ct.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex similarity index 100% rename from lib/mix/tasks/emqx.ct.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex diff --git a/lib/mix/tasks/emqx.dialyzer.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex similarity index 97% rename from lib/mix/tasks/emqx.dialyzer.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex index 9a2a5ad16..0b0e82bdd 100644 --- a/lib/mix/tasks/emqx.dialyzer.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex @@ -1,12 +1,6 @@ defmodule Mix.Tasks.Emqx.Dialyzer do use Mix.Task - alias EMQXUmbrella.MixProject, as: UMP - - if UMP.new_mix_build?() do - Code.require_file("emqx.ct.ex", __DIR__) - end - alias Mix.Tasks.Emqx.Ct, as: ECt @requirements ["compile", "loadpaths"] diff --git a/lib/mix/tasks/emqx.eunit.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex similarity index 95% rename from lib/mix/tasks/emqx.eunit.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex index 2ec0e7abc..4b1d399f7 100644 --- a/lib/mix/tasks/emqx.eunit.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex @@ -1,12 +1,6 @@ defmodule Mix.Tasks.Emqx.Eunit do use Mix.Task - alias EMQXUmbrella.MixProject, as: UMP - - if UMP.new_mix_build?() do - Code.require_file("emqx.ct.ex", __DIR__) - end - alias Mix.Tasks.Emqx.Ct, as: ECt # todo: invoke the equivalent of `make merge-config` as a requirement... diff --git a/lib/mix/tasks/emqx.proper.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex similarity index 97% rename from lib/mix/tasks/emqx.proper.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex index 55dbbe23c..1c9b6754e 100644 --- a/lib/mix/tasks/emqx.proper.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex @@ -1,8 +1,6 @@ defmodule Mix.Tasks.Emqx.Proper do use Mix.Task - # Code.require_file("emqx.ct.ex", __DIR__) - alias Mix.Tasks.Emqx.Ct, as: ECt # todo: invoke the equivalent of `make merge-config` as a requirement... diff --git a/apps/emqx_mix_utils/mix.exs b/apps/emqx_mix_utils/mix.exs new file mode 100644 index 000000000..d239e28fe --- /dev/null +++ b/apps/emqx_mix_utils/mix.exs @@ -0,0 +1,26 @@ +defmodule EMQXMixUtils.MixProject do + use Mix.Project + alias EMQXUmbrella.MixProject, as: UMP + + def project do + [ + app: :emqx_mix_utils, + version: "0.1.0", + build_path: "../../_build", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [extra_applications: UMP.extra_applications()] + end + + def deps() do + [] + end +end diff --git a/apps/emqx_s3/mix.exs b/apps/emqx_s3/mix.exs index d5b145280..0f6d16f11 100644 --- a/apps/emqx_s3/mix.exs +++ b/apps/emqx_s3/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXS3.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, UMP.common_dep(:gproc), UMP.common_dep(:ehttpc), diff --git a/apps/emqx_telemetry/mix.exs b/apps/emqx_telemetry/mix.exs index 224296d2f..c29306fff 100644 --- a/apps/emqx_telemetry/mix.exs +++ b/apps/emqx_telemetry/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXTelemetry.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_conf, in_umbrella: true} diff --git a/mix.exs b/mix.exs index 24a9ec820..0741dae9e 100644 --- a/mix.exs +++ b/mix.exs @@ -40,8 +40,6 @@ defmodule EMQXUmbrella.MixProject do if new_mix_build?() do [ - # TODO: these lines will be uncommented when we switch to using mix as the manager - # for all umbrella apps. apps_path: "apps", apps: applications(profile_info.release_type, profile_info.edition_type) |> Keyword.keys(), @@ -315,6 +313,7 @@ defmodule EMQXUmbrella.MixProject do false end end) + |> Enum.reject(fn {app, _} -> app == :emqx_mix_utils end) |> Enum.reject(fn {app, _} -> app in excluded_apps end) end @@ -1316,26 +1315,22 @@ defmodule EMQXUmbrella.MixProject do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.ct.ex") Mix.Task.run("emqx.ct", args) end defp do_eunit(args) do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.eunit.ex") Mix.Task.run("emqx.eunit", args) end defp do_proper(args) do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.proper.ex") Mix.Task.run("emqx.proper", args) end defp do_dialyzer(args) do - Code.require_file("lib/mix/tasks/emqx.dialyzer.ex") Mix.Task.run("emqx.dialyzer", args) end From 0555a8ec615cd972714f4c150e1d93c06aadef1c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 11 Jul 2024 17:45:13 -0300 Subject: [PATCH 13/14] fix(mix): bizarre compilation order bug with `emqx` profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some bizarre reason, if the `:apps` key is defined in the `project()` callback in the root umbrella `mix.exs`, it messes up the compilation order that mix follows when compiling the project from scratch. Specifically, in the `emqx` profile, even though `:emqx_utils` is an explicit dependency of `:emqx_ds_builtin_local`, mix insisted in compiling the latter before the former, and failing, obviously. Removing the explicit `:apps` from the project definition solved this. 🫠 --- mix.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mix.exs b/mix.exs index 0741dae9e..5368d1509 100644 --- a/mix.exs +++ b/mix.exs @@ -41,8 +41,6 @@ defmodule EMQXUmbrella.MixProject do if new_mix_build?() do [ apps_path: "apps", - apps: - applications(profile_info.release_type, profile_info.edition_type) |> Keyword.keys(), erlc_options: erlc_options(profile_info, version), version: version, deps: deps(profile_info, version), From 4a08bfc93f10c02fc08e9338ebb38c0c7e74943f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jul 2024 11:31:00 -0300 Subject: [PATCH 14/14] feat(mix ct): improve failure logging --- apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex index d03ddb414..0c85bb355 100644 --- a/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex @@ -52,7 +52,7 @@ defmodule Mix.Tasks.Emqx.Ct do testcase: opts |> Map.fetch!(:cases) |> Enum.map(&to_charlist/1), readable: 'true', name: node_name, - ct_hooks: [:cth_readable_shell], + ct_hooks: [:cth_readable_shell, :cth_readable_failonly], logdir: to_charlist(logdir) )