diff --git a/.github/workflows/.zipignore2 b/.github/workflows/.zipignore2 new file mode 100644 index 000000000..f0d0058a7 --- /dev/null +++ b/.github/workflows/.zipignore2 @@ -0,0 +1 @@ +*/.github/* diff --git a/Makefile b/Makefile index ae92feced..6aaaf58d7 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ CT_COVER_EXPORT_PREFIX ?= $(PROFILE) export REBAR_GIT_CLONE_OPTIONS += --depth=1 +ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar + .PHONY: default default: $(REBAR) $(PROFILE) @@ -58,8 +60,12 @@ ensure-mix-rebar3: $(REBAR) ensure-mix-rebar: $(REBAR) @mix local.rebar --if-missing --force + +.PHONY: elixir-common-deps +elixir-common-deps: $(ELIXIR_COMMON_DEPS) + .PHONY: mix-deps-get -mix-deps-get: $(ELIXIR_COMMON_DEPS) +mix-deps-get: elixir-common-deps @mix deps.get .PHONY: eunit diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs index 279332bfe..c9de2b118 100644 --- a/apps/emqx/mix.exs +++ b/apps/emqx/mix.exs @@ -8,7 +8,7 @@ defmodule EMQX.MixProject do app: :emqx, version: "0.1.0", build_path: "../../_build", - erlc_paths: UMP.erlc_paths(), + erlc_paths: erlc_paths(), erlc_options: [ {:i, "src"} | UMP.erlc_options() @@ -36,8 +36,9 @@ defmodule EMQX.MixProject do def deps() do ## FIXME!!! go though emqx.app.src and add missing stuff... [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx_utils, in_umbrella: true}, - # {:emqx_ds_backends, in_umbrella: true}, + {:emqx_ds_backends, in_umbrella: true}, UMP.common_dep(:gproc), UMP.common_dep(:gen_rpc), @@ -53,6 +54,15 @@ defmodule EMQX.MixProject do ] ++ UMP.quicer_dep() end + defp erlc_paths() do + paths = UMP.erlc_paths() + if UMP.test_env?() do + ["integration_test" | paths] + else + paths + end + end + defp extra_dirs() do dirs = ["src", "etc"] if UMP.test_env?() do diff --git a/apps/emqx_auth/mix.exs b/apps/emqx_auth/mix.exs index aacc0bda6..e3438c27b 100644 --- a/apps/emqx_auth/mix.exs +++ b/apps/emqx_auth/mix.exs @@ -28,6 +28,7 @@ defmodule EMQXAuth.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true} ] diff --git a/apps/emqx_bridge/mix.exs b/apps/emqx_bridge/mix.exs index 8322b5606..dd6b91b94 100644 --- a/apps/emqx_bridge/mix.exs +++ b/apps/emqx_bridge/mix.exs @@ -28,6 +28,7 @@ defmodule EMQXBridge.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_resource, in_umbrella: true}, {:emqx_connector, in_umbrella: true}, diff --git a/apps/emqx_bridge_hstreamdb/mix.exs b/apps/emqx_bridge_hstreamdb/mix.exs index 87f8b80c2..8c21da7aa 100644 --- a/apps/emqx_bridge_hstreamdb/mix.exs +++ b/apps/emqx_bridge_hstreamdb/mix.exs @@ -24,7 +24,8 @@ defmodule EMQXBridgeHstreamdb.MixProject do def deps() do [ {:hstreamdb_erl, - github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"}, + github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1", + system_env: UMP.emqx_app_system_env()}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_connector, in_umbrella: true, runtime: false}, diff --git a/apps/emqx_dashboard_sso/mix.exs b/apps/emqx_dashboard_sso/mix.exs index 13a44f61f..94b99aceb 100644 --- a/apps/emqx_dashboard_sso/mix.exs +++ b/apps/emqx_dashboard_sso/mix.exs @@ -26,7 +26,8 @@ defmodule EMQXDashboardSso.MixProject do {:emqx_ctl, in_umbrella: true}, {:emqx_ldap, in_umbrella: true}, {:emqx_dashboard, in_umbrella: true}, - {:esaml, github: "emqx/esaml", tag: "v1.1.3"} + {:esaml, github: "emqx/esaml", tag: "v1.1.3"}, + {:oidcc, github: "emqx/oidcc", tag: "v3.2.0-1"}, ] end end diff --git a/apps/emqx_durable_storage/mix.exs b/apps/emqx_durable_storage/mix.exs index fd93f3bf5..59626a679 100644 --- a/apps/emqx_durable_storage/mix.exs +++ b/apps/emqx_durable_storage/mix.exs @@ -7,9 +7,14 @@ defmodule EMQXDurableStorage.MixProject do app: :emqx_durable_storage, version: "0.1.0", build_path: "../../_build", - # config_path: "../../config/config.exs", + compilers: [:yecc, :leex, :elixir, :asn1, :erlang, :app], erlc_options: UMP.erlc_options(), - erlc_paths: UMP.erlc_paths(), + erlc_paths: ["gen_src" | UMP.erlc_paths()], + # used by our `compile.asn1` compiler + asn1_srcs: [ + %{src: "./asn.1/DurableMessage.asn", + compile_opts: [:per, :noobj, outdir: ~c"gen_src"]} + ], deps_path: "../../deps", lockfile: "../../mix.lock", elixir: "~> 1.14", @@ -28,6 +33,7 @@ defmodule EMQXDurableStorage.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx_utils, in_umbrella: true}, UMP.common_dep(:rocksdb), UMP.common_dep(:gproc), diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs index 4e72cfa24..5baf6dac4 100644 --- a/apps/emqx_exhook/mix.exs +++ b/apps/emqx_exhook/mix.exs @@ -36,6 +36,7 @@ defmodule EMQXExhook.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, UMP.common_dep(:grpc) diff --git a/apps/emqx_gateway/mix.exs b/apps/emqx_gateway/mix.exs index 7c4daeb46..fb1b48d40 100644 --- a/apps/emqx_gateway/mix.exs +++ b/apps/emqx_gateway/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXGateway.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 11271b329..3183564df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -48,6 +48,8 @@ init_per_suite(Conf) -> Apps = emqx_cth_suite:start( [ emqx_conf, + emqx_auth, + emqx_auth_mnesia, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, {emqx_gateway, ?CONF_DEFAULT} diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 936856c2e..e09d9356e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) -> Apps = emqx_cth_suite:start( [ emqx_conf, + emqx_auth_http, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()} diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 5b262158f..923b71ced 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) -> Apps = emqx_cth_suite:start( [ {emqx_conf, "authorization { no_match = deny, cache { enable = false } }"}, + emqx_auth_http, {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()} | emqx_gateway_test_utils:all_gateway_apps() ], diff --git a/apps/emqx_gateway_exproto/mix.exs b/apps/emqx_gateway_exproto/mix.exs index e3564694d..558fe9ed7 100644 --- a/apps/emqx_gateway_exproto/mix.exs +++ b/apps/emqx_gateway_exproto/mix.exs @@ -34,6 +34,7 @@ defmodule EMQXGatewayExproto.MixProject do def deps() do test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: [] test_deps ++ [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_gateway, in_umbrella: true}, diff --git a/apps/emqx_gcp_device/mix.exs b/apps/emqx_gcp_device/mix.exs index 94e1dc077..cafdf6a12 100644 --- a/apps/emqx_gcp_device/mix.exs +++ b/apps/emqx_gcp_device/mix.exs @@ -27,6 +27,7 @@ defmodule EMQXGCPDevice.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_auth, in_umbrella: true}, UMP.common_dep(:jose), diff --git a/apps/emqx_license/mix.exs b/apps/emqx_license/mix.exs index e0d2937de..a01256dad 100644 --- a/apps/emqx_license/mix.exs +++ b/apps/emqx_license/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXLicense.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, diff --git a/apps/emqx_machine/mix.exs b/apps/emqx_machine/mix.exs index 37fb59d5d..4ea0eef79 100644 --- a/apps/emqx_machine/mix.exs +++ b/apps/emqx_machine/mix.exs @@ -30,6 +30,8 @@ defmodule EMQXMachine.MixProject do {:emqx_dashboard, in_umbrella: true, runtime: false}, {:emqx_management, in_umbrella: true, runtime: false}, UMP.common_dep(:covertool), + UMP.common_dep(:system_monitor), + UMP.common_dep(:redbug), ] end end diff --git a/apps/emqx_exhook/lib/emqx/grpc/template/client.eex b/apps/emqx_mix_utils/lib/emqx/grpc/template/client.eex similarity index 100% rename from apps/emqx_exhook/lib/emqx/grpc/template/client.eex rename to apps/emqx_mix_utils/lib/emqx/grpc/template/client.eex diff --git a/apps/emqx_exhook/lib/emqx/grpc/template/service.eex b/apps/emqx_mix_utils/lib/emqx/grpc/template/service.eex similarity index 100% rename from apps/emqx_exhook/lib/emqx/grpc/template/service.eex rename to apps/emqx_mix_utils/lib/emqx/grpc/template/service.eex diff --git a/apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex new file mode 100644 index 000000000..afec3f62c --- /dev/null +++ b/apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex @@ -0,0 +1,82 @@ +defmodule Mix.Tasks.Compile.Asn1 do + use Mix.Task.Compiler + + @recursive true + @manifest_vsn 1 + @manifest "compile.asn1" + # TODO: use manifest to track generated files? + + @impl true + def manifests(), do: [manifest()] + defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest) + + @impl true + def run(_args) do + add_to_path_and_cache(:asn1) + + Mix.Project.get!() + config = Mix.Project.config() + app_root = File.cwd!() + + asn1_srcs = config[:asn1_srcs] || [] + manifest_data = read_manifest(manifest()) + manifest_modified_time = Mix.Utils.last_modified(manifest()) + Enum.each(asn1_srcs, &compile(&1, app_root, manifest_modified_time)) + write_manifest(manifest(), manifest_data) + + {:noop, []} + end + + defp compile(src, app_root, manifest_modified_time) do + %{ + src: src_path, + compile_opts: compile_opts + } = src + src_path = + app_root + |> Path.join(src_path) + |> Path.expand() + if stale?(src_path, manifest_modified_time) do + Mix.shell().info("compiling asn1 file: #{src_path}") + :ok = :asn1ct.compile(to_charlist(src_path), compile_opts) + else + Mix.shell().info("file is up to date, not compiling: #{src_path}") + end + end + + defp stale?(file, manifest_modified_time) do + with true <- File.exists?(file), + false <- Mix.Utils.stale?([file], [manifest_modified_time]) do + false + else + _ -> true + end + end + + defp read_manifest(file) do + try do + file |> File.read!() |> :erlang.binary_to_term() + rescue + _ -> %{} + else + {@manifest_vsn, data} when is_map(data) -> data + _ -> %{} + end + end + + defp write_manifest(file, data) do + Mix.shell().info("writing manifest #{file}") + File.mkdir_p!(Path.dirname(file)) + File.write!(file, :erlang.term_to_binary({@manifest_vsn, data})) + end + + def add_to_path_and_cache(lib_name) do + :code.lib_dir() + |> Path.join("#{lib_name}-*") + |> Path.wildcard() + |> hd() + |> Path.join("ebin") + |> to_charlist() + |> :code.add_path(:cache) + end +end diff --git a/apps/emqx/lib/mix/tasks/compile.copy_srcs.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.copy_srcs.ex similarity index 100% rename from apps/emqx/lib/mix/tasks/compile.copy_srcs.ex rename to apps/emqx_mix_utils/lib/mix/tasks/compile.copy_srcs.ex diff --git a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex b/apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex similarity index 97% rename from apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex rename to apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex index 615f98c29..0a33f62b7 100644 --- a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/compile.grpc.ex @@ -1,6 +1,8 @@ defmodule Mix.Tasks.Compile.Grpc do use Mix.Task.Compiler + alias EMQXUmbrella.MixProject, as: UMP + @recursive true @manifest_vsn 1 @manifest "compile.grpc" @@ -46,6 +48,9 @@ defmodule Mix.Tasks.Compile.Grpc do write_manifest(manifest(), manifest_data) {:noop, []} + after + Application.unload(:gpb) + Application.unload(:syntax_tools) end defp compile_pb(proto_src, context) do @@ -95,7 +100,7 @@ defmodule Mix.Tasks.Compile.Grpc do :return_errors, i: to_charlist(gpb_include_dir), outdir: to_charlist(ebin_path) - ] + ] ++ UMP.erlc_options() ) # todo: error handling & logging case compile_res do diff --git a/lib/mix/tasks/emqx.ct.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex similarity index 95% rename from lib/mix/tasks/emqx.ct.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex index ed75f3adc..0c85bb355 100644 --- a/lib/mix/tasks/emqx.ct.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.ct.ex @@ -48,10 +48,11 @@ defmodule Mix.Tasks.Emqx.Ct do abort_if_missing_suites: true, auto_compile: false, suite: opts |> Map.fetch!(:suites) |> Enum.map(&to_charlist/1), + group: opts |> Map.fetch!(:group_paths) |> Enum.map(fn gp -> Enum.map(gp, &String.to_atom/1) end), testcase: opts |> Map.fetch!(:cases) |> Enum.map(&to_charlist/1), readable: 'true', name: node_name, - ct_hooks: [:cth_readable_shell], + ct_hooks: [:cth_readable_shell, :cth_readable_failonly], logdir: to_charlist(logdir) ) @@ -198,11 +199,13 @@ defmodule Mix.Tasks.Emqx.Ct do args, strict: [ suites: :string, - groups: :string, + group_paths: :string, cases: :string]) - |> IO.inspect(label: :opts) suites = get_name_list(opts, :suites) - groups = get_name_list(opts, :groups) + group_paths = + opts + |> get_name_list(:group_paths) + |> Enum.map(& String.split(&1, ".", trim: true)) cases = get_name_list(opts, :cases) if suites == [] do @@ -211,7 +214,7 @@ defmodule Mix.Tasks.Emqx.Ct do %{ suites: suites, - groups: groups, + group_paths: group_paths, cases: cases } end diff --git a/apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex new file mode 100644 index 000000000..0b0e82bdd --- /dev/null +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex @@ -0,0 +1,158 @@ +defmodule Mix.Tasks.Emqx.Dialyzer do + use Mix.Task + + alias Mix.Tasks.Emqx.Ct, as: ECt + + @requirements ["compile", "loadpaths"] + + @excluded_mods ( + [ + :emqx_exproto_v_1_connection_unary_handler_bhvr, + :emqx_exproto_v_1_connection_handler_client, + :emqx_exproto_v_1_connection_handler_bhvr, + :emqx_exproto_v_1_connection_adapter_client, + :emqx_exproto_v_1_connection_adapter_bhvr, + :emqx_exproto_v_1_connection_unary_handler_client, + :emqx_exhook_v_2_hook_provider_client, + :emqx_exhook_v_2_hook_provider_bhvr, + Mix.Tasks.Compile.Grpc, + Mix.Tasks.Compile.CopySrcs, + ] + |> MapSet.new(&to_string/1) + ) + + @impl true + def run(_args) do + ECt.add_to_path_and_cache(:dialyzer) + + %{ + umbrella_apps: umbrella_apps, + dep_apps: dep_apps + } = resolve_apps() + umbrella_files = Enum.flat_map(umbrella_apps, & resolve_files/1) + dep_files = Enum.flat_map(dep_apps, & resolve_files/1) + files = + (umbrella_files ++ dep_files) + |> Enum.reject(fn path -> + name = Path.basename(path, ".beam") + MapSet.member?(@excluded_mods, name) + end) + |> Enum.map(&to_charlist/1) + warning_files = + umbrella_files + |> Enum.reject(fn path -> + name = Path.basename(path, ".beam") + MapSet.member?(@excluded_mods, name) + end) + |> Enum.map(&to_charlist/1) + warning_apps = Enum.sort(umbrella_apps) + + try do + :dialyzer.run( + analysis_type: :incremental, + warnings: [ + :unmatched_returns, + :error_handling + ], + # plt_location: ~c".", + # plt_prefix: ~c"emqx_dialyzer", + warning_files: warning_files, + warning_files_rec: warning_files, + # apps: umbrella_apps ++ dep_apps, + # warning_apps: warning_apps, + get_warnings: false, + files: files, + files_rec: files + ) + catch + {:dialyzer_error, msg} -> + {:dialyzer_error, to_string(msg)} + err -> + {:throw, err} + end + |> IO.inspect(limit: :infinity) + end + + defp resolve_apps() do + base_apps = MapSet.new([:erts, :crypto]) + # excluded_apps = MapSet.new([:elixir]) + excluded_apps = MapSet.new() + acc = %{ + umbrella_apps: [], + dep_apps: base_apps + } + + Mix.Dep.Umbrella.loaded() + |> Enum.reduce(acc, fn dep, acc -> + # IO.inspect(dep) + props = dep.opts[:app_properties] + optional_apps = Keyword.get(props, :optional_applications, []) + apps = Keyword.get(props, :applications, []) + included_apps = Keyword.get(props, :included_applications, []) + dep_apps = MapSet.new(optional_apps ++ apps ++ included_apps) + acc + |> Map.update!(:umbrella_apps, & [dep.app | &1]) + |> Map.update!(:dep_apps, & MapSet.union(&1, dep_apps)) + end) + |> then(fn acc -> + dep_apps = + acc.dep_apps + |> MapSet.difference(MapSet.new(acc.umbrella_apps)) + |> MapSet.difference(excluded_apps) + |> Enum.reduce(MapSet.new(), &find_nested_apps/2) + |> MapSet.difference(excluded_apps) + |> Enum.filter(&app_present?/1) + %{acc | dep_apps: dep_apps} + end) + end + + defp app_present?(app) do + match?({:ok, _}, ebin_dir(app)) + end + + defp find_nested_apps(app, seen) do + if MapSet.member?(seen, app) do + seen + else + seen = MapSet.put(seen, app) + apps = case :application.get_key(app, :applications) do + {:ok, apps} -> apps + :undefined -> [] + end + included_apps = case :application.get_key(app, :included_applications) do + {:ok, apps} -> apps + :undefined -> [] + end + optional_apps = case :application.get_key(app, :optional_applications) do + {:ok, apps} -> apps + :undefined -> [] + end + Enum.reduce(apps ++ included_apps, seen, &find_nested_apps/2) + end + end + + defp resolve_files(app) do + with {:ok, dir} <- ebin_dir(app) do + Mix.Utils.extract_files([dir], [:beam]) + else + _ -> [] + end + end + + defp ebin_dir(app) do + with dir when is_list(dir) <- :code.lib_dir(app, :ebin), + dir = to_string(dir), + true <- File.dir?(dir) || {:error, :not_a_dir} do + {:ok, to_string(dir)} + else + error -> + Mix.shell().info(IO.ANSI.format([ + [:yellow, + "Unknown application: #{app}; error: #{inspect(error)}", + "; if this is is an optional application, ignore." + ], + ])) + :error + end + end +end diff --git a/apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex new file mode 100644 index 000000000..4b1d399f7 --- /dev/null +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex @@ -0,0 +1,122 @@ +defmodule Mix.Tasks.Emqx.Eunit do + use Mix.Task + + alias Mix.Tasks.Emqx.Ct, as: ECt + + # todo: invoke the equivalent of `make merge-config` as a requirement... + @requirements ["compile", "loadpaths"] + + @impl true + def run(args) do + Mix.debug(true) + IO.inspect(args) + + + Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1) + + ECt.ensure_whole_emqx_project_is_loaded!() + ECt.unload_emqx_applications!() + + {_, 0} = System.cmd("epmd", ["-daemon"]) + node_name = :"test@127.0.0.1" + :net_kernel.start([node_name, :longnames]) + + # unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect + # the version without the `-test` suffix. + System.fetch_env!("PROFILE") + |> String.replace_suffix("-test", "") + |> then(& System.put_env("PROFILE", &1)) + + args + |> parse_args!() + |> discover_tests() + |> :eunit.test( + verbose: true, + print_depth: 100 + ) + |> case do + :ok -> :ok + :error -> Mix.raise("errors found in tests") + end + end + + defp add_to_path_and_cache(lib_name) do + :code.lib_dir() + |> Path.join("#{lib_name}-*") + |> Path.wildcard() + |> hd() + |> Path.join("ebin") + |> to_charlist() + |> :code.add_path(:cache) + end + + defp parse_args!(args) do + {opts, _rest} = OptionParser.parse!( + args, + strict: [ + cases: :string, + modules: :string, + ] + ) + cases = + opts + |> get_name_list(:cases) + |> Enum.flat_map(&resolve_test_fns!/1) + modules = + opts + |> get_name_list(:modules) + |> Enum.map(&String.to_atom/1) + + %{ + cases: cases, + modules: modules, + } + end + + defp get_name_list(opts, key) do + opts + |> Keyword.get(key, "") + |> String.split(",", trim: true) + end + + defp resolve_test_fns!(mod_fn_str) do + {mod, fun} = case String.split(mod_fn_str, ":") do + [mod, fun] -> + {String.to_atom(mod), String.to_atom(fun)} + _ -> + Mix.raise("Bad test case spec; must of `MOD:FUN` form. Got: #{mod_fn_str}`") + end + if not has_test_case?(mod, fun) do + Mix.raise("Module #{mod} does not export test case #{fun}") + end + + if to_string(fun) =~ ~r/_test_$/ do + apply(mod, fun, []) + else + [Function.capture(mod, fun, 0)] + end + end + + defp has_test_case?(mod, fun) do + try do + mod.module_info(:functions) + |> Enum.find(& &1 == {fun, 0}) + |> then(& !! &1) + rescue + UndefinedFunctionError -> false + end + end + + defp discover_tests(%{cases: [], modules: []} = _opts) do + Mix.Dep.Umbrella.cached() + |> Enum.map(& {:application, &1.app}) + end + defp discover_tests(%{cases: cases, modules: modules}) do + Enum.concat( + [ + cases, + Enum.map(modules, & {:module, &1}) + ] + ) + end +end diff --git a/lib/mix/tasks/emqx.proper.ex b/apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex similarity index 97% rename from lib/mix/tasks/emqx.proper.ex rename to apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex index 55dbbe23c..1c9b6754e 100644 --- a/lib/mix/tasks/emqx.proper.ex +++ b/apps/emqx_mix_utils/lib/mix/tasks/emqx.proper.ex @@ -1,8 +1,6 @@ defmodule Mix.Tasks.Emqx.Proper do use Mix.Task - # Code.require_file("emqx.ct.ex", __DIR__) - alias Mix.Tasks.Emqx.Ct, as: ECt # todo: invoke the equivalent of `make merge-config` as a requirement... diff --git a/apps/emqx_mix_utils/mix.exs b/apps/emqx_mix_utils/mix.exs new file mode 100644 index 000000000..d239e28fe --- /dev/null +++ b/apps/emqx_mix_utils/mix.exs @@ -0,0 +1,26 @@ +defmodule EMQXMixUtils.MixProject do + use Mix.Project + alias EMQXUmbrella.MixProject, as: UMP + + def project do + [ + app: :emqx_mix_utils, + version: "0.1.0", + build_path: "../../_build", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [extra_applications: UMP.extra_applications()] + end + + def deps() do + [] + end +end diff --git a/apps/emqx_modules/mix.exs b/apps/emqx_modules/mix.exs index f271d7729..b88171536 100644 --- a/apps/emqx_modules/mix.exs +++ b/apps/emqx_modules/mix.exs @@ -26,7 +26,8 @@ defmodule EMQXModules.MixProject do {:emqx, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, - {:emqx_conf, in_umbrella: true} + {:emqx_conf, in_umbrella: true}, + UMP.common_dep(:observer_cli) ] end end diff --git a/apps/emqx_rule_engine/mix.exs b/apps/emqx_rule_engine/mix.exs index 0db98f21a..6463a8532 100644 --- a/apps/emqx_rule_engine/mix.exs +++ b/apps/emqx_rule_engine/mix.exs @@ -22,7 +22,7 @@ defmodule EMQXRuleEngine.MixProject do end def deps() do - [ + UMP.jq_dep() ++ [ {:emqx, in_umbrella: true}, {:emqx_ctl, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, @@ -31,6 +31,7 @@ defmodule EMQXRuleEngine.MixProject do {:emqx_bridge, in_umbrella: true}, UMP.common_dep(:rulesql), UMP.common_dep(:emqtt), + UMP.common_dep(:uuid), ] end end diff --git a/apps/emqx_s3/mix.exs b/apps/emqx_s3/mix.exs index d5b145280..0f6d16f11 100644 --- a/apps/emqx_s3/mix.exs +++ b/apps/emqx_s3/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXS3.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, UMP.common_dep(:gproc), UMP.common_dep(:ehttpc), diff --git a/apps/emqx_schema_registry/mix.exs b/apps/emqx_schema_registry/mix.exs index 0abffad6f..d286b3292 100644 --- a/apps/emqx_schema_registry/mix.exs +++ b/apps/emqx_schema_registry/mix.exs @@ -28,7 +28,7 @@ defmodule EMQXSchemaRegistry.MixProject do {:emqx_rule_engine, in_umbrella: true}, {:erlavro, github: "emqx/erlavro", tag: "2.10.0"}, {:jesse, github: "emqx/jesse", tag: "1.8.0"}, - UMP.common_dep(:gpb), + UMP.common_dep(:gpb, runtime: true), ] end end diff --git a/apps/emqx_telemetry/mix.exs b/apps/emqx_telemetry/mix.exs index 224296d2f..c29306fff 100644 --- a/apps/emqx_telemetry/mix.exs +++ b/apps/emqx_telemetry/mix.exs @@ -26,6 +26,7 @@ defmodule EMQXTelemetry.MixProject do def deps() do [ + {:emqx_mix_utils, in_umbrella: true, runtime: false}, {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}, {:emqx_conf, in_umbrella: true} diff --git a/lib/mix/tasks/emqx.eunit.ex b/lib/mix/tasks/emqx.eunit.ex deleted file mode 100644 index a2c0dfe39..000000000 --- a/lib/mix/tasks/emqx.eunit.ex +++ /dev/null @@ -1,58 +0,0 @@ -defmodule Mix.Tasks.Emqx.Eunit do - use Mix.Task - - # Code.require_file("emqx.ct.ex", __DIR__) - - alias Mix.Tasks.Emqx.Ct, as: ECt - - # todo: invoke the equivalent of `make merge-config` as a requirement... - @requirements ["compile", "loadpaths"] - - @impl true - def run(args) do - Mix.debug(true) - IO.inspect(args) - - - Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1) - - ECt.ensure_whole_emqx_project_is_loaded!() - ECt.unload_emqx_applications!() - - {_, 0} = System.cmd("epmd", ["-daemon"]) - node_name = :"test@127.0.0.1" - :net_kernel.start([node_name, :longnames]) - - # unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect - # the version without the `-test` suffix. - System.fetch_env!("PROFILE") - |> String.replace_suffix("-test", "") - |> then(& System.put_env("PROFILE", &1)) - - discover_tests() - |> :eunit.test( - verbose: true, - print_depth: 100 - ) - |> case do - :ok -> :ok - :error -> Mix.raise("errors found in tests") - end - end - - defp add_to_path_and_cache(lib_name) do - :code.lib_dir() - |> Path.join("#{lib_name}-*") - |> Path.wildcard() - |> hd() - |> Path.join("ebin") - |> to_charlist() - |> :code.add_path(:cache) - end - - ## TODO: allow filtering modules and test names - defp discover_tests() do - Mix.Dep.Umbrella.cached() - |> Enum.map(& {:application, &1.app}) - end -end diff --git a/mix.exs b/mix.exs index 9e9e778b0..5368d1509 100644 --- a/mix.exs +++ b/mix.exs @@ -40,11 +40,7 @@ defmodule EMQXUmbrella.MixProject do if new_mix_build?() do [ - # TODO: these lines will be uncommented when we switch to using mix as the manager - # for all umbrella apps. apps_path: "apps", - apps: - applications(profile_info.release_type, profile_info.edition_type) |> Keyword.keys(), erlc_options: erlc_options(profile_info, version), version: version, deps: deps(profile_info, version), @@ -160,7 +156,7 @@ defmodule EMQXUmbrella.MixProject do {:ssl_verify_fun, "1.1.7", override: true}, common_dep(:rfc3339), common_dep(:bcrypt), - {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, + common_dep(:uuid), {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, common_dep(:ra), {:mimerl, "1.2.0", override: true} @@ -169,12 +165,22 @@ defmodule EMQXUmbrella.MixProject do def extra_release_apps() do [ - {:redbug, github: "emqx/redbug", tag: "2.0.10"}, - {:observer_cli, "1.7.1"}, - {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"} + common_dep(:redbug), + common_dep(:observer_cli), + common_dep(:system_monitor) ] end + def common_dep(dep_name, overrides) do + case common_dep(dep_name) do + {^dep_name, opts} -> + {dep_name, Keyword.merge(opts, overrides)} + + {^dep_name, tag, opts} when is_binary(tag) -> + {dep_name, tag, Keyword.merge(opts, overrides)} + end + end + def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true} def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true} def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true} @@ -186,7 +192,12 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:ranch), do: {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true} def common_dep(:ehttpc), do: {:ehttpc, github: "emqx/ehttpc", tag: "0.4.14", override: true} def common_dep(:jiffy), do: {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true} - def common_dep(:grpc), do: {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true} + + def common_dep(:grpc), + do: + {:grpc, + github: "emqx/grpc-erl", tag: "0.6.12", override: true, system_env: emqx_app_system_env()} + def common_dep(:cowboy), do: {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true} def common_dep(:jsone), do: {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true} def common_dep(:ecpool), do: {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true} @@ -207,6 +218,13 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:esasl), do: {:esasl, github: "emqx/esasl", tag: "0.2.1"} def common_dep(:gen_rpc), do: {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true} + def common_dep(:system_monitor), + do: {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"} + + def common_dep(:uuid), do: {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true} + def common_dep(:redbug), do: {:redbug, github: "emqx/redbug", tag: "2.0.10"} + def common_dep(:observer_cli), do: {:observer_cli, "1.7.1"} + def common_dep(:jose), do: {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2", override: true} @@ -252,7 +270,7 @@ defmodule EMQXUmbrella.MixProject do github: "kafka4beam/snabbkaffe", tag: "1.0.10", override: true, - system_env: emqx_app_system_env(profile_info(), pkg_vsn()) + system_env: emqx_app_system_env() } ############################################################################################### @@ -293,6 +311,7 @@ defmodule EMQXUmbrella.MixProject do false end end) + |> Enum.reject(fn {app, _} -> app == :emqx_mix_utils end) |> Enum.reject(fn {app, _} -> app in excluded_apps end) end @@ -416,16 +435,24 @@ defmodule EMQXUmbrella.MixProject do ) end - ############################################################################################### - # END DEPRECATED FOR MIX BLOCK - ############################################################################################### - def emqx_app_system_env(profile_info, version) do erlc_options(profile_info, version) |> dump_as_erl() |> then(&[{"ERL_COMPILER_OPTIONS", &1}]) end + def emqx_app_system_env() do + k = {__MODULE__, :emqx_app_system_env} + + get_memoized(k, fn -> + emqx_app_system_env(profile_info(), pkg_vsn()) + end) + end + + ############################################################################################### + # END DEPRECATED FOR MIX BLOCK + ############################################################################################### + defp erlc_options(%{edition_type: edition_type}, version) do [ :debug_info, @@ -542,6 +569,7 @@ defmodule EMQXUmbrella.MixProject do } = check_profile!() base_steps = [ + &merge_config/1, &make_docs/1, :assemble, &create_RELEASES/1, @@ -780,6 +808,12 @@ defmodule EMQXUmbrella.MixProject do # Custom Steps ############################################################################# + # Gathers i18n files and merge them before producing docs and schemas. + defp merge_config(release) do + {_, 0} = System.cmd("bash", ["-c", "./scripts/merge-config.escript"]) + release + end + defp make_docs(release) do profile = System.get_env("MIX_ENV") os_cmd("build", [profile, "docs"]) @@ -1104,7 +1138,7 @@ defmodule EMQXUmbrella.MixProject do defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema defp emqx_schema_mod(:community), do: :emqx_conf_schema - defp jq_dep() do + def jq_dep() do if enable_jq?(), do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}], else: [] @@ -1267,7 +1301,8 @@ defmodule EMQXUmbrella.MixProject do [ ct: &do_ct/1, eunit: &do_eunit/1, - proper: &do_proper/1 + proper: &do_proper/1, + dialyzer: &do_dialyzer/1 ] end @@ -1278,24 +1313,25 @@ defmodule EMQXUmbrella.MixProject do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.ct.ex") Mix.Task.run("emqx.ct", args) end defp do_eunit(args) do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.eunit.ex") Mix.Task.run("emqx.eunit", args) end defp do_proper(args) do ensure_test_mix_env!() set_test_env!(true) - Code.require_file("lib/mix/tasks/emqx.proper.ex") Mix.Task.run("emqx.proper", args) end + defp do_dialyzer(args) do + Mix.Task.run("emqx.dialyzer", args) + end + defp ensure_test_mix_env!() do Mix.env() |> to_string()