diff --git a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex index 4fb6ab2da..615f98c29 100644 --- a/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex +++ b/apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex @@ -2,8 +2,14 @@ defmodule Mix.Tasks.Compile.Grpc do use Mix.Task.Compiler @recursive true + @manifest_vsn 1 + @manifest "compile.grpc" # TODO: use manifest to track generated files? + @impl true + def manifests(), do: [manifest()] + defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest) + @impl true def run(_args) do Mix.Project.get!() @@ -14,6 +20,10 @@ defmodule Mix.Tasks.Compile.Grpc do out_dir: out_dir } = config[:grpc_opts] + add_to_path_and_cache(:syntax_tools) + :ok = Application.ensure_loaded(:syntax_tools) + :ok = Application.ensure_loaded(:gpb) + app_root = File.cwd!() app_build_path = Mix.Project.app_path(config) @@ -22,12 +32,30 @@ defmodule Mix.Tasks.Compile.Grpc do |> Enum.map(& Path.join([app_root, &1])) |> Mix.Utils.extract_files([:proto]) - Enum.each(proto_srcs, & compile_pb(&1, app_root, app_build_path, out_dir, gpb_opts)) + manifest_data = read_manifest(manifest()) + context = %{ + manifest_data: manifest_data, + app_root: app_root, + app_build_path: app_build_path, + out_dir: out_dir, + gpb_opts: gpb_opts, + } + + Enum.each(proto_srcs, & compile_pb(&1, context)) + + write_manifest(manifest(), manifest_data) {:noop, []} end - defp compile_pb(proto_src, app_root, app_build_path, out_dir, gpb_opts) do + defp compile_pb(proto_src, context) do + %{ + app_root: app_root, + app_build_path: app_build_path, + out_dir: out_dir, + gpb_opts: gpb_opts, + } = context + manifest_modified_time = Mix.Utils.last_modified(manifest()) ebin_path = Path.join([app_build_path, "ebin"]) basename = proto_src |> Path.basename(".proto") |> to_charlist() prefix = Keyword.get(gpb_opts, :module_name_prefix, '') @@ -43,48 +71,65 @@ defmodule Mix.Tasks.Compile.Grpc do rename: {:msg_name, :snake_case}, rename: {:msg_fqname, :base_name}, ] - File.mkdir_p!(out_dir) - # TODO: better error logging... - :ok = :gpb_compile.file( - to_charlist(proto_src), - opts ++ gpb_opts - ) + + if stale?(proto_src, manifest_modified_time) do + Mix.shell().info("compiling proto file: #{proto_src}") + File.mkdir_p!(out_dir) + # TODO: better error logging... + :ok = :gpb_compile.file( + to_charlist(proto_src), + opts ++ gpb_opts + ) + else + Mix.shell().info("proto file up to date, not compiling: #{proto_src}") + end + generated_src = Path.join([app_root, out_dir, "#{mod_name}.erl"]) - |> IO.inspect(label: :generated_src) - generated_ebin = Path.join([ebin_path, "#{mod_name}.beam"]) - |> IO.inspect(label: :generated_ebin) gpb_include_dir = :code.lib_dir(:gpb, :include) - compile_res = :compile.file( - to_charlist(generated_src), - [ - :return_errors, - i: to_charlist(gpb_include_dir), - outdir: to_charlist(ebin_path) - ] - ) - # todo: error handling & logging - case compile_res do - {:ok, _} -> - :ok + if stale?(generated_src, manifest_modified_time) do + Mix.shell().info("compiling proto module: #{generated_src}") + compile_res = :compile.file( + to_charlist(generated_src), + [ + :return_errors, + i: to_charlist(gpb_include_dir), + outdir: to_charlist(ebin_path) + ] + ) + # todo: error handling & logging + case compile_res do + {:ok, _} -> + :ok - {:ok, _, _warnings} -> - :ok + {:ok, _, _warnings} -> + :ok + end + else + Mix.shell().info("file up to date, not compiling: #{generated_src}") end mod_name |> List.to_atom() |> :code.purge() - {:module, mod} = + {:module, _mod} = ebin_path |> Path.join(mod_name) |> to_charlist() |> :code.load_abs() mod_name = List.to_atom(mod_name) - service_quoted = EEx.compile_file("lib/emqx/grpc/template/service.eex") - client_quoted = EEx.compile_file("lib/emqx/grpc/template/client.eex") + service_quoted = + [__DIR__, "../../", "emqx/grpc/template/service.eex"] + |> Path.join() + |> Path.expand() + |> EEx.compile_file() + client_quoted = + [__DIR__, "../../", "emqx/grpc/template/client.eex"] + |> Path.join() + |> Path.expand() + |> EEx.compile_file() mod_name.get_service_names() |> Enum.each(fn service -> @@ -107,29 +152,74 @@ defmodule Mix.Tasks.Compile.Grpc do |> Macro.underscore() |> String.replace("/", "_") |> String.replace(~r/(.)([0-9]+)/, "\\1_\\2") - {result, _bindings} = Code.eval_quoted( - service_quoted, - methods: methods, - module_name: snake_service, - unmodified_service_name: service_name) - result = String.replace(result, ~r/\n\n\n+/, "\n\n\n") - output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"]) - File.write!(output_src, result) - {result, _bindings} = Code.eval_quoted( - client_quoted, + bindings = [ methods: methods, pb_module: mod_name, module_name: snake_service, - unmodified_service_name: service_name) - result = String.replace(result, ~r/\n\n\n+/, "\n\n\n") - output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"]) - File.write!(output_src, result) + unmodified_service_name: service_name + ] - {{:service, service_name}, methods} + bhvr_output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"]) + if stale?(bhvr_output_src, manifest_modified_time) do + render_and_write(service_quoted, bhvr_output_src, bindings) + else + Mix.shell().info("file up to date, not compiling: #{bhvr_output_src}") + end + + client_output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"]) + if stale?(client_output_src, manifest_modified_time) do + render_and_write(client_quoted, client_output_src, bindings) + else + Mix.shell().info("file up to date, not compiling: #{client_output_src}") + end + + :ok end) end) - mod_name + :ok + end + + defp stale?(file, manifest_modified_time) do + with true <- File.exists?(file), + false <- Mix.Utils.stale?([file], [manifest_modified_time]) do + false + else + _ -> true + end + end + + defp read_manifest(file) do + try do + file |> File.read!() |> :erlang.binary_to_term() + rescue + _ -> %{} + else + {@manifest_vsn, data} when is_map(data) -> data + _ -> %{} + end + end + + defp write_manifest(file, data) do + Mix.shell().info("writing manifest #{file}") + File.mkdir_p!(Path.dirname(file)) + File.write!(file, :erlang.term_to_binary({@manifest_vsn, data})) + end + + defp render_and_write(quoted_file, output_src, bindings) do + {result, _bindings} = Code.eval_quoted(quoted_file, bindings) + result = String.replace(result, ~r/\n\n\n+/, "\n\n\n") + File.write!(output_src, result) + end + + def add_to_path_and_cache(lib_name) do + :code.lib_dir() + |> Path.join("#{lib_name}-*") + |> Path.wildcard() + |> hd() + |> Path.join("ebin") + |> to_charlist() + |> :code.add_path(:cache) end end diff --git a/apps/emqx_gateway_exproto/mix.exs b/apps/emqx_gateway_exproto/mix.exs index c5da8d49e..72d9bd785 100644 --- a/apps/emqx_gateway_exproto/mix.exs +++ b/apps/emqx_gateway_exproto/mix.exs @@ -7,6 +7,16 @@ defmodule EMQXGatewayExproto.MixProject do app: :emqx_gateway_exproto, version: "0.1.0", build_path: "../../_build", + compilers: [:elixir, :grpc, :erlang, :app], + # used by our `Mix.Tasks.Compile.Grpc` compiler + grpc_opts: %{ + gpb_opts: [ + module_name_prefix: 'emqx_', + module_name_suffix: '_pb', + ], + proto_dirs: ["priv/protos"], + out_dir: "src" + }, erlc_options: UMP.erlc_options(), erlc_paths: UMP.erlc_paths(), deps_path: "../../deps", @@ -22,7 +32,7 @@ defmodule EMQXGatewayExproto.MixProject do end def deps() do - test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true}], else: [] + test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: [] test_deps ++ [ {:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true},