perf: use manifest to track proto file compilation

apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex

@@ -2,8 +2,14 @@ defmodule Mix.Tasks.Compile.Grpc do
   use Mix.Task.Compiler
 
   @recursive true
+  @manifest_vsn 1
+  @manifest "compile.grpc"
   # TODO: use manifest to track generated files?
 
+  @impl true
+  def manifests(), do: [manifest()]
+  defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest)
+
   @impl true
   def run(_args) do
     Mix.Project.get!()
@@ -14,6 +20,10 @@ defmodule Mix.Tasks.Compile.Grpc do
         out_dir: out_dir
     } = config[:grpc_opts]
 
+    add_to_path_and_cache(:syntax_tools)
+    :ok = Application.ensure_loaded(:syntax_tools)
+    :ok = Application.ensure_loaded(:gpb)
+
     app_root = File.cwd!()
     app_build_path = Mix.Project.app_path(config)
 
@@ -22,12 +32,30 @@ defmodule Mix.Tasks.Compile.Grpc do
       |> Enum.map(& Path.join([app_root, &1]))
       |> Mix.Utils.extract_files([:proto])
 
-    Enum.each(proto_srcs, & compile_pb(&1, app_root, app_build_path, out_dir, gpb_opts))
+    manifest_data = read_manifest(manifest())
+    context = %{
+      manifest_data: manifest_data,
+      app_root: app_root,
+      app_build_path: app_build_path,
+      out_dir: out_dir,
+      gpb_opts: gpb_opts,
+    }
+
+    Enum.each(proto_srcs, & compile_pb(&1, context))
+
+    write_manifest(manifest(), manifest_data)
 
     {:noop, []}
   end
 
-  defp compile_pb(proto_src, app_root, app_build_path, out_dir, gpb_opts) do
+  defp compile_pb(proto_src, context) do
+    %{
+      app_root: app_root,
+      app_build_path: app_build_path,
+      out_dir: out_dir,
+      gpb_opts: gpb_opts,
+    } = context
+    manifest_modified_time = Mix.Utils.last_modified(manifest())
     ebin_path = Path.join([app_build_path, "ebin"])
     basename = proto_src |> Path.basename(".proto") |> to_charlist()
     prefix = Keyword.get(gpb_opts, :module_name_prefix, '')
@@ -43,48 +71,65 @@ defmodule Mix.Tasks.Compile.Grpc do
       rename: {:msg_name, :snake_case},
       rename: {:msg_fqname, :base_name},
     ]
-    File.mkdir_p!(out_dir)
+
-    # TODO: better error logging...
+    if stale?(proto_src, manifest_modified_time) do
-    :ok = :gpb_compile.file(
+      Mix.shell().info("compiling proto file: #{proto_src}")
-      to_charlist(proto_src),
+      File.mkdir_p!(out_dir)
-      opts ++ gpb_opts
+      # TODO: better error logging...
-    )
+      :ok = :gpb_compile.file(
+        to_charlist(proto_src),
+        opts ++ gpb_opts
+      )
+    else
+      Mix.shell().info("proto file up to date, not compiling: #{proto_src}")
+    end
+
     generated_src = Path.join([app_root, out_dir, "#{mod_name}.erl"])
-    |> IO.inspect(label: :generated_src)
-    generated_ebin = Path.join([ebin_path, "#{mod_name}.beam"])
-    |> IO.inspect(label: :generated_ebin)
     gpb_include_dir = :code.lib_dir(:gpb, :include)
 
-    compile_res = :compile.file(
+    if stale?(generated_src, manifest_modified_time) do
-      to_charlist(generated_src),
+      Mix.shell().info("compiling proto module: #{generated_src}")
-      [
+      compile_res = :compile.file(
-        :return_errors,
+        to_charlist(generated_src),
-        i: to_charlist(gpb_include_dir),
+        [
-        outdir: to_charlist(ebin_path)
+          :return_errors,
-      ]
+          i: to_charlist(gpb_include_dir),
-    )
+          outdir: to_charlist(ebin_path)
-    # todo: error handling & logging
+        ]
-    case compile_res do
+      )
-      {:ok, _} ->
+      # todo: error handling & logging
-        :ok
+      case compile_res do
+        {:ok, _} ->
+          :ok
 
-      {:ok, _, _warnings} ->
+        {:ok, _, _warnings} ->
-        :ok
+          :ok
+      end
+    else
+      Mix.shell().info("file up to date, not compiling: #{generated_src}")
     end
 
     mod_name
     |> List.to_atom()
     |> :code.purge()
 
-    {:module, mod} =
+    {:module, _mod} =
       ebin_path
       |> Path.join(mod_name)
       |> to_charlist()
       |> :code.load_abs()
 
     mod_name = List.to_atom(mod_name)
-    service_quoted = EEx.compile_file("lib/emqx/grpc/template/service.eex")
+    service_quoted =
-    client_quoted = EEx.compile_file("lib/emqx/grpc/template/client.eex")
+      [__DIR__, "../../", "emqx/grpc/template/service.eex"]
+      |> Path.join()
+      |> Path.expand()
+      |> EEx.compile_file()
+    client_quoted =
+      [__DIR__, "../../", "emqx/grpc/template/client.eex"]
+      |> Path.join()
+      |> Path.expand()
+      |> EEx.compile_file()
 
     mod_name.get_service_names()
     |> Enum.each(fn service ->
@@ -107,29 +152,74 @@ defmodule Mix.Tasks.Compile.Grpc do
           |> Macro.underscore()
           |> String.replace("/", "_")
           |> String.replace(~r/(.)([0-9]+)/, "\\1_\\2")
-        {result, _bindings} = Code.eval_quoted(
-          service_quoted,
-          methods: methods,
-          module_name: snake_service,
-          unmodified_service_name: service_name)
-        result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
-        output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"])
-        File.write!(output_src, result)
 
-        {result, _bindings} = Code.eval_quoted(
+        bindings = [
-          client_quoted,
           methods: methods,
           pb_module: mod_name,
           module_name: snake_service,
-          unmodified_service_name: service_name)
+          unmodified_service_name: service_name
-        result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
+        ]
-        output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"])
-        File.write!(output_src, result)
 
-        {{:service, service_name}, methods}
+        bhvr_output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"])
+        if stale?(bhvr_output_src, manifest_modified_time) do
+          render_and_write(service_quoted, bhvr_output_src, bindings)
+        else
+          Mix.shell().info("file up to date, not compiling: #{bhvr_output_src}")
+        end
+
+        client_output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"])
+        if stale?(client_output_src, manifest_modified_time) do
+          render_and_write(client_quoted, client_output_src, bindings)
+        else
+          Mix.shell().info("file up to date, not compiling: #{client_output_src}")
+        end
+
+        :ok
       end)
     end)
 
-    mod_name
+    :ok
+  end
+
+  defp stale?(file, manifest_modified_time) do
+    with true <- File.exists?(file),
+         false <- Mix.Utils.stale?([file], [manifest_modified_time]) do
+      false
+    else
+      _ -> true
+    end
+  end
+
+  defp read_manifest(file) do
+    try do
+      file |> File.read!() |> :erlang.binary_to_term()
+    rescue
+      _ -> %{}
+    else
+      {@manifest_vsn, data} when is_map(data) -> data
+      _ -> %{}
+    end
+  end
+
+  defp write_manifest(file, data) do
+    Mix.shell().info("writing manifest #{file}")
+    File.mkdir_p!(Path.dirname(file))
+    File.write!(file, :erlang.term_to_binary({@manifest_vsn, data}))
+  end
+
+  defp render_and_write(quoted_file, output_src, bindings) do
+    {result, _bindings} = Code.eval_quoted(quoted_file, bindings)
+    result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
+    File.write!(output_src, result)
+  end
+
+  def add_to_path_and_cache(lib_name) do
+    :code.lib_dir()
+    |> Path.join("#{lib_name}-*")
+    |> Path.wildcard()
+    |> hd()
+    |> Path.join("ebin")
+    |> to_charlist()
+    |> :code.add_path(:cache)
   end
 end

apps/emqx_gateway_exproto/mix.exs

@@ -7,6 +7,16 @@ defmodule EMQXGatewayExproto.MixProject do
       app: :emqx_gateway_exproto,
       version: "0.1.0",
       build_path: "../../_build",
+      compilers: [:elixir, :grpc, :erlang, :app],
+      # used by our `Mix.Tasks.Compile.Grpc` compiler
+      grpc_opts: %{
+        gpb_opts: [
+          module_name_prefix: 'emqx_',
+          module_name_suffix: '_pb',
+        ],
+        proto_dirs: ["priv/protos"],
+        out_dir: "src"
+      },
       erlc_options: UMP.erlc_options(),
       erlc_paths: UMP.erlc_paths(),
       deps_path: "../../deps",
@@ -22,7 +32,7 @@ defmodule EMQXGatewayExproto.MixProject do
   end
 
   def deps() do
-    test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true}], else: []
+    test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: []
     test_deps ++ [
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},