perf: use manifest to track proto file compilation
This commit is contained in:
Thales Macedo Garitezi
parent
19f3b030f9
commit
8843fcbbf4
|
|
@ -2,8 +2,14 @@ defmodule Mix.Tasks.Compile.Grpc do
|
|||
use Mix.Task.Compiler
|
||||
|
||||
@recursive true
|
||||
@manifest_vsn 1
|
||||
@manifest "compile.grpc"
|
||||
# TODO: use manifest to track generated files?
|
||||
|
||||
@impl true
|
||||
def manifests(), do: [manifest()]
|
||||
defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest)
|
||||
|
||||
@impl true
|
||||
def run(_args) do
|
||||
Mix.Project.get!()
|
||||
|
|
@ -14,6 +20,10 @@ defmodule Mix.Tasks.Compile.Grpc do
|
|||
out_dir: out_dir
|
||||
} = config[:grpc_opts]
|
||||
|
||||
add_to_path_and_cache(:syntax_tools)
|
||||
:ok = Application.ensure_loaded(:syntax_tools)
|
||||
:ok = Application.ensure_loaded(:gpb)
|
||||
|
||||
app_root = File.cwd!()
|
||||
app_build_path = Mix.Project.app_path(config)
|
||||
|
||||
|
|
@ -22,12 +32,30 @@ defmodule Mix.Tasks.Compile.Grpc do
|
|||
|> Enum.map(& Path.join([app_root, &1]))
|
||||
|> Mix.Utils.extract_files([:proto])
|
||||
|
||||
Enum.each(proto_srcs, & compile_pb(&1, app_root, app_build_path, out_dir, gpb_opts))
|
||||
manifest_data = read_manifest(manifest())
|
||||
context = %{
|
||||
manifest_data: manifest_data,
|
||||
app_root: app_root,
|
||||
app_build_path: app_build_path,
|
||||
out_dir: out_dir,
|
||||
gpb_opts: gpb_opts,
|
||||
}
|
||||
|
||||
Enum.each(proto_srcs, & compile_pb(&1, context))
|
||||
|
||||
write_manifest(manifest(), manifest_data)
|
||||
|
||||
{:noop, []}
|
||||
end
|
||||
|
||||
defp compile_pb(proto_src, app_root, app_build_path, out_dir, gpb_opts) do
|
||||
defp compile_pb(proto_src, context) do
|
||||
%{
|
||||
app_root: app_root,
|
||||
app_build_path: app_build_path,
|
||||
out_dir: out_dir,
|
||||
gpb_opts: gpb_opts,
|
||||
} = context
|
||||
manifest_modified_time = Mix.Utils.last_modified(manifest())
|
||||
ebin_path = Path.join([app_build_path, "ebin"])
|
||||
basename = proto_src |> Path.basename(".proto") |> to_charlist()
|
||||
prefix = Keyword.get(gpb_opts, :module_name_prefix, '')
|
||||
|
|
@ -43,48 +71,65 @@ defmodule Mix.Tasks.Compile.Grpc do
|
|||
rename: {:msg_name, :snake_case},
|
||||
rename: {:msg_fqname, :base_name},
|
||||
]
|
||||
File.mkdir_p!(out_dir)
|
||||
# TODO: better error logging...
|
||||
:ok = :gpb_compile.file(
|
||||
to_charlist(proto_src),
|
||||
opts ++ gpb_opts
|
||||
)
|
||||
|
||||
if stale?(proto_src, manifest_modified_time) do
|
||||
Mix.shell().info("compiling proto file: #{proto_src}")
|
||||
File.mkdir_p!(out_dir)
|
||||
# TODO: better error logging...
|
||||
:ok = :gpb_compile.file(
|
||||
to_charlist(proto_src),
|
||||
opts ++ gpb_opts
|
||||
)
|
||||
else
|
||||
Mix.shell().info("proto file up to date, not compiling: #{proto_src}")
|
||||
end
|
||||
|
||||
generated_src = Path.join([app_root, out_dir, "#{mod_name}.erl"])
|
||||
|> IO.inspect(label: :generated_src)
|
||||
generated_ebin = Path.join([ebin_path, "#{mod_name}.beam"])
|
||||
|> IO.inspect(label: :generated_ebin)
|
||||
gpb_include_dir = :code.lib_dir(:gpb, :include)
|
||||
|
||||
compile_res = :compile.file(
|
||||
to_charlist(generated_src),
|
||||
[
|
||||
:return_errors,
|
||||
i: to_charlist(gpb_include_dir),
|
||||
outdir: to_charlist(ebin_path)
|
||||
]
|
||||
)
|
||||
# todo: error handling & logging
|
||||
case compile_res do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
if stale?(generated_src, manifest_modified_time) do
|
||||
Mix.shell().info("compiling proto module: #{generated_src}")
|
||||
compile_res = :compile.file(
|
||||
to_charlist(generated_src),
|
||||
[
|
||||
:return_errors,
|
||||
i: to_charlist(gpb_include_dir),
|
||||
outdir: to_charlist(ebin_path)
|
||||
]
|
||||
)
|
||||
# todo: error handling & logging
|
||||
case compile_res do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
{:ok, _, _warnings} ->
|
||||
:ok
|
||||
{:ok, _, _warnings} ->
|
||||
:ok
|
||||
end
|
||||
else
|
||||
Mix.shell().info("file up to date, not compiling: #{generated_src}")
|
||||
end
|
||||
|
||||
mod_name
|
||||
|> List.to_atom()
|
||||
|> :code.purge()
|
||||
|
||||
{:module, mod} =
|
||||
{:module, _mod} =
|
||||
ebin_path
|
||||
|> Path.join(mod_name)
|
||||
|> to_charlist()
|
||||
|> :code.load_abs()
|
||||
|
||||
mod_name = List.to_atom(mod_name)
|
||||
service_quoted = EEx.compile_file("lib/emqx/grpc/template/service.eex")
|
||||
client_quoted = EEx.compile_file("lib/emqx/grpc/template/client.eex")
|
||||
service_quoted =
|
||||
[__DIR__, "../../", "emqx/grpc/template/service.eex"]
|
||||
|> Path.join()
|
||||
|> Path.expand()
|
||||
|> EEx.compile_file()
|
||||
client_quoted =
|
||||
[__DIR__, "../../", "emqx/grpc/template/client.eex"]
|
||||
|> Path.join()
|
||||
|> Path.expand()
|
||||
|> EEx.compile_file()
|
||||
|
||||
mod_name.get_service_names()
|
||||
|> Enum.each(fn service ->
|
||||
|
|
@ -107,29 +152,74 @@ defmodule Mix.Tasks.Compile.Grpc do
|
|||
|> Macro.underscore()
|
||||
|> String.replace("/", "_")
|
||||
|> String.replace(~r/(.)([0-9]+)/, "\\1_\\2")
|
||||
{result, _bindings} = Code.eval_quoted(
|
||||
service_quoted,
|
||||
methods: methods,
|
||||
module_name: snake_service,
|
||||
unmodified_service_name: service_name)
|
||||
result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
|
||||
output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"])
|
||||
File.write!(output_src, result)
|
||||
|
||||
{result, _bindings} = Code.eval_quoted(
|
||||
client_quoted,
|
||||
bindings = [
|
||||
methods: methods,
|
||||
pb_module: mod_name,
|
||||
module_name: snake_service,
|
||||
unmodified_service_name: service_name)
|
||||
result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
|
||||
output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"])
|
||||
File.write!(output_src, result)
|
||||
unmodified_service_name: service_name
|
||||
]
|
||||
|
||||
{{:service, service_name}, methods}
|
||||
bhvr_output_src = Path.join([app_root, out_dir, "#{snake_service}_bhvr.erl"])
|
||||
if stale?(bhvr_output_src, manifest_modified_time) do
|
||||
render_and_write(service_quoted, bhvr_output_src, bindings)
|
||||
else
|
||||
Mix.shell().info("file up to date, not compiling: #{bhvr_output_src}")
|
||||
end
|
||||
|
||||
client_output_src = Path.join([app_root, out_dir, "#{snake_service}_client.erl"])
|
||||
if stale?(client_output_src, manifest_modified_time) do
|
||||
render_and_write(client_quoted, client_output_src, bindings)
|
||||
else
|
||||
Mix.shell().info("file up to date, not compiling: #{client_output_src}")
|
||||
end
|
||||
|
||||
:ok
|
||||
end)
|
||||
end)
|
||||
|
||||
mod_name
|
||||
:ok
|
||||
end
|
||||
|
||||
defp stale?(file, manifest_modified_time) do
|
||||
with true <- File.exists?(file),
|
||||
false <- Mix.Utils.stale?([file], [manifest_modified_time]) do
|
||||
false
|
||||
else
|
||||
_ -> true
|
||||
end
|
||||
end
|
||||
|
||||
defp read_manifest(file) do
|
||||
try do
|
||||
file |> File.read!() |> :erlang.binary_to_term()
|
||||
rescue
|
||||
_ -> %{}
|
||||
else
|
||||
{@manifest_vsn, data} when is_map(data) -> data
|
||||
_ -> %{}
|
||||
end
|
||||
end
|
||||
|
||||
defp write_manifest(file, data) do
|
||||
Mix.shell().info("writing manifest #{file}")
|
||||
File.mkdir_p!(Path.dirname(file))
|
||||
File.write!(file, :erlang.term_to_binary({@manifest_vsn, data}))
|
||||
end
|
||||
|
||||
defp render_and_write(quoted_file, output_src, bindings) do
|
||||
{result, _bindings} = Code.eval_quoted(quoted_file, bindings)
|
||||
result = String.replace(result, ~r/\n\n\n+/, "\n\n\n")
|
||||
File.write!(output_src, result)
|
||||
end
|
||||
|
||||
def add_to_path_and_cache(lib_name) do
|
||||
:code.lib_dir()
|
||||
|> Path.join("#{lib_name}-*")
|
||||
|> Path.wildcard()
|
||||
|> hd()
|
||||
|> Path.join("ebin")
|
||||
|> to_charlist()
|
||||
|> :code.add_path(:cache)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -7,6 +7,16 @@ defmodule EMQXGatewayExproto.MixProject do
|
|||
app: :emqx_gateway_exproto,
|
||||
version: "0.1.0",
|
||||
build_path: "../../_build",
|
||||
compilers: [:elixir, :grpc, :erlang, :app],
|
||||
# used by our `Mix.Tasks.Compile.Grpc` compiler
|
||||
grpc_opts: %{
|
||||
gpb_opts: [
|
||||
module_name_prefix: 'emqx_',
|
||||
module_name_suffix: '_pb',
|
||||
],
|
||||
proto_dirs: ["priv/protos"],
|
||||
out_dir: "src"
|
||||
},
|
||||
erlc_options: UMP.erlc_options(),
|
||||
erlc_paths: UMP.erlc_paths(),
|
||||
deps_path: "../../deps",
|
||||
|
|
@ -22,7 +32,7 @@ defmodule EMQXGatewayExproto.MixProject do
|
|||
end
|
||||
|
||||
def deps() do
|
||||
test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true}], else: []
|
||||
test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: []
|
||||
test_deps ++ [
|
||||
{:emqx, in_umbrella: true},
|
||||
{:emqx_utils, in_umbrella: true},
|
||||
|
|
|
|||
Loading…
Reference in New Issue