feat: emqx_release_helper

apps/emqx_release_helper/.formatter.exs

@@ -0,0 +1,11 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"],
+  locals_without_parens: [
+    start_type: 1,
+    overlay: 1,
+    mkdir: 1,
+    copy: 2,
+    template: 2
+  ]
+]

apps/emqx_release_helper/.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+emqx_release_helper-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

apps/emqx_release_helper/README.md

@@ -0,0 +1,21 @@
+# EmqxReleaseHelper
+
+**TODO: Add description**
+
+## Installation
+
+If [available in Hex](https://hex.pm/docs/publish), the package can be installed
+by adding `emqx_release_helper` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:emqx_release_helper, "~> 0.1.0"}
+  ]
+end
+```
+
+Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
+and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
+be found at [https://hexdocs.pm/emqx_release_helper](https://hexdocs.pm/emqx_release_helper).
+

apps/emqx_release_helper/lib/emqx_release_helper.ex

@@ -0,0 +1,130 @@
+defmodule EmqxReleaseHelper do
+  def applications do
+    config = profile_vars()
+
+    EmqxReleaseHelper.Applications.__all__()
+    |> Enum.filter(fn
+      %{enable?: fun} -> fun.(config)
+      _ -> true
+    end)
+    |> Enum.map(fn %{name: name, start_type: start_type} -> {name, start_type} end)
+  end
+
+  def run(release) do
+    config = Map.merge(profile_vars(), release_vars(release))
+
+    release
+    |> EmqxReleaseHelper.Overlay.run(config)
+    |> EmqxReleaseHelper.Applications.run(config)
+  end
+
+  def profile_vars() do
+    "RELEASE_PROFILE"
+    |> System.get_env("emqx")
+    |> String.to_existing_atom()
+    |> case do
+      :emqx ->
+        %{
+          release_type: :cloud,
+          package_type: :bin
+        }
+    end
+    |> Map.merge(%{
+      project_path: EMQXUmbrella.MixProject.project_path(),
+      enable_bcrypt: EMQXUmbrella.MixProject.enable_bcrypt(),
+      enable_plugin_emqx_modules: false,
+      enable_plugin_emqx_retainer: true,
+      apps_paths: Mix.Project.apps_paths(),
+      built_on_arch: get_arch()
+    })
+    |> then(fn %{release_type: release_type} = config ->
+      Map.merge(config, profile_vars(:release_type, release_type))
+    end)
+    |> then(fn %{package_type: package_type} = config ->
+      Map.merge(config, profile_vars(:package_type, package_type))
+    end)
+  end
+
+  defp profile_vars(:release_type, :cloud) do
+    %{
+      emqx_description: "EMQ X Broker",
+      enable_plugin_emqx_rule_engine: true,
+      enable_plugin_emqx_bridge_mqtt: false
+    }
+  end
+
+  defp profile_vars(:release_type, :edge) do
+    %{
+      emqx_description: "EMQ X Edge",
+      enable_plugin_emqx_rule_engine: false,
+      enable_plugin_emqx_bridge_mqtt: true
+    }
+  end
+
+  defp profile_vars(:package_type, :bin) do
+    %{
+      platform_bin_dir: "bin",
+      platform_data_dir: "data",
+      platform_etc_dir: "etc",
+      platform_lib_dir: "lib",
+      platform_log_dir: "log",
+      platform_plugins_dir: "etc/plugins",
+      runner_root_dir: EMQXUmbrella.MixProject.project_path(),
+      runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
+      runner_etc_dir: "$RUNNER_ROOT_DIR/etc",
+      runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
+      runner_log_dir: "$RUNNER_ROOT_DIR/log",
+      runner_data_dir: "$RUNNER_ROOT_DIR/data",
+      runner_user: ""
+    }
+  end
+
+  defp profile_vars(:package_type, :pkg) do
+    %{
+      platform_bin_dir: "",
+      platform_data_dir: "/var/lib/emqx",
+      platform_etc_dir: "/etc/emqx",
+      platform_lib_dir: "",
+      platform_log_dir: "/var/log/emqx",
+      platform_plugins_dir: "/var/lib/emqx/plugins",
+      runner_root_dir: "/usr/lib/emqx",
+      runner_bin_dir: "/usr/bin",
+      runner_etc_dir: "/etc/emqx",
+      runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
+      runner_log_dir: "/var/log/emqx",
+      runner_data_dir: "/var/lib/emqx",
+      runner_user: "emqx"
+    }
+  end
+
+  defp release_vars(release) do
+    %{
+      release_version: release.version,
+      release_path: release.path,
+      release_version_path: release.version_path
+    }
+  end
+
+  defp get_arch do
+    major_version = System.otp_release()
+
+    otp_release =
+      [:code.root_dir(), "releases", major_version, "OTP_VERSION"]
+      |> Path.join()
+      |> File.read()
+      |> case do
+        {:ok, version} -> String.trim(version)
+        {:error, _} -> major_version
+      end
+
+    wordsize =
+      try do
+        :erlang.system_info({:wordsize, :external}) * 8
+      rescue
+        _ ->
+          :erlang.system_info(:wordsize) * 8
+      end
+
+    Enum.join([otp_release, :erlang.system_info(:system_architecture), wordsize], "-")
+  end
+end

apps/emqx_release_helper/lib/emqx_release_helper/applications.ex

@@ -0,0 +1,161 @@
+defmodule EmqxReleaseHelper.Applications do
+  use EmqxReleaseHelper.DSL.Application
+
+  application :emqx do
+    start_type :permanent
+
+    overlay %{release_type: release_type} do
+      copy "etc/certs", "etc/certs"
+
+      template "etc/acl.conf", "etc/acl.conf"
+      template "etc/emqx.conf", "etc/emqx.conf"
+      template "etc/ssl_dist.conf", "etc/ssl_dist.conf"
+      template "etc/emqx_#{release_type}/vm.args", "etc/vm.args"
+    end
+  end
+
+  application :emqx_connector do
+    start_type :permanent
+  end
+
+  application :emqx_modules do
+    start_type :permanent
+  end
+
+  application :emqx_resource do
+    start_type :permanent
+  end
+
+  application :emqx_plugin_libs do
+    start_type :load
+  end
+
+  application :emqx_authz do
+    start_type :permanent
+    overlay :plugin
+  end
+
+  application :emqx_data_bridge do
+    start_type :permanent
+    overlay :plugin
+  end
+
+  application :emqx_sn do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_authentication do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_dashboard do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_management do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_statsd do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_stomp do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_bridge_mqtt do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_retainer do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_telemetry do
+    start_type :permanent
+    overlay :plugin
+  end
+
+  application :emqx_coap do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_rule_engine do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_web_hook do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_exhook, %{release_type: :cloud} do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_exproto, %{release_type: :cloud} do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_prometheus, %{release_type: :cloud} do
+    start_type :load
+    overlay :plugin
+  end
+
+  application :emqx_lwm2m, %{release_type: :cloud} do
+    start_type :load
+    overlay :plugin
+
+    overlay do
+      copy "lwm2m_xml", "etc/lwm2m_xml"
+    end
+  end
+
+  application :emqx_psk_file, %{release_type: :cloud} do
+    start_type :load
+    overlay :plugin
+
+    overlay do
+      copy "etc/psk.txt", "etc/psk.txt"
+    end
+  end
+
+  application :bcrypt, %{enable_bcrypt: true, release_type: :cloud} do
+    start_type :permanent
+  end
+
+  application :xmerl, %{release_type: :cloud} do
+    start_type :permanent
+  end
+
+  def run(release, config) do
+    %{project_path: project_path, apps_paths: apps_paths} = config
+
+    __all__()
+    |> Enum.filter(fn %{name: name} -> Map.has_key?(apps_paths, name) end)
+    |> Enum.filter(fn
+      %{enable?: fun} -> fun.(config)
+      _ -> true
+    end)
+    |> Enum.each(fn %{name: name, overlays: overlays} ->
+      app_path = Map.get(apps_paths, name)
+      config = Map.put(config, :app_source_path, Path.join(project_path, app_path))
+      Enum.each(overlays, fn overlay -> overlay.(config) end)
+    end)
+
+    release
+  end
+end

apps/emqx_release_helper/lib/emqx_release_helper/dsl.ex

@@ -0,0 +1,20 @@
+defmodule EmqxReleaseHelper.DSL do
+  defmacro __using__(_) do
+    quote do
+      import unquote(__MODULE__)
+      Module.register_attribute(__MODULE__, :applications, accumulate: true)
+      import EmqxReleaseHelper.DSL.Application
+      # @before_compile unquote(__MODULE__)
+    end
+  end
+
+  defmacro overlay(_) do
+  end
+
+  defmacro overlay(_, _) do
+  end
+
+  # defmacro __before_compile__(%Macro.Env{}=env) do
+  #   EmqxReleaseHelper.DSL.Application.__before_compile__(env)
+  # end
+end

apps/emqx_release_helper/lib/emqx_release_helper/dsl/application.ex

@@ -0,0 +1,162 @@
+defmodule EmqxReleaseHelper.DSL.Application do
+  defmacro __using__(_) do
+    quote do
+      import unquote(__MODULE__)
+      Module.register_attribute(__MODULE__, :applications, accumulate: true)
+      @before_compile unquote(__MODULE__)
+    end
+  end
+
+  defmacro application(app, condition, do: block) do
+    func =
+      Macro.escape(
+        quote do
+          fn config -> match?(unquote(condition), config) end
+        end
+      )
+
+    quote do
+      @current_application %{name: unquote(app), enable?: unquote(func)}
+      @overlays []
+      unquote(block)
+      overlays = Enum.reverse(@overlays)
+      @applications Map.put(@current_application, :overlays, overlays)
+      @current_application nil
+      @overlays []
+    end
+  end
+
+  defmacro application(app, do: block) do
+    quote do
+      @current_application %{name: unquote(app)}
+      @overlays []
+      unquote(block)
+      overlays = Enum.reverse(@overlays)
+      @applications Map.put(@current_application, :overlays, overlays)
+      @current_application nil
+      @overlays []
+    end
+  end
+
+  defmacro start_type(type) do
+    quote do
+      @current_application Map.put(@current_application, :start_type, unquote(type))
+    end
+  end
+
+  defmacro overlay(:plugin) do
+    block =
+      Macro.escape(
+        quote do
+          &plugin_overlay/1
+        end
+      )
+
+    quote do
+      @overlays [unquote(block) | @overlays]
+    end
+  end
+
+  defmacro overlay(do: block) do
+    block =
+      Macro.escape(
+        quote do
+          fn unquote(Macro.var(:config, nil)) ->
+            unquote(block)
+          end
+        end
+      )
+
+    quote do
+      @overlays [unquote(block) | @overlays]
+    end
+  end
+
+  defmacro overlay(literal_config, do: block) do
+    block =
+      Macro.escape(
+        quote do
+          fn unquote(literal_config) = unquote(Macro.var(:config, nil)) ->
+            unquote(block)
+          end
+        end
+      )
+
+    quote do
+      @overlays [unquote(block) | @overlays]
+    end
+  end
+
+  defmacro copy(from_path, to_path) do
+    from_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:app_source_path)
+        |> Path.join(unquote(from_path))
+      end
+
+    to_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:release_path)
+        |> Path.join(unquote(to_path))
+      end
+
+    quote do
+      EmqxReleaseHelper.DSL.Overlay.run_copy(unquote(from_path), unquote(to_path))
+    end
+  end
+
+  defmacro template(from_path, to_path) do
+    from_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:app_source_path)
+        |> Path.join(unquote(from_path))
+      end
+
+    to_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:release_path)
+        |> Path.join(unquote(to_path))
+      end
+
+    quote do
+      EmqxReleaseHelper.DSL.Overlay.run_template(
+        unquote(from_path),
+        unquote(to_path),
+        unquote(Macro.var(:config, nil))
+      )
+    end
+  end
+
+  def plugin_overlay(%{app_source_path: app_source_path, release_path: release_path} = config) do
+    "#{app_source_path}/etc"
+    |> File.ls()
+    |> case do
+      {:ok, files} -> files
+      {:error, _} -> []
+    end
+    |> Enum.filter(fn file -> String.ends_with?(file, ".conf") end)
+    |> Enum.each(fn file ->
+      EmqxReleaseHelper.DSL.Overlay.run_template(
+        "#{app_source_path}/etc/#{file}",
+        "#{release_path}/etc/plugins/#{file}",
+        config
+      )
+    end)
+  end
+
+  defmacro __before_compile__(%Macro.Env{module: module}) do
+    block =
+      module
+      |> Module.get_attribute(:applications)
+      |> Enum.reverse()
+      |> Enum.map(fn app -> {:%{}, [], Map.to_list(app)} end)
+
+    quote do
+      def __all__, do: unquote(block)
+    end
+  end
+end

apps/emqx_release_helper/lib/emqx_release_helper/dsl/overlay.ex

@@ -0,0 +1,130 @@
+defmodule EmqxReleaseHelper.DSL.Overlay do
+  defmacro __using__(_) do
+    quote do
+      import unquote(__MODULE__)
+      Module.register_attribute(__MODULE__, :overlays, accumulate: true)
+      @before_compile unquote(__MODULE__)
+    end
+  end
+
+  defmacro overlay(do: block) do
+    block =
+      Macro.escape(
+        quote do
+          fn unquote(Macro.var(:config, nil)) ->
+            unquote(block)
+          end
+        end
+      )
+
+    quote do
+      @overlays unquote(block)
+    end
+  end
+
+  defmacro overlay(literal_config, do: block) do
+    block =
+      Macro.escape(
+        quote do
+          fn unquote(literal_config) = unquote(Macro.var(:config, nil)) ->
+            unquote(block)
+          end
+        end
+      )
+
+    quote do
+      @overlays unquote(block)
+    end
+  end
+
+  defmacro mkdir(path) do
+    path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:release_path)
+        |> Path.join(unquote(path))
+      end
+
+    quote do
+      run_mkdir(unquote(path))
+    end
+  end
+
+  defmacro copy(from_path, to_path) do
+    from_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:project_path)
+        |> Path.join(unquote(from_path))
+      end
+
+    to_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:release_path)
+        |> Path.join(unquote(to_path))
+      end
+
+    quote do
+      run_copy(unquote(from_path), unquote(to_path))
+    end
+  end
+
+  defmacro template(from_path, to_path) do
+    from_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:project_path)
+        |> Path.join(unquote(from_path))
+      end
+
+    to_path =
+      quote do
+        unquote(Macro.var(:config, nil))
+        |> Map.get(:release_path)
+        |> Path.join(unquote(to_path))
+      end
+
+    quote do
+      run_template(
+        unquote(from_path),
+        unquote(to_path),
+        unquote(Macro.var(:config, nil))
+      )
+    end
+  end
+
+  def run_mkdir(path) do
+    File.mkdir_p!(path)
+  end
+
+  def run_copy(from_path, to_path) do
+    to_path |> Path.dirname() |> File.mkdir_p!()
+    File.cp_r!(from_path, to_path)
+  end
+
+  def run_template(from_path, to_path, config) do
+    config = Enum.map(config, fn {key, value} -> {to_charlist(key), value} end)
+    to_path |> Path.dirname() |> File.mkdir_p!()
+
+    content =
+      from_path
+      |> File.read!()
+      |> :bbmustache.render(config)
+
+    File.write!(to_path, content)
+  end
+
+  defmacro __before_compile__(%Macro.Env{module: module}) do
+    block =
+      module
+      |> Module.get_attribute(:overlays)
+      |> Enum.reverse()
+
+    quote do
+      def __overlays__ do
+        unquote(block)
+      end
+    end
+  end
+end

apps/emqx_release_helper/lib/emqx_release_helper/emqx.ex

@@ -0,0 +1,11 @@
+defmodule EmqxReleaseHelper.EMQX do
+  # use EmqxReleaseHelper
+
+  # with config do
+
+  # end
+
+  # module :emqx, with: config do
+
+  # end
+end

apps/emqx_release_helper/lib/emqx_release_helper/overlay.ex

@@ -0,0 +1,43 @@
+defmodule EmqxReleaseHelper.Overlay do
+  use EmqxReleaseHelper.DSL.Overlay
+
+  overlay %{release_version: release_version} do
+    mkdir "log"
+    mkdir "etc"
+    mkdir "etc/plugins"
+    mkdir "data"
+    mkdir "data/mnesia"
+    mkdir "data/configs"
+    mkdir "data/patches"
+    mkdir "data/scripts"
+
+    copy "bin/install_upgrade.escript", "bin/install_upgrade.escript"
+
+    copy "bin/node_dump", "bin/node_dump"
+    copy "bin/nodetool", "bin/nodetool"
+    copy "bin/nodetool", "bin/nodetool-#{release_version}"
+
+    # copy "bin/emqx", "bin/emqx"
+    # copy "bin/emqx_ctl", "bin/emqx_ctl"
+
+    # for relup
+    copy "bin/emqx", "bin/emqx-#{release_version}"
+    copy "bin/emqx_ctl", "bin/emqx_ctl-#{release_version}"
+
+    copy "bin/install_upgrade.escript", "bin/install_upgrade.escript-#{release_version}"
+
+    template "data/loaded_plugins.tmpl", "data/loaded_plugins"
+
+    template "data/loaded_modules.tmpl", "data/loaded_modules"
+
+    template "data/emqx_vars", "releases/emqx_vars"
+    template "data/BUILT_ON", "releases/#{release_version}/BUILT_ON"
+    # template "bin/emqx.cmd", "bin/emqx.cmd"
+    # template "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"
+  end
+
+  def run(release, config) do
+    Enum.each(__overlays__(), fn overlay -> overlay.(config) end)
+    release
+  end
+end

apps/emqx_release_helper/mix.exs

@@ -0,0 +1,29 @@
+defmodule EMQXReleaseHelper.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :emqx_release_helper,
+      version: "0.1.0",
+      build_path: "../../_build",
+      config_path: "../../config/config.exs",
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.12",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  defp deps do
+    [
+      {:bbmustache, "~> 1.12"}
+    ]
+  end
+end

apps/emqx_release_helper/test/emqx_release_helper_test.exs

@@ -0,0 +1,8 @@
+defmodule EmqxReleaseHelperTest do
+  use ExUnit.Case
+  doctest EmqxReleaseHelper
+
+  test "greets the world" do
+    assert EmqxReleaseHelper.hello() == :world
+  end
+end

apps/emqx_release_helper/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

config/config.exs

@@ -0,0 +1 @@
+import Config

config/runtime.exs

@@ -0,0 +1,9 @@
+import Config
+# import MyApplication.Hocon
+
+# config :my_app, :key, :value
+
+# hocon :emqx,
+#    schema: :emqx_schema,
+#    config_file: "etc/emqx.conf"
+config :mnesia, dir: '/tmp/mnesia'

mix.exs

@@ -4,9 +4,10 @@ defmodule EMQXUmbrella.MixProject do
   def project do
     [
       apps_path: "apps",
-      version: "0.1.0",
+      version: "5.0.0",
       start_permanent: Mix.env() == :prod,
-      deps: deps()
+      deps: deps(),
+      releases: releases()
     ]
   end
 
@@ -30,6 +31,26 @@ defmodule EMQXUmbrella.MixProject do
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.2", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
       {:typerefl, github: "k32/typerefl", tag: "0.6.2", manager: :rebar3, override: true}
+      | (enable_bcrypt() && [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0"}]) || []
     ]
   end
+
+  defp releases do
+    [
+      emqx: fn ->
+        [
+          applications: EmqxReleaseHelper.applications(),
+          steps: [:assemble, &EmqxReleaseHelper.run/1]
+        ]
+      end
+    ]
+  end
+
+  def enable_bcrypt do
+    not match?({:win_32, _}, :os.type())
+  end
+
+  def project_path do
+    Path.expand("..", __ENV__.file)
+  end
 end

mix.lock

@@ -1,4 +1,6 @@
 %{
+  "bbmustache": {:hex, :bbmustache, "1.12.1", "857fbdf86bda46d07201b0e7a969820cb763a7c174c485fd0780d7e033efe9f0", [:rebar3], [], "hexpm", "f4320778c31a821a2a664db8894618abb79c1af7bbf7c03c703c8868d9bb09fe"},
+  "bcrypt": {:git, "https://github.com/emqx/erlang-bcrypt.git", "dc2ba66acf2332c111362d01137746eefecc5e90", [tag: "0.6.0"]},
   "cowboy": {:git, "https://github.com/emqx/cowboy.git", "b89d4689a04149b1a4a3641280aa5c5643f921b2", [tag: "2.8.2"]},
   "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"},
   "cuttlefish": {:git, "https://github.com/emqx/cuttlefish.git", "1180224fb60d87ef41307c949453248d4ebef761", []},