From 917575de5a5493ecffa593ddb4275c07da1beb57 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 20 Dec 2021 17:41:15 -0300 Subject: [PATCH 01/31] chore(mix): minimal elixir mix release build This commit enables a minimal working build of EMQX release using Mix. However, to properly start the release, several configuration steps are still missing. A `mix_release.sh` script does a few hacks to get the release built with Mix to start properly, by first assuming that `make emqx` has been run prior to the release, ran once to generate the `app.*.config` files, and then it copies that and some other files to the expected places. Also, `emqx_telemetry` hangs the start procedure because it thinks it's in an official release and tries to make a request. We disable it temporarily via config just to get a working build for now. --- .github/workflows/elixir_release.yml | 39 + .tool-versions | 1 + apps/emqx/mix.exs | 39 + apps/emqx_authn/mix.exs | 21 + apps/emqx_authz/mix.exs | 23 + apps/emqx_auto_subscribe/mix.exs | 21 + apps/emqx_bridge/mix.exs | 21 + apps/emqx_conf/mix.exs | 23 + apps/emqx_connector/mix.exs | 28 + apps/emqx_dashboard/mix.exs | 23 + apps/emqx_exhook/mix.exs | 29 + .../{src/coap => }/include/emqx_coap.hrl | 0 .../exproto => }/include/emqx_exproto.hrl | 0 .../{src/lwm2m => }/include/emqx_lwm2m.hrl | 0 .../{src/mqttsn => }/include/emqx_sn.hrl | 0 .../{src/stomp => }/include/emqx_stomp.hrl | 0 apps/emqx_gateway/mix.exs | 32 + apps/emqx_gateway/src/coap/emqx_coap_api.erl | 2 +- .../src/coap/emqx_coap_channel.erl | 2 +- .../emqx_gateway/src/coap/emqx_coap_frame.erl | 4 +- .../src/coap/emqx_coap_medium.erl | 2 +- .../src/coap/emqx_coap_message.erl | 2 +- .../src/coap/emqx_coap_session.erl | 2 +- apps/emqx_gateway/src/coap/emqx_coap_tm.erl | 2 +- .../src/coap/emqx_coap_transport.erl | 2 +- .../coap/handler/emqx_coap_mqtt_handler.erl | 2 +- .../coap/handler/emqx_coap_pubsub_handler.erl | 2 +- .../src/exproto/emqx_exproto_channel.erl | 2 +- .../src/exproto/emqx_exproto_gsvr.erl | 2 +- .../src/lwm2m/emqx_lwm2m_channel.erl | 4 +- .../emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl | 4 +- .../src/lwm2m/emqx_lwm2m_message.erl | 2 +- .../src/lwm2m/emqx_lwm2m_session.erl | 4 +- .../emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl | 2 +- .../src/lwm2m/emqx_lwm2m_xml_object.erl | 2 +- .../src/lwm2m/emqx_lwm2m_xml_object_db.erl | 2 +- .../src/mqttsn/emqx_sn_broadcast.erl | 2 +- .../src/mqttsn/emqx_sn_channel.erl | 2 +- .../emqx_gateway/src/mqttsn/emqx_sn_frame.erl | 2 +- .../src/mqttsn/emqx_sn_registry.erl | 2 +- .../src/stomp/emqx_stomp_channel.erl | 2 +- .../src/stomp/emqx_stomp_frame.erl | 2 +- .../src/stomp/emqx_stomp_heartbeat.erl | 2 +- .../emqx_gateway/test/emqx_coap_api_SUITE.erl | 2 +- apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl | 4 +- .../test/emqx_lwm2m_api_SUITE.erl | 4 +- apps/emqx_gateway/test/emqx_tlv_SUITE.erl | 4 +- .../test/props/prop_emqx_sn_frame.erl | 2 +- apps/emqx_machine/mix.exs | 21 + apps/emqx_management/mix.exs | 21 + apps/emqx_modules/mix.exs | 21 + apps/emqx_plugin_libs/mix.exs | 21 + apps/emqx_plugins/mix.exs | 21 + apps/emqx_prometheus/mix.exs | 21 + apps/emqx_prometheus/rebar.config | 3 +- apps/emqx_psk/mix.exs | 21 + apps/emqx_resource/mix.exs | 21 + apps/emqx_resource/src/emqx_resource.app.src | 4 +- apps/emqx_retainer/mix.exs | 21 + apps/emqx_rule_engine/mix.exs | 21 + apps/emqx_slow_subs/mix.exs | 21 + apps/emqx_statsd/mix.exs | 21 + lib/emqx/mix/common.ex | 107 +++ lib/mix/release.exs | 878 ++++++++++++++++++ mix.exs | 206 ++++ mix.lock | 59 ++ mix_release.sh | 26 + rel/env.bat.eex | 5 + rel/env.sh.eex | 17 + rel/remote.vm.args.eex | 11 + rel/vm.args.eex | 11 + 71 files changed, 1914 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/elixir_release.yml create mode 100644 apps/emqx/mix.exs create mode 100644 apps/emqx_authn/mix.exs create mode 100644 apps/emqx_authz/mix.exs create mode 100644 apps/emqx_auto_subscribe/mix.exs create mode 100644 apps/emqx_bridge/mix.exs create mode 100644 apps/emqx_conf/mix.exs create mode 100644 apps/emqx_connector/mix.exs create mode 100644 apps/emqx_dashboard/mix.exs create mode 100644 apps/emqx_exhook/mix.exs rename apps/emqx_gateway/{src/coap => }/include/emqx_coap.hrl (100%) rename apps/emqx_gateway/{src/exproto => }/include/emqx_exproto.hrl (100%) rename apps/emqx_gateway/{src/lwm2m => }/include/emqx_lwm2m.hrl (100%) rename apps/emqx_gateway/{src/mqttsn => }/include/emqx_sn.hrl (100%) rename apps/emqx_gateway/{src/stomp => }/include/emqx_stomp.hrl (100%) create mode 100644 apps/emqx_gateway/mix.exs create mode 100644 apps/emqx_machine/mix.exs create mode 100644 apps/emqx_management/mix.exs create mode 100644 apps/emqx_modules/mix.exs create mode 100644 apps/emqx_plugin_libs/mix.exs create mode 100644 apps/emqx_plugins/mix.exs create mode 100644 apps/emqx_prometheus/mix.exs create mode 100644 apps/emqx_psk/mix.exs create mode 100644 apps/emqx_resource/mix.exs create mode 100644 apps/emqx_retainer/mix.exs create mode 100644 apps/emqx_rule_engine/mix.exs create mode 100644 apps/emqx_slow_subs/mix.exs create mode 100644 apps/emqx_statsd/mix.exs create mode 100644 lib/emqx/mix/common.ex create mode 100644 lib/mix/release.exs create mode 100644 mix.exs create mode 100644 mix.lock create mode 100755 mix_release.sh create mode 100644 rel/env.bat.eex create mode 100644 rel/env.sh.eex create mode 100644 rel/remote.vm.args.eex create mode 100644 rel/vm.args.eex diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml new file mode 100644 index 000000000..f2aef28c9 --- /dev/null +++ b/.github/workflows/elixir_release.yml @@ -0,0 +1,39 @@ +# FIXME: temporary workflow for testing; remove later +name: Elixir Build (temporary) + +concurrency: + group: mix-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + - name: "[hack] make emqx with rebar3 to get configs" + run: | + make emqx + cd _build/emqx/rel/emqx + bin/emqx start + bin/emqx stop + - name: setup mix + run: | + mix local.hex --force + mix local.rebar --force + mix deps.get + - name: elixir release + run: ./mix_release.sh + - name: start release + run: | + cd _build/dev/rel/emqx + bin/emqx daemon_iex + - name: check if started + run: | + sleep 10 + nc -zv localhost 1883 diff --git a/.tool-versions b/.tool-versions index a6568713b..1c687075b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ erlang 24.1.5-3 +elixir 1.13.1-otp-24 diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs new file mode 100644 index 000000000..847002cc2 --- /dev/null +++ b/apps/emqx/mix.exs @@ -0,0 +1,39 @@ +defmodule EMQX.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + [ + mod: EMQX.Mix.Common.from_erl!(:emqx, :mod), + applications: EMQX.Mix.Common.from_erl!(:emqx, :applications) + ] + end + + # since emqx app is more complicated than others, we manually set + # its dependencies here + defp deps() do + [ + {:lc, git: "https://github.com/qzhuyan/lc.git", tag: "0.1.2"}, + {:gproc, git: "https://github.com/uwiger/gproc", tag: "0.8.0"}, + {:typerefl, git: "https://github.com/k32/typerefl", tag: "0.8.5"}, + {:jiffy, git: "https://github.com/emqx/jiffy", tag: "1.0.5"}, + {:cowboy, git: "https://github.com/emqx/cowboy", tag: "2.9.0"}, + {:esockd, git: "https://github.com/emqx/esockd", tag: "5.9.0"}, + {:ekka, git: "https://github.com/emqx/ekka", tag: "0.11.1"}, + {:gen_rpc, git: "https://github.com/emqx/gen_rpc", tag: "2.5.1"}, + {:hocon, git: "https://github.com/emqx/hocon.git", tag: "0.22.0"}, + {:pbkdf2, git: "https://github.com/emqx/erlang-pbkdf2.git", tag: "2.0.4"}, + {:recon, git: "https://github.com/ferd/recon", tag: "2.5.1"}, + {:snabbkaffe, git: "https://github.com/kafka4beam/snabbkaffe.git", tag: "0.16.0"} + ] + end +end diff --git a/apps/emqx_authn/mix.exs b/apps/emqx_authn/mix.exs new file mode 100644 index 000000000..07af06495 --- /dev/null +++ b/apps/emqx_authn/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXAuthn.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_authn + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_authz/mix.exs b/apps/emqx_authz/mix.exs new file mode 100644 index 000000000..06a5792b4 --- /dev/null +++ b/apps/emqx_authz/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXAuthz.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_authz + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app, + extra_applications: [:crypto] + ) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_auto_subscribe/mix.exs b/apps/emqx_auto_subscribe/mix.exs new file mode 100644 index 000000000..d60e8f03e --- /dev/null +++ b/apps/emqx_auto_subscribe/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXAutoSubscribe.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_auto_subscribe + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_bridge/mix.exs b/apps/emqx_bridge/mix.exs new file mode 100644 index 000000000..629f1295c --- /dev/null +++ b/apps/emqx_bridge/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXBridge.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_bridge + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_conf/mix.exs b/apps/emqx_conf/mix.exs new file mode 100644 index 000000000..1ad98ee70 --- /dev/null +++ b/apps/emqx_conf/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXConf.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_conf + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs new file mode 100644 index 000000000..86c40a6d2 --- /dev/null +++ b/apps/emqx_connector/mix.exs @@ -0,0 +1,28 @@ +defmodule EMQXConnector.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_connector + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx_resource, in_umbrella: true, runtime: false}, + {:epgsql, github: "epgsql/epgsql", tag: "4.4.0"}, + {:mysql, github: "emqx/mysql-otp", tag: "1.7.1"}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, + {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.6.7"}, + {:mongodb, github: "emqx/mongodb-erlang", tag: "v3.0.10"} + ] + end +end diff --git a/apps/emqx_dashboard/mix.exs b/apps/emqx_dashboard/mix.exs new file mode 100644 index 000000000..e70feef50 --- /dev/null +++ b/apps/emqx_dashboard/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXDashboard.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_dashboard + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs new file mode 100644 index 000000000..85f67b40c --- /dev/null +++ b/apps/emqx_exhook/mix.exs @@ -0,0 +1,29 @@ +defmodule EMQXExhook.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_exhook + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps(), + compilers: [:protos | Mix.compilers()], + aliases: ["compile.protos": &protos/1] + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end + + defp protos(_args) do + __ENV__.file + |> Path.dirname() + |> EMQX.Mix.Common.compile_protos() + end +end diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_gateway/src/coap/include/emqx_coap.hrl rename to apps/emqx_gateway/include/emqx_coap.hrl diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_gateway/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl rename to apps/emqx_gateway/include/emqx_exproto.hrl diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_gateway/include/emqx_lwm2m.hrl diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_gateway/include/emqx_sn.hrl similarity index 100% rename from apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl rename to apps/emqx_gateway/include/emqx_sn.hrl diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl rename to apps/emqx_gateway/include/emqx_stomp.hrl diff --git a/apps/emqx_gateway/mix.exs b/apps/emqx_gateway/mix.exs new file mode 100644 index 000000000..07267088d --- /dev/null +++ b/apps/emqx_gateway/mix.exs @@ -0,0 +1,32 @@ +defmodule EMQXGateway.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_gateway + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps(), + compilers: [:protos | Mix.compilers()], + aliases: ["compile.protos": &protos/1] + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp protos(_args) do + __ENV__.file + |> Path.dirname() + |> EMQX.Mix.Common.compile_protos() + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() ++ + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 0efb8043e..f07e2fbf8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -18,7 +18,7 @@ -behaviour(minirest_api). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([api_spec/0]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index ab079b587..da51def5d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -43,7 +43,7 @@ -export_type([channel/0]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index a567ea7f4..d8d88a4cc 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -28,8 +28,8 @@ , is_message/1 ]). --include("include/emqx_coap.hrl"). --include("apps/emqx/include/types.hrl"). +-include("emqx_coap.hrl"). +-include_lib("emqx/include/types.hrl"). -define(VERSION, 1). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl index 020e38496..d34b1f90c 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ empty/0, reset/1, reset/2 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index 3851b3428..93f15fb6d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -34,7 +34,7 @@ -export([ set/3, set_payload/2, get_option/2 , get_option/3, set_payload_block/3, set_payload_block/4]). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index 9c3a8c451..49d5fe68e 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -18,7 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ new/0 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index d6d05fd40..f6247b6bb 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -28,7 +28,7 @@ -export_type([manager/0, event_result/1]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -type direction() :: in | out. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 07e522309..bb09745b0 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -1,7 +1,7 @@ -module(emqx_coap_transport). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 47bf14d9b..98cb74a42 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 608eae92a..1ff3b9cbb 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index bb8072358..464dda263 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index 1b9afca62..95626abbe 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -19,7 +19,7 @@ % -behaviour(emqx_exproto_v_1_connection_adapter_bhvr). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index c01c6adc5..9d6eee5cc 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_channel). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). %% API -export([ info/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 1ee27228b..d7b056903 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_cmd). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). -export([ mqtt_to_coap/2 , coap_to_mqtt/4 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl index 70844e6d6..f14cf9d9c 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl @@ -23,7 +23,7 @@ , translate_json/1 ]). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index bee1bedcd..c797c5a08 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -18,8 +18,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). %% API -export([ new/0, init/4, update/3, parse_object_list/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl index 94ea31bf8..02c5ecd9f 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -define(TLV_TYPE_OBJECT_INSTANCE, 0). -define(TLV_TYPE_RESOURCE_INSTANCE, 1). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index d744a23f9..9549c62bb 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 3cef3c19e..92c0c6ebe 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl index 0b90d843c..1c188ce95 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ start_link/2 diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index 74c8d2c12..7a18f1236 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl index 59acc71ce..9bac188c8 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl @@ -20,7 +20,7 @@ -behaviour(emqx_gateway_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -export([ initial_parse_state/1 , serialize_opts/0 diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl index 411a34aec..25c7bd2b0 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl @@ -21,7 +21,7 @@ -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ start_link/2 diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index dfca57cf1..82cfdb142 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl index d75f77abe..18b5c8f55 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl @@ -70,7 +70,7 @@ -behaviour(emqx_gateway_frame). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ initial_parse_state/1 , parse/2 diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl index 99a1508e1..72217552b 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl @@ -17,7 +17,7 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ init/1 , check/3 diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 5e342ec76..3af3418f4 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index dd8fc2f7f..5f74c09c7 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 0782ab1b3..547775dbc 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index 1191d748e..56b8f1c1a 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl index 9e12a7bd4..b377864bb 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include_lib("src/mqttsn/include/emqx_sn.hrl"). +-include_lib("emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_machine/mix.exs b/apps/emqx_machine/mix.exs new file mode 100644 index 000000000..2943fe1fa --- /dev/null +++ b/apps/emqx_machine/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXMachine.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_machine + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_management/mix.exs b/apps/emqx_management/mix.exs new file mode 100644 index 000000000..d175bc801 --- /dev/null +++ b/apps/emqx_management/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXManagement.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_management + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_modules/mix.exs b/apps/emqx_modules/mix.exs new file mode 100644 index 000000000..e41d907cc --- /dev/null +++ b/apps/emqx_modules/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXModules.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_modules + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_plugin_libs/mix.exs b/apps/emqx_plugin_libs/mix.exs new file mode 100644 index 000000000..74ee89c02 --- /dev/null +++ b/apps/emqx_plugin_libs/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPluginLibs.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_plugin_libs + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_plugins/mix.exs b/apps/emqx_plugins/mix.exs new file mode 100644 index 000000000..4b5ffdda2 --- /dev/null +++ b/apps/emqx_plugins/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPlugins.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_plugins + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_prometheus/mix.exs b/apps/emqx_prometheus/mix.exs new file mode 100644 index 000000000..84985fc41 --- /dev/null +++ b/apps/emqx_prometheus/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPrometheus.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_prometheus + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app, deps: deps()) + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() + end +end diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 02e81b295..c18bde7a0 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -1,5 +1,6 @@ {deps, - [{prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v3.1.1"}}} + [ %% FIXME: tag this as v3.1.3 + {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_psk/mix.exs b/apps/emqx_psk/mix.exs new file mode 100644 index 000000000..3395a5a33 --- /dev/null +++ b/apps/emqx_psk/mix.exs @@ -0,0 +1,21 @@ +defmodule EmqxPSK.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_psk + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_resource/mix.exs b/apps/emqx_resource/mix.exs new file mode 100644 index 000000000..7520758e7 --- /dev/null +++ b/apps/emqx_resource/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXResource.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_resource + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 1b93aa0de..8e5d12fea 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -8,9 +8,9 @@ [kernel, stdlib, gproc, - hocon, jsx, - emqx + emqx, + emqx_conf ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_retainer/mix.exs b/apps/emqx_retainer/mix.exs new file mode 100644 index 000000000..57d1f9ddf --- /dev/null +++ b/apps/emqx_retainer/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXRetainer.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_retainer + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_rule_engine/mix.exs b/apps/emqx_rule_engine/mix.exs new file mode 100644 index 000000000..bb5c0c895 --- /dev/null +++ b/apps/emqx_rule_engine/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXRuleEngine.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_rule_engine + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_slow_subs/mix.exs b/apps/emqx_slow_subs/mix.exs new file mode 100644 index 000000000..15f845837 --- /dev/null +++ b/apps/emqx_slow_subs/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXSlowSubs.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_slow_subs + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_statsd/mix.exs b/apps/emqx_statsd/mix.exs new file mode 100644 index 000000000..4969eff3e --- /dev/null +++ b/apps/emqx_statsd/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXStatsd.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_statsd + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() + end +end diff --git a/lib/emqx/mix/common.ex b/lib/emqx/mix/common.ex new file mode 100644 index 000000000..5a276cb0c --- /dev/null +++ b/lib/emqx/mix/common.ex @@ -0,0 +1,107 @@ +defmodule EMQX.Mix.Common do + @kernel_apps [:kernel, :stdlib, :sasl, :elixir] + + def project(app, overrides \\ []) when is_atom(app) and app != nil do + %{ + vsn: version, + description: description + } = + app + |> erl_app_props!() + |> Map.take([:vsn, :description]) + |> Map.new(fn {k, v} -> {k, to_string(v)} end) + + Keyword.merge( + [ + app: app, + version: version, + description: description, + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.13" + ], + overrides + ) + end + + def application(app, overrides \\ []) when app != nil do + {deps, overrides} = Keyword.pop(overrides, :deps, []) + # get only the dependency names + deps = Enum.map(deps, &elem(&1, 0)) + + app + |> erl_app_props!() + |> Map.take([:registered, :mod, :applications]) + |> Map.update!(:applications, + fn apps -> + deps ++ apps -- @kernel_apps + end) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + |> Keyword.new() + |> Keyword.merge(overrides) + end + + def erl_apps(app) when app != nil do + from_erl!(app, :applications) + end + + def erl_app_props!(app) do + path = Path.join("src", "#{app}.app.src") + {:ok, [{:application, ^app, props}]} = :file.consult(path) + Map.new(props) + end + + def from_erl!(app, key) when app != nil do + app + |> erl_app_props!() + |> Map.fetch!(key) + end + + def from_rebar_deps!() do + path = "rebar.config" + {:ok, props} = :file.consult(path) + + props + |> Keyword.fetch!(:deps) + |> Enum.map(&rebar_to_mix_dep/1) + end + + def rebar_to_mix_dep({name, {:git, url, {:tag, tag}}}), + do: {name, git: to_string(url), tag: to_string(tag)} + + def rebar_to_mix_dep({name, {:git, url, {:ref, ref}}}), + do: {name, git: to_string(url), ref: to_string(ref)} + + def rebar_to_mix_dep({name, {:git, url, {:branch, branch}}}), + do: {name, git: to_string(url), branch: to_string(branch)} + + def rebar_to_mix_dep({name, vsn}) when is_list(vsn), + do: {name, to_string(vsn)} + + def compile_protos(mix_filepath) do + app_path = Path.dirname(mix_filepath) + + config = [ + :use_packages, + :maps, + :strings_as_binaries, + rename: {:msg_name, :snake_case}, + rename: {:msg_fqname, :base_name}, + i: '.', + report_errors: false, + o: app_path |> Path.join("src") |> to_charlist(), + module_name_prefix: 'emqx_', + module_name_suffix: '_pb' + ] + + app_path + |> Path.join("priv/protos/*.proto") + |> Path.wildcard() + |> Enum.map(&to_charlist/1) + |> Enum.each(&:gpb_compile.file(&1, config)) + + :ok + end +end diff --git a/lib/mix/release.exs b/lib/mix/release.exs new file mode 100644 index 000000000..d6c032877 --- /dev/null +++ b/lib/mix/release.exs @@ -0,0 +1,878 @@ +defmodule Mix.Release do + @moduledoc """ + Defines the release structure and convenience for assembling releases. + """ + + @doc """ + The Mix.Release struct has the following read-only fields: + + * `:name` - the name of the release as an atom + * `:version` - the version of the release as a string or + `{:from_app, app_name}` + * `:path` - the path to the release root + * `:version_path` - the path to the release version inside the release + * `:applications` - a map of application with their definitions + * `:erts_source` - the ERTS source as a charlist (or nil) + * `:erts_version` - the ERTS version as a charlist + + The following fields may be modified as long as they keep their defined types: + + * `:boot_scripts` - a map of boot scripts with the boot script name + as key and a keyword list with **all** applications that are part of + it and their modes as value + * `:config_providers` - a list of `{config_provider, term}` tuples where the + first element is a module that implements the `Config.Provider` behaviour + and `term` is the value given to it on `c:Config.Provider.init/1` + * `:options` - a keyword list with all other user supplied release options + * `:overlays` - a list of extra files added to the release. If you have a custom + step adding extra files to a release, you can add these files to the `:overlays` + field so they are also considered on further commands, such as tar/zip. Each entry + in overlays is the relative path to the release root of each file + * `:steps` - a list of functions that receive the release and returns a release. + Must also contain the atom `:assemble` which is the internal assembling step. + May also contain the atom `:tar` to create a tarball of the release. + + """ + defstruct [ + :name, + :version, + :path, + :version_path, + :applications, + :boot_scripts, + :erts_source, + :erts_version, + :config_providers, + :options, + :overlays, + :steps + ] + + @type mode :: :permanent | :transient | :temporary | :load | :none + @type application :: atom() + @type t :: %__MODULE__{ + name: atom(), + version: String.t(), + path: String.t(), + version_path: String.t() | {:from_app, application()}, + applications: %{application() => keyword()}, + boot_scripts: %{atom() => [{application(), mode()}]}, + erts_version: charlist(), + erts_source: charlist() | nil, + config_providers: [{module, term}], + options: keyword(), + overlays: list(String.t()), + steps: [(t -> t) | :assemble, ...] + } + + @default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent] + @safe_modes [:permanent, :temporary, :transient] + @unsafe_modes [:load, :none] + @significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c + @copy_app_dirs ["priv"] + + @doc false + @spec from_config!(atom, keyword, keyword) :: t + def from_config!(name, config, overrides) do + {name, apps, opts} = find_release(name, config) + + unless Atom.to_string(name) =~ ~r/^[a-z][a-z0-9_]*$/ do + Mix.raise( + "Invalid release name. A release name must start with a lowercase ASCII letter, " <> + "followed by lowercase ASCII letters, numbers, or underscores, got: #{inspect(name)}" + ) + end + + opts = + [overwrite: false, quiet: false, strip_beams: true] + |> Keyword.merge(opts) + |> Keyword.merge(overrides) + + {include_erts, opts} = Keyword.pop(opts, :include_erts, true) + {erts_source, erts_lib_dir, erts_version} = erts_data(include_erts) + + deps_apps = Mix.Project.deps_apps() + loaded_apps = apps |> Keyword.keys() |> load_apps(deps_apps, %{}, erts_lib_dir, [], :root) + + # Make sure IEx is either an active part of the release or add it as none. + {loaded_apps, apps} = + if Map.has_key?(loaded_apps, :iex) do + {loaded_apps, apps} + else + {load_apps([:iex], deps_apps, loaded_apps, erts_lib_dir, [], :root), apps ++ [iex: :none]} + end + + start_boot = build_start_boot(loaded_apps, apps) + start_clean_boot = build_start_clean_boot(start_boot) + + {path, opts} = + Keyword.pop_lazy(opts, :path, fn -> + Path.join([Mix.Project.build_path(config), "rel", Atom.to_string(name)]) + end) + + path = Path.absname(path) + + {version, opts} = + Keyword.pop_lazy(opts, :version, fn -> + config[:version] || + Mix.raise( + "No :version found. Please make sure a :version is set in your project definition " <> + "or inside the release the configuration" + ) + end) + + version = + case version do + {:from_app, app} -> + Application.load(app) + version = Application.spec(app, :vsn) + + if !version do + Mix.raise( + "Could not find version for #{inspect(app)}, please make sure the application exists" + ) + end + + to_string(version) + + "" -> + Mix.raise("The release :version cannot be an empty string") + + _ -> + version + end + + {config_providers, opts} = Keyword.pop(opts, :config_providers, []) + {steps, opts} = Keyword.pop(opts, :steps, [:assemble]) + validate_steps!(steps) + + %Mix.Release{ + name: name, + version: version, + path: path, + version_path: Path.join([path, "releases", version]), + erts_source: erts_source, + erts_version: erts_version, + applications: loaded_apps, + boot_scripts: %{start: start_boot, start_clean: start_clean_boot}, + config_providers: config_providers, + options: opts, + overlays: [], + steps: steps + } + end + + defp find_release(name, config) do + {name, opts_fun_or_list} = lookup_release(name, config) || infer_release(config) + opts = if is_function(opts_fun_or_list, 0), do: opts_fun_or_list.(), else: opts_fun_or_list + {apps, opts} = Keyword.pop(opts, :applications, []) + + if apps == [] and Mix.Project.umbrella?(config) do + bad_umbrella!() + end + + app = Keyword.get(config, :app) + apps = Keyword.merge(@default_apps, apps) + + if is_nil(app) or Keyword.has_key?(apps, app) do + {name, apps, opts} + else + {name, apps ++ [{app, :permanent}], opts} + end + end + + defp lookup_release(nil, config) do + case Keyword.get(config, :releases, []) do + [] -> + nil + + [{name, opts}] -> + {name, opts} + + [_ | _] -> + case Keyword.get(config, :default_release) do + nil -> + Mix.raise( + "\"mix release\" was invoked without a name but there are multiple releases. " <> + "Please call \"mix release NAME\" or set :default_release in your project configuration" + ) + + name -> + lookup_release(name, config) + end + end + end + + defp lookup_release(name, config) do + if opts = config[:releases][name] do + {name, opts} + else + found = Keyword.get(config, :releases, []) + + Mix.raise( + "Unknown release #{inspect(name)}. " <> + "The available releases are: #{inspect(Keyword.keys(found))}" + ) + end + end + + defp infer_release(config) do + if Mix.Project.umbrella?(config) do + bad_umbrella!() + else + {Keyword.fetch!(config, :app), []} + end + end + + defp bad_umbrella! do + Mix.raise(""" + Umbrella projects require releases to be explicitly defined with \ + a non-empty applications key that chooses which umbrella children \ + should be part of the releases: + + releases: [ + foo: [ + applications: [child_app_foo: :permanent] + ], + bar: [ + applications: [child_app_bar: :permanent] + ] + ] + + Alternatively you can perform the release from the children applications + """) + end + + defp erts_data(erts_data) when is_function(erts_data) do + erts_data(erts_data.()) + end + + defp erts_data(false) do + {nil, :code.lib_dir(), :erlang.system_info(:version)} + end + + defp erts_data(true) do + version = :erlang.system_info(:version) + {:filename.join(:code.root_dir(), 'erts-#{version}'), :code.lib_dir(), version} + end + + defp erts_data(erts_source) when is_binary(erts_source) do + if File.exists?(erts_source) do + [_, erts_version] = erts_source |> Path.basename() |> String.split("-") + erts_lib_dir = erts_source |> Path.dirname() |> Path.join("lib") |> to_charlist() + {to_charlist(erts_source), erts_lib_dir, to_charlist(erts_version)} + else + Mix.raise("Could not find ERTS system at #{inspect(erts_source)}") + end + end + + defp load_apps(apps, deps_apps, seen, otp_root, optional, type) do + for app <- apps, reduce: seen do + seen -> + if reentrant_seen = reentrant(seen, app, type) do + reentrant_seen + else + load_app(app, deps_apps, seen, otp_root, optional, type) + end + end + end + + defp reentrant(seen, app, type) do + properties = seen[app] + + cond do + is_nil(properties) -> + nil + + type != :root and properties[:type] != type -> + if properties[:type] == :root do + put_in(seen[app][:type], type) + else + Mix.raise( + "#{inspect(app)} is listed both as a regular application and as an included application" + ) + end + + true -> + seen + end + end + + defp load_app(app, deps_apps, seen, otp_root, optional, type) do + cond do + path = app not in deps_apps && otp_path(otp_root, app) -> + do_load_app(app, path, deps_apps, seen, otp_root, true, type) + + path = code_path(app) -> + do_load_app(app, path, deps_apps, seen, otp_root, false, type) + + app in optional -> + seen + + true -> + Mix.raise("Could not find application #{inspect(app)}") + end + end + + defp otp_path(otp_root, app) do + path = Path.join(otp_root, "#{app}-*") + + case Path.wildcard(path) do + [] -> nil + paths -> paths |> Enum.sort() |> List.last() |> to_charlist() + end + end + + defp code_path(app) do + case :code.lib_dir(app) do + {:error, :bad_name} -> nil + path -> path + end + end + + defp do_load_app(app, path, deps_apps, seen, otp_root, otp_app?, type) do + case :file.consult(Path.join(path, "ebin/#{app}.app")) do + {:ok, terms} -> + [{:application, ^app, properties}] = terms + value = [path: path, otp_app?: otp_app?, type: type] ++ properties + seen = Map.put(seen, app, value) + applications = Keyword.get(properties, :applications, []) + optional = Keyword.get(properties, :optional_applications, []) + seen = load_apps(applications, deps_apps, seen, otp_root, optional, :depended) + included_applications = Keyword.get(properties, :included_applications, []) + load_apps(included_applications, deps_apps, seen, otp_root, [], :included) + + {:error, reason} -> + Mix.raise("Could not load #{app}.app. Reason: #{inspect(reason)}") + end + end + + defp build_start_boot(all_apps, specified_apps) do + specified_apps ++ + Enum.sort( + for( + {app, props} <- all_apps, + not List.keymember?(specified_apps, app, 0), + do: {app, default_mode(props)} + ) + ) + end + + defp default_mode(props) do + if props[:type] == :included, do: :load, else: :permanent + end + + defp build_start_clean_boot(boot) do + for({app, _mode} <- boot, do: {app, :none}) + |> Keyword.put(:stdlib, :permanent) + |> Keyword.put(:kernel, :permanent) + end + + defp validate_steps!(steps) do + valid_atoms = [:assemble, :tar] + + if not is_list(steps) or Enum.any?(steps, &(&1 not in valid_atoms and not is_function(&1, 1))) do + Mix.raise(""" + The :steps option must be a list of: + + * anonymous function that receives one argument + * the atom :assemble or :tar + + Got: #{inspect(steps)} + """) + end + + if Enum.count(steps, &(&1 == :assemble)) != 1 do + Mix.raise("The :steps option must contain the atom :assemble once, got: #{inspect(steps)}") + end + + if :assemble in Enum.drop_while(steps, &(&1 != :tar)) do + Mix.raise("The :tar step must come after :assemble") + end + + if Enum.count(steps, &(&1 == :tar)) > 1 do + Mix.raise("The :steps option can only contain the atom :tar once") + end + + :ok + end + + @doc """ + Makes the `sys.config` structure. + + If there are config providers, then a value is injected into + the `:elixir` application configuration in `sys_config` to be + read during boot and trigger the providers. + + It uses the following release options to customize its behaviour: + + * `:reboot_system_after_config` + * `:start_distribution_during_config` + * `:prune_runtime_sys_config_after_boot` + + In case there are no config providers, it doesn't change `sys_config`. + """ + @spec make_sys_config(t, keyword(), Config.Provider.config_path()) :: + :ok | {:error, String.t()} + def make_sys_config(release, sys_config, config_provider_path) do + {sys_config, runtime_config?} = + merge_provider_config(release, sys_config, config_provider_path) + + path = Path.join(release.version_path, "sys.config") + + args = [runtime_config?, sys_config] + format = "%% coding: utf-8~n%% RUNTIME_CONFIG=~s~n~tw.~n" + File.mkdir_p!(Path.dirname(path)) + File.write!(path, IO.chardata_to_string(:io_lib.format(format, args))) + + case :file.consult(path) do + {:ok, _} -> + :ok + + {:error, reason} -> + invalid = + for {app, kv} <- sys_config, + {key, value} <- kv, + not valid_config?(value), + do: """ + + Application: #{inspect(app)} + Key: #{inspect(key)} + Value: #{inspect(value)} + """ + + message = + case invalid do + [] -> + "Could not read configuration file. Reason: #{inspect(reason)}" + + _ -> + "Could not read configuration file. It has invalid configuration terms " <> + "such as functions, references, and pids. Please make sure your configuration " <> + "is made of numbers, atoms, strings, maps, tuples and lists. The following entries " <> + "are wrong:\n#{Enum.join(invalid)}" + end + + {:error, message} + end + end + + defp valid_config?(m) when is_map(m), + do: Enum.all?(Map.delete(m, :__struct__), &valid_config?/1) + + defp valid_config?(l) when is_list(l), do: Enum.all?(l, &valid_config?/1) + defp valid_config?(t) when is_tuple(t), do: Enum.all?(Tuple.to_list(t), &valid_config?/1) + defp valid_config?(o), do: is_number(o) or is_atom(o) or is_binary(o) + + defp merge_provider_config(%{config_providers: []}, sys_config, _), do: {sys_config, false} + + defp merge_provider_config(release, sys_config, config_path) do + {reboot?, extra_config, initial_config} = start_distribution(release) + + prune_runtime_sys_config_after_boot = + Keyword.get(release.options, :prune_runtime_sys_config_after_boot, false) + + opts = [ + extra_config: initial_config, + prune_runtime_sys_config_after_boot: prune_runtime_sys_config_after_boot, + reboot_system_after_config: reboot?, + validate_compile_env: validate_compile_env(release) + ] + + init_config = Config.Provider.init(release.config_providers, config_path, opts) + {Config.Reader.merge(sys_config, init_config ++ extra_config), reboot?} + end + + defp validate_compile_env(release) do + with true <- Keyword.get(release.options, :validate_compile_env, true), + [_ | _] = compile_env <- compile_env(release) do + compile_env + else + _ -> false + end + end + + defp compile_env(release) do + for {_, properties} <- release.applications, + triplet <- Keyword.get(properties, :compile_env, []), + do: triplet + end + + defp start_distribution(%{options: opts}) do + reboot? = Keyword.get(opts, :reboot_system_after_config, false) + early_distribution? = Keyword.get(opts, :start_distribution_during_config, false) + + if not reboot? or early_distribution? do + {reboot?, [], []} + else + {true, [kernel: [start_distribution: false]], [kernel: [start_distribution: true]]} + end + end + + @doc """ + Copies the cookie to the given path. + + If a cookie option was given, we compare it with + the contents of the file (if any), and ask the user + if they want to override. + + If there is no option, we generate a random one + the first time. + """ + @spec make_cookie(t, Path.t()) :: :ok + def make_cookie(release, path) do + cond do + cookie = release.options[:cookie] -> + Mix.Generator.create_file(path, cookie, quiet: true) + :ok + + File.exists?(path) -> + :ok + + true -> + File.write!(path, random_cookie()) + :ok + end + end + + defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32)) + + @doc """ + Makes the start_erl.data file with the + ERTS version and release versions. + """ + @spec make_start_erl(t, Path.t()) :: :ok + def make_start_erl(release, path) do + File.write!(path, "#{release.erts_version} #{release.version}") + :ok + end + + @doc """ + Makes boot scripts. + + It receives a path to the boot file, without extension, such as + `releases/0.1.0/start` and this command will write `start.rel`, + `start.boot`, and `start.script` to the given path, returning + `{:ok, rel_path}` or `{:error, message}`. + + The boot script uses the RELEASE_LIB environment variable, which must + be accordingly set with `--boot-var` and point to the release lib dir. + """ + @spec make_boot_script(t, Path.t(), [{application(), mode()}], [String.t()]) :: + :ok | {:error, String.t()} + def make_boot_script(release, path, modes, prepend_paths \\ []) do + with {:ok, rel_spec} <- build_release_spec(release, modes) do + File.write!(path <> ".rel", consultable(rel_spec)) + + sys_path = String.to_charlist(path) + + sys_options = [ + :silent, + :no_dot_erlang, + :no_warn_sasl, + variables: build_variables(release), + path: build_paths(release) + ] + + case :systools.make_script(sys_path, sys_options) do + {:ok, _module, _warnings} -> + script_path = sys_path ++ '.script' + {:ok, [{:script, rel_info, instructions}]} = :file.consult(script_path) + + instructions = + instructions + |> post_stdlib_applies(release) + |> prepend_paths_to_script(prepend_paths) + + script = {:script, rel_info, instructions} + File.write!(script_path, consultable(script)) + :ok = :systools.script2boot(sys_path) + + {:error, module, info} -> + message = module.format_error(info) |> to_string() |> String.trim() + {:error, message} + end + end + end + + defp build_variables(release) do + for {_, properties} <- release.applications, + not Keyword.fetch!(properties, :otp_app?), + uniq: true, + do: {'RELEASE_LIB', properties |> Keyword.fetch!(:path) |> :filename.dirname()} + end + + defp build_paths(release) do + for {_, properties} <- release.applications, + Keyword.fetch!(properties, :otp_app?), + do: properties |> Keyword.fetch!(:path) |> Path.join("ebin") |> to_charlist() + end + + defp build_release_spec(release, modes) do + %{ + name: name, + version: version, + erts_version: erts_version, + applications: apps, + options: options + } = release + + skip_mode_validation_for = + options + |> Keyword.get(:skip_mode_validation_for, []) + |> MapSet.new() + + rel_apps = + for {app, mode} <- modes do + properties = Map.get(apps, app) || throw({:error, "Unknown application #{inspect(app)}"}) + children = Keyword.get(properties, :applications, []) + app in skip_mode_validation_for || validate_mode!(app, mode, modes, children) + build_app_for_release(app, mode, properties) + end + + {:ok, {:release, {to_charlist(name), to_charlist(version)}, {:erts, erts_version}, rel_apps}} + catch + {:error, message} -> {:error, message} + end + + defp validate_mode!(app, mode, modes, children) do + safe_mode? = mode in @safe_modes + + if not safe_mode? and mode not in @unsafe_modes do + throw( + {:error, + "Unknown mode #{inspect(mode)} for #{inspect(app)}. " <> + "Valid modes are: #{inspect(@safe_modes ++ @unsafe_modes)}"} + ) + end + + for child <- children do + child_mode = Keyword.get(modes, child) + + cond do + is_nil(child_mode) -> + throw( + {:error, + "Application #{inspect(app)} is listed in the release boot, " <> + "but it depends on #{inspect(child)}, which isn't"} + ) + + safe_mode? and child_mode in @unsafe_modes -> + throw( + {:error, + """ + Application #{inspect(app)} has mode #{inspect(mode)} but it depends on \ + #{inspect(child)} which is set to #{inspect(child_mode)}. If you really want \ + to set such mode for #{inspect(child)} make sure that all applications that depend \ + on it are also set to :load or :none, otherwise your release will fail to boot + """} + ) + + true -> + :ok + end + end + end + + defp build_app_for_release(app, mode, properties) do + vsn = Keyword.fetch!(properties, :vsn) + + case Keyword.get(properties, :included_applications, []) do + [] -> {app, vsn, mode} + included_apps -> {app, vsn, mode, included_apps} + end + end + + defp post_stdlib_applies(instructions, release) do + {pre, [stdlib | post]} = + Enum.split_while( + instructions, + &(not match?({:apply, {:application, :start_boot, [:stdlib, _]}}, &1)) + ) + + pre ++ [stdlib] ++ config_provider_apply(release) ++ post + end + + defp config_provider_apply(%{config_providers: []}), + do: [] + + defp config_provider_apply(_), + do: [{:apply, {Config.Provider, :boot, []}}] + + defp prepend_paths_to_script(instructions, []), do: instructions + + defp prepend_paths_to_script(instructions, prepend_paths) do + prepend_paths = Enum.map(prepend_paths, &String.to_charlist/1) + + Enum.map(instructions, fn + {:path, paths} -> + if Enum.any?(paths, &List.starts_with?(&1, '$RELEASE_LIB')) do + {:path, prepend_paths ++ paths} + else + {:path, paths} + end + + other -> + other + end) + end + + defp consultable(term) do + IO.chardata_to_string(:io_lib.format("%% coding: utf-8~n~tp.~n", [term])) + end + + @doc """ + Finds a template path for the release. + """ + def rel_templates_path(release, path) do + Path.join(release.options[:rel_templates_path] || "rel", path) + end + + @doc """ + Copies ERTS if the release is configured to do so. + + Returns true if the release was copied, false otherwise. + """ + @spec copy_erts(t) :: boolean() + def copy_erts(%{erts_source: nil}) do + false + end + + def copy_erts(release) do + destination = Path.join(release.path, "erts-#{release.erts_version}/bin") + File.mkdir_p!(destination) + + release.erts_source + |> Path.join("bin") + |> File.cp_r!(destination, fn _, _ -> false end) + + _ = File.rm(Path.join(destination, "erl")) + _ = File.rm(Path.join(destination, "erl.ini")) + + destination + |> Path.join("erl") + |> File.write!(~S""" + #!/bin/sh + SELF=$(readlink "$0" || true) + if [ -z "$SELF" ]; then SELF="$0"; fi + BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" + ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" + EMU=beam + PROGNAME=$(echo "$0" | sed 's/.*\///') + export EMU + export ROOTDIR + export BINDIR + export PROGNAME + exec "$BINDIR/erlexec" ${1+"$@"} + """) + + File.chmod!(Path.join(destination, "erl"), 0o755) + true + end + + @doc """ + Copies the given application specification into the release. + + It assumes the application exists in the release. + """ + @spec copy_app(t, application) :: boolean() + def copy_app(release, app) do + properties = Map.fetch!(release.applications, app) + vsn = Keyword.fetch!(properties, :vsn) + + source_app = Keyword.fetch!(properties, :path) + target_app = Path.join([release.path, "lib", "#{app}-#{vsn}"]) + + if is_nil(release.erts_source) and Keyword.fetch!(properties, :otp_app?) do + false + else + File.rm_rf!(target_app) + File.mkdir_p!(target_app) + + copy_ebin(release, Path.join(source_app, "ebin"), Path.join(target_app, "ebin")) + + for dir <- @copy_app_dirs do + source_dir = Path.join(source_app, dir) + target_dir = Path.join(target_app, dir) + + source_dir = + case File.read_link(source_dir) do + {:ok, link_target} -> Path.expand(link_target, source_app) + _ -> source_dir + end + + File.exists?(source_dir) && File.cp_r!(source_dir, target_dir) + end + + true + end + end + + @doc """ + Copies the ebin directory at `source` to `target` + respecting release options such a `:strip_beams`. + """ + @spec copy_ebin(t, Path.t(), Path.t()) :: boolean() + def copy_ebin(release, source, target) do + with {:ok, [_ | _] = files} <- File.ls(source) do + File.mkdir_p!(target) + + strip_options = + release.options + |> Keyword.get(:strip_beams, true) + |> parse_strip_beams_options() + + for file <- files do + source_file = Path.join(source, file) + target_file = Path.join(target, file) + + with true <- is_list(strip_options) and String.ends_with?(file, ".beam"), + {:ok, binary} <- strip_beam(File.read!(source_file), strip_options) do + File.write!(target_file, binary) + else + _ -> + # Use File.cp!/3 to preserve file mode for any executables stored + # in the ebin directory. + File.cp!(source_file, target_file) + end + end + + true + else + _ -> false + end + end + + @doc """ + Strips a beam file for a release. + + This keeps only significant chunks necessary for the VM operation, + discarding documentation, debug info, compile information and others. + + The exact chunks that are kept are not documented and may change in + future versions. + """ + @spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()} + def strip_beam(binary, options \\ []) when is_list(options) do + chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1) + all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep) + + case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do + {:ok, {_, chunks}} -> + chunks = for {name, chunk} <- chunks, is_binary(chunk), do: {name, chunk} + {:ok, binary} = :beam_lib.build_module(chunks) + {:ok, :zlib.gzip(binary)} + + {:error, _, _} = error -> + error + end + end + + defp parse_strip_beams_options(options) do + case options do + options when is_list(options) -> options + true -> [] + false -> nil + end + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 000000000..29278cf40 --- /dev/null +++ b/mix.exs @@ -0,0 +1,206 @@ +defmodule EMQXUmbrella.MixProject do + use Mix.Project + + # Temporary hack while 1.13.2 is not released + System.version() + |> Version.parse!() + |> Version.compare(Version.parse!("1.13.2")) + |> Kernel.==(:lt) + |> if(do: Code.require_file("lib/mix/release.exs")) + + def project do + [ + apps_path: "apps", + version: pkg_vsn(), + deps: deps(), + releases: releases() + ] + end + + defp deps do + # we need several overrides here because dependencies specify + # other exact versions, and not ranges. + [ + {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, + {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, + {:gproc, "0.8.0", override: true}, + {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, + {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true}, + {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.11.1", override: true}, + {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.2.7"}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, + {:replayq, "0.3.3", override: true}, + {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, + {:observer_cli, "1.7.1"}, + {:system_monitor, github: "klarna-incubator/system_monitor", tag: "2.2.0"}, + # in conflict by emqtt and hocon + {:getopt, "1.0.2", override: true}, + {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.22.0"}, + {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1"}, + {:esasl, github: "emqx/esasl", tag: "0.2.0"}, + {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, + # in conflict by ehttpc and emqtt + {:gun, github: "emqx/gun", tag: "1.3.6", override: true}, + # in conflict by emqx_connectior and system_monitor + {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, + # in conflict by mongodb and eredis_cluster + {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, + # in conflict by gun and emqtt + {:cowlib, "2.8.0", override: true}, + # in conflict by cowboy_swagger and cowboy + {:ranch, "1.8.0", override: true}, + # in conflict by emqx and observer_cli + {:recon, github: "ferd/recon", tag: "2.5.1", override: true} + ] ++ bcrypt_dep() ++ quicer_dep() + end + + defp releases do + [ + emqx: [ + applications: [ + logger: :permanent, + esasl: :load, + crypto: :permanent, + public_key: :permanent, + asn1: :permanent, + syntax_tools: :permanent, + ssl: :permanent, + os_mon: :permanent, + inets: :permanent, + compiler: :permanent, + runtime_tools: :permanent, + hocon: :load, + emqx: :load, + emqx_conf: :load, + emqx_machine: :permanent, + mria: :load, + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_statsd: :permanent, + emqx_retainer: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent + ], + skip_mode_validation_for: [ + :emqx_gateway, + :emqx_dashboard, + :emqx_resource, + :emqx_connector, + :emqx_exhook, + :emqx_bridge, + :emqx_modules, + :emqx_management, + :emqx_statsd, + :emqx_retainer, + :emqx_prometheus, + :emqx_plugins + ], + steps: [:assemble, ©_files/1] + ] + ] + end + + def copy_files(release) do + etc = Path.join(release.path, "etc") + + # FIXME: Remove?? + File.mkdir_p!(etc) + File.cp!("apps/emqx_authz/etc/acl.conf", Path.join(etc, "acl.conf")) + + release + end + + def bcrypt_dep() do + if enable_bcrypt?(), + do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0"}], + else: [] + end + + def quicer_dep() do + if enable_quicer?(), + # in conflict with emqx and emqtt + do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}], + else: [] + end + + def enable_bcrypt?() do + not win32?() + end + + def enable_quicer?() do + not Enum.any?([ + build_without_quic?(), + win32?(), + centos6?() + ]) + end + + def project_path() do + Path.expand("..", __ENV__.file) + end + + def pkg_vsn() do + project_path() + |> Path.join("pkg-vsn.sh") + |> System.cmd([]) + |> elem(0) + |> String.trim() + |> String.split("-") + |> Enum.reverse() + |> tl() + |> Enum.reverse() + |> fix_vsn() + |> Enum.join("-") + end + + # FIXME: remove hack + defp fix_vsn([vsn | extras]) do + if Version.parse(vsn) == :error do + [vsn <> ".0" | extras] + else + [vsn | extras] + end + end + + defp win32?(), + do: match?({:win_32, _}, :os.type()) + + defp centos6?() do + case File.read("/etc/centos-release") do + {:ok, "CentOS release 6" <> _} -> + true + + _ -> + false + end + end + + defp build_without_quic?() do + opt = System.get_env("BUILD_WITHOUT_QUIC", "false") + + String.downcase(opt) != "false" + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 000000000..54812891f --- /dev/null +++ b/mix.lock @@ -0,0 +1,59 @@ +%{ + "bcrypt": {:git, "https://github.com/emqx/erlang-bcrypt.git", "dc2ba66acf2332c111362d01137746eefecc5e90", [tag: "0.6.0"]}, + "bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]}, + "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]}, + "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]}, + "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, + "earmark": {:hex, :earmark, "1.4.19", "3854a17305c880cc46305af15fb1630568d23a709aba21aaa996ced082fc29d7", [:mix], [{:earmark_parser, ">= 1.4.18", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "d5a8c9f9e37159a8fdd3ea8437fb4e229eaf56d5129b9a011dc4780a4872079d"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"}, + "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]}, + "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, + "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, + "ekka": {:git, "https://github.com/emqx/ekka.git", "005fd6bb94199dc2ecb4ba03284f253b408e02d9", [tag: "0.11.1"]}, + "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, + "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, + "epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]}, + "eredis": {:hex, :eredis, "1.2.0", "0b8e9cfc2c00fa1374cd107ea63b49be08d933df2cf175e6a89b73dd9c380de4", [:rebar3], [], "hexpm", "d9b5abef2c2c8aba8f32aa018203e0b3dc8b1157773b254ab1d4c2002317f1e1"}, + "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster.git", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, + "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, + "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]}, + "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]}, + "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, + "gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]}, + "getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"}, + "gpb": {:hex, :gpb, "4.19.2", "e841a1c5d5b82b6fbaddd2b6968c66ad48b34adc3b0d974e55cca653bd28ccc5", [:make, :rebar3], [], "hexpm", "3f4cb7f263e94675a598501a6dc96ec17f93ecb28c29bea3bea3a71c53a3317a"}, + "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"}, + "grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]}, + "gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]}, + "hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]}, + "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, + "jiffy": {:git, "https://github.com/emqx/jiffy.git", "baa1f4e750ae3c5c9e54f9c2e52280b7fc24a8d9", [tag: "1.0.5"]}, + "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, + "jsx": {:hex, :jsx, "2.9.0", "d2f6e5f069c00266cad52fb15d87c428579ea4d7d73a33669e12679e203329dd", [:mix, :rebar3], [], "hexpm", "8ee1db1cabafdd578a2776a6aaae87c2a8ce54b47b59e9ec7dab5d7eb71cd8dc"}, + "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, + "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, + "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, + "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, + "mongodb": {:git, "https://github.com/emqx/mongodb-erlang.git", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, + "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, + "mysql": {:git, "https://github.com/emqx/mysql-otp.git", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, + "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, + "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]}, + "poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]}, + "prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]}, + "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]}, + "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"}, + "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]}, + "rulesql": {:git, "https://github.com/emqx/rulesql.git", "fec11b1a3cbf98480d19c06d3aca10442e1e02a9", [tag: "0.1.4"]}, + "sext": {:hex, :sext, "1.8.0", "90a95b889f5c781b70bbcf44278b763148e313c376b60d87ce664cb1c1dd29b5", [:rebar3], [], "hexpm", "bc6016cb8690baf677eacacfe6e7cadfec8dc7e286cbbed762f6cd55b0678e73"}, + "snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]}, + "ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]}, + "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"}, + "system_monitor": {:git, "https://github.com/klarna-incubator/system_monitor.git", "a106f72e8e2251d90e1d95944c139946dac1a07f", [tag: "2.2.0"]}, + "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"}, + "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]}, +} diff --git a/mix_release.sh b/mix_release.sh new file mode 100755 index 000000000..321f0fbe1 --- /dev/null +++ b/mix_release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +mix release --overwrite + +## FIXME: temporary hacks to get the needed configurations for the +## release to properly start. + +## Assumes that `make emqx` has been run before the mix build to +## generate the correct configs. + +mkdir -p _build/dev/rel/emqx/data/configs/ +LATEST_APP_CONFIG=$(ls -rt _build/emqx/rel/emqx/data/configs/app*.config | tail -n 1) + +# FIXME! +cp ${LATEST_APP_CONFIG} _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#_build/emqx/rel/emqx/etc/emqx.conf#_build/dev/rel/emqx/etc/emqx.conf#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#logger_level,warning#logger_level,debug#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#level => warning#level => debug#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config + + +# cp _build/emqx/rel/emqx/releases/emqx_vars _build/dev/rel/emqx/releases/ +cp _build/emqx/rel/emqx/etc/emqx.conf _build/dev/rel/emqx/etc/ + +echo "telemetry { enable = false }" >> _build/dev/rel/emqx/etc/emqx.conf diff --git a/rel/env.bat.eex b/rel/env.bat.eex new file mode 100644 index 000000000..60beb808a --- /dev/null +++ b/rel/env.bat.eex @@ -0,0 +1,5 @@ +@echo off +rem Set the release to work across nodes. +rem RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +rem set RELEASE_DISTRIBUTION=name +rem set RELEASE_NODE=<%= @release.name %> diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 000000000..70cfce36a --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,17 @@ +#!/bin/sh + +# Sets and enables heart (recommended only in daemon mode) +# case $RELEASE_COMMAND in +# daemon*) +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# ;; +# *) +# ;; +# esac + +# Set the release to work across nodes. +# RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +# export RELEASE_DISTRIBUTION=name +# export RELEASE_NODE=<%= @release.name %> diff --git a/rel/remote.vm.args.eex b/rel/remote.vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/remote.vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/rel/vm.args.eex b/rel/vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 From 2ff46a6dbbe7546f830fd6132411b5088f911dd9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Dec 2021 17:28:39 -0300 Subject: [PATCH 02/31] chore(mix): create a few more required files during release Since Mix does not support hot upgrades out of the box, it does not create a `RELEASES` file by default. Here, we introduce functionality similar to what `relx` does in order for that file to be generated. https://github.com/erlware/relx/blob/fdc8d7237ef757162cbf198a56c761065e54b49e/src/rlx_assemble.erl#L928-L948 The `RELEASES` file, in its turn, is required for `nodetool` to work properly, and `nodetool` is required for us to generate several required config files using `hocon`. Since the `start{_clean}.boot` file used by the files generated by Elixir must be explicitly defined in the `RELEASE_LIB` `boot_var`, we apply a small patch in the existing `nodetool` in order to inject that required parameter when calling the escript. --- mix.exs | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 29278cf40..7525df5a1 100644 --- a/mix.exs +++ b/mix.exs @@ -118,17 +118,89 @@ defmodule EMQXUmbrella.MixProject do :emqx_prometheus, :emqx_plugins ], - steps: [:assemble, ©_files/1] + steps: [ + :assemble, + &create_RELEASES/1, + ©_files/1, + ©_nodetool/1, + ] ] ] end def copy_files(release) do + overwrite? = Keyword.get(release.options, :overwrite, false) + + bin = Path.join(release.path, "bin") etc = Path.join(release.path, "etc") - # FIXME: Remove?? - File.mkdir_p!(etc) - File.cp!("apps/emqx_authz/etc/acl.conf", Path.join(etc, "acl.conf")) + Mix.Generator.create_directory(bin) + Mix.Generator.create_directory(etc) + + Mix.Generator.copy_file( + "apps/emqx_authz/etc/acl.conf", + Path.join(etc, "acl.conf"), + force: overwrite? + ) + + # FIXME: check if cloud/edge!! + Mix.Generator.copy_file( + "apps/emqx/etc/emqx_cloud/vm.args", + Path.join(etc, "vm.args"), + force: overwrite? + ) + Mix.Generator.copy_file( + "apps/emqx/etc/emqx_cloud/vm.args", + Path.join(release.version_path, "vm.args"), + force: overwrite? + ) + + + release + end + + # needed by nodetool and by release_handler + def create_RELEASES(release) do + apps = Enum.map(release.applications, fn {app_name, app_props} -> + app_vsn = Keyword.fetch!(app_props, :vsn) + app_path = + "./lib" + |> Path.join("#{app_name}-#{app_vsn}") + |> to_charlist() + {app_name, app_vsn, app_path} + end) + release_entry = + [{ + :release, + to_charlist(release.name), + to_charlist(release.version), + release.erts_version, + apps, + :permanent + }] + + release.path + |> Path.join("releases") + |> Path.join("RELEASES") + |> File.open!([:write, :utf8], fn handle -> + IO.puts(handle, "%% coding: utf-8") + :io.format(handle, '~tp.~n', [release_entry]) + end) + + release + end + + def copy_nodetool(release) do + [shebang, rest] = + "bin/nodetool" + |> File.read!() + |> String.split("\n", parts: 2) + + path = Path.join([release.path, "bin", "nodetool"]) + # the elixir version of escript + start.boot required the boot_var + # RELEASE_LIB to be defined. + boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib" + File.write!(path, [shebang, "\n", boot_var, "\n", rest]) release end From 34d6dbca61e91d0a048c0e637619174cb7c8738b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Dec 2021 20:38:11 -0300 Subject: [PATCH 03/31] chore(mix): simplify build by using rebar3 for umbrella apps By treating the apps in the umbrella as dependencies to be managed and built by rebar3, we can simplify the maintenance of the release, at the cost of increased build times: using Mix as before, it could track changed files better than using rebar. But the complexity and possibility of discrepancies make it using rebar much more compelling. --- .formatter.exs | 6 + .tool-versions | 2 +- apps/emqx/mix.exs | 39 ------- apps/emqx_authn/mix.exs | 21 ---- apps/emqx_authn/rebar.config | 3 +- apps/emqx_authz/mix.exs | 23 ---- apps/emqx_authz/rebar.config | 2 +- apps/emqx_auto_subscribe/mix.exs | 21 ---- apps/emqx_auto_subscribe/rebar.config | 3 +- apps/emqx_bridge/mix.exs | 21 ---- apps/emqx_bridge/rebar.config | 3 +- apps/emqx_conf/mix.exs | 23 ---- apps/emqx_conf/rebar.config | 3 +- apps/emqx_conf/src/emqx_conf.app.src | 1 - apps/emqx_connector/mix.exs | 28 ----- apps/emqx_connector/rebar.config | 1 + .../emqx_connector/src/emqx_connector.app.src | 1 + apps/emqx_dashboard/mix.exs | 23 ---- apps/emqx_dashboard/rebar.config | 4 +- apps/emqx_exhook/mix.exs | 29 ----- apps/emqx_exhook/rebar.config | 3 +- apps/emqx_gateway/mix.exs | 32 ------ apps/emqx_gateway/rebar.config | 1 + apps/emqx_machine/mix.exs | 21 ---- apps/emqx_machine/rebar.config | 2 + apps/emqx_management/mix.exs | 21 ---- apps/emqx_management/rebar.config | 3 +- apps/emqx_modules/mix.exs | 21 ---- apps/emqx_modules/rebar.config | 3 +- apps/emqx_plugin_libs/mix.exs | 21 ---- apps/emqx_plugins/mix.exs | 21 ---- apps/emqx_plugins/rebar.config | 2 + apps/emqx_prometheus/mix.exs | 21 ---- apps/emqx_prometheus/rebar.config | 4 +- apps/emqx_psk/mix.exs | 21 ---- apps/emqx_psk/rebar.config | 3 +- apps/emqx_resource/mix.exs | 21 ---- apps/emqx_retainer/mix.exs | 21 ---- apps/emqx_retainer/rebar.config | 3 +- apps/emqx_rule_engine/mix.exs | 21 ---- apps/emqx_rule_engine/rebar.config | 3 +- apps/emqx_slow_subs/mix.exs | 21 ---- apps/emqx_slow_subs/rebar.config | 2 + apps/emqx_statsd/mix.exs | 21 ---- lib/emqx/mix/common.ex | 107 ------------------ mix.exs | 94 ++++++++++----- mix.lock | 11 +- mix_release.sh | 1 + rebar.config.erl | 1 + 49 files changed, 116 insertions(+), 668 deletions(-) create mode 100644 .formatter.exs delete mode 100644 apps/emqx/mix.exs delete mode 100644 apps/emqx_authn/mix.exs delete mode 100644 apps/emqx_authz/mix.exs delete mode 100644 apps/emqx_auto_subscribe/mix.exs delete mode 100644 apps/emqx_bridge/mix.exs delete mode 100644 apps/emqx_conf/mix.exs delete mode 100644 apps/emqx_connector/mix.exs delete mode 100644 apps/emqx_dashboard/mix.exs delete mode 100644 apps/emqx_exhook/mix.exs delete mode 100644 apps/emqx_gateway/mix.exs delete mode 100644 apps/emqx_machine/mix.exs create mode 100644 apps/emqx_machine/rebar.config delete mode 100644 apps/emqx_management/mix.exs delete mode 100644 apps/emqx_modules/mix.exs delete mode 100644 apps/emqx_plugin_libs/mix.exs delete mode 100644 apps/emqx_plugins/mix.exs create mode 100644 apps/emqx_plugins/rebar.config delete mode 100644 apps/emqx_prometheus/mix.exs delete mode 100644 apps/emqx_psk/mix.exs delete mode 100644 apps/emqx_resource/mix.exs delete mode 100644 apps/emqx_retainer/mix.exs delete mode 100644 apps/emqx_rule_engine/mix.exs delete mode 100644 apps/emqx_slow_subs/mix.exs create mode 100644 apps/emqx_slow_subs/rebar.config delete mode 100644 apps/emqx_statsd/mix.exs delete mode 100644 lib/emqx/mix/common.ex diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 000000000..5cad8a0ad --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,6 @@ +[ + inputs: [ + "mix.exs", + "config/*.exs" + ] +] diff --git a/.tool-versions b/.tool-versions index 1c687075b..cdb7ea9a8 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.1.5-3 +erlang 24.1.2 elixir 1.13.1-otp-24 diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs deleted file mode 100644 index 847002cc2..000000000 --- a/apps/emqx/mix.exs +++ /dev/null @@ -1,39 +0,0 @@ -defmodule EMQX.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - [ - mod: EMQX.Mix.Common.from_erl!(:emqx, :mod), - applications: EMQX.Mix.Common.from_erl!(:emqx, :applications) - ] - end - - # since emqx app is more complicated than others, we manually set - # its dependencies here - defp deps() do - [ - {:lc, git: "https://github.com/qzhuyan/lc.git", tag: "0.1.2"}, - {:gproc, git: "https://github.com/uwiger/gproc", tag: "0.8.0"}, - {:typerefl, git: "https://github.com/k32/typerefl", tag: "0.8.5"}, - {:jiffy, git: "https://github.com/emqx/jiffy", tag: "1.0.5"}, - {:cowboy, git: "https://github.com/emqx/cowboy", tag: "2.9.0"}, - {:esockd, git: "https://github.com/emqx/esockd", tag: "5.9.0"}, - {:ekka, git: "https://github.com/emqx/ekka", tag: "0.11.1"}, - {:gen_rpc, git: "https://github.com/emqx/gen_rpc", tag: "2.5.1"}, - {:hocon, git: "https://github.com/emqx/hocon.git", tag: "0.22.0"}, - {:pbkdf2, git: "https://github.com/emqx/erlang-pbkdf2.git", tag: "2.0.4"}, - {:recon, git: "https://github.com/ferd/recon", tag: "2.5.1"}, - {:snabbkaffe, git: "https://github.com/kafka4beam/snabbkaffe.git", tag: "0.16.0"} - ] - end -end diff --git a/apps/emqx_authn/mix.exs b/apps/emqx_authn/mix.exs deleted file mode 100644 index 07af06495..000000000 --- a/apps/emqx_authn/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXAuthn.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_authn - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 73696b033..3def1d00e 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_authz/mix.exs b/apps/emqx_authz/mix.exs deleted file mode 100644 index 06a5792b4..000000000 --- a/apps/emqx_authz/mix.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule EMQXAuthz.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_authz - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app, - extra_applications: [:crypto] - ) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index ba38cc642..b1479d434 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -1,5 +1,5 @@ {erl_opts, [debug_info, nowarn_unused_import]}. -{deps, []}. +{deps, [{emqx, {path, "../emqx"}}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_auto_subscribe/mix.exs b/apps/emqx_auto_subscribe/mix.exs deleted file mode 100644 index d60e8f03e..000000000 --- a/apps/emqx_auto_subscribe/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXAutoSubscribe.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_auto_subscribe - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 88793f7ba..91cdd39d9 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_bridge/mix.exs b/apps/emqx_bridge/mix.exs deleted file mode 100644 index 629f1295c..000000000 --- a/apps/emqx_bridge/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXBridge.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_bridge - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 3fd6b41e0..d24d23f8c 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_conf/mix.exs b/apps/emqx_conf/mix.exs deleted file mode 100644 index 1ad98ee70..000000000 --- a/apps/emqx_conf/mix.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule EMQXConf.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_conf - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [ - {:emqx, in_umbrella: true} - ] - end -end diff --git a/apps/emqx_conf/rebar.config b/apps/emqx_conf/rebar.config index e0456112b..35563d233 100644 --- a/apps/emqx_conf/rebar.config +++ b/apps/emqx_conf/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 563c9dc1d..e9fbbb4b9 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -3,7 +3,6 @@ {vsn, "0.1.0"}, {registered, []}, {mod, {emqx_conf_app, []}}, - {included_applications, [hocon]}, {applications, [kernel, stdlib]}, {env, []}, {modules, []} diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs deleted file mode 100644 index 86c40a6d2..000000000 --- a/apps/emqx_connector/mix.exs +++ /dev/null @@ -1,28 +0,0 @@ -defmodule EMQXConnector.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_connector - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [ - {:emqx_resource, in_umbrella: true, runtime: false}, - {:epgsql, github: "epgsql/epgsql", tag: "4.4.0"}, - {:mysql, github: "emqx/mysql-otp", tag: "1.7.1"}, - {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, - {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.6.7"}, - {:mongodb, github: "emqx/mongodb-erlang", tag: "v3.0.10"} - ] - end -end diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 58706e950..f919fc943 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -4,6 +4,7 @@ ]}. {deps, [ + {emqx, {path, "../emqx"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index fe8bb6c97..d83d16764 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -12,6 +12,7 @@ eredis_cluster, eredis, epgsql, + eldap2, mysql, mongodb, ehttpc, diff --git a/apps/emqx_dashboard/mix.exs b/apps/emqx_dashboard/mix.exs deleted file mode 100644 index e70feef50..000000000 --- a/apps/emqx_dashboard/mix.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule EMQXDashboard.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_dashboard - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [ - {:emqx, in_umbrella: true} - ] - end -end diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index d0a1fbde4..618fc203d 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,4 +1,6 @@ -{deps, []}. +{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} + , {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs deleted file mode 100644 index 85f67b40c..000000000 --- a/apps/emqx_exhook/mix.exs +++ /dev/null @@ -1,29 +0,0 @@ -defmodule EMQXExhook.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_exhook - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps(), - compilers: [:protos | Mix.compilers()], - aliases: ["compile.protos": &protos/1] - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end - - defp protos(_args) do - __ENV__.file - |> Path.dirname() - |> EMQX.Mix.Common.compile_protos() - end -end diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index afdaad084..8229c3a55 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} + [ {emqx, {path, "../emqx"}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_gateway/mix.exs b/apps/emqx_gateway/mix.exs deleted file mode 100644 index 07267088d..000000000 --- a/apps/emqx_gateway/mix.exs +++ /dev/null @@ -1,32 +0,0 @@ -defmodule EMQXGateway.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_gateway - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps(), - compilers: [:protos | Mix.compilers()], - aliases: ["compile.protos": &protos/1] - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp protos(_args) do - __ENV__.file - |> Path.dirname() - |> EMQX.Mix.Common.compile_protos() - end - - defp deps() do - EMQX.Mix.Common.from_rebar_deps!() ++ - [ - {:emqx, in_umbrella: true} - ] - end -end diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 44f74eacf..c94a84de9 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. {deps, [ + {emqx, {path, "../emqx"}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. diff --git a/apps/emqx_machine/mix.exs b/apps/emqx_machine/mix.exs deleted file mode 100644 index 2943fe1fa..000000000 --- a/apps/emqx_machine/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXMachine.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_machine - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_machine/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_management/mix.exs b/apps/emqx_management/mix.exs deleted file mode 100644 index d175bc801..000000000 --- a/apps/emqx_management/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXManagement.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_management - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index e5e336e34..0cc5ace2a 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_modules/mix.exs b/apps/emqx_modules/mix.exs deleted file mode 100644 index e41d907cc..000000000 --- a/apps/emqx_modules/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXModules.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_modules - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 7b30a8fd8..07646091a 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -1 +1,2 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_plugin_libs/mix.exs b/apps/emqx_plugin_libs/mix.exs deleted file mode 100644 index 74ee89c02..000000000 --- a/apps/emqx_plugin_libs/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXPluginLibs.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_plugin_libs - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_plugins/mix.exs b/apps/emqx_plugins/mix.exs deleted file mode 100644 index 4b5ffdda2..000000000 --- a/apps/emqx_plugins/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXPlugins.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_plugins - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_plugins/rebar.config b/apps/emqx_plugins/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_plugins/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_prometheus/mix.exs b/apps/emqx_prometheus/mix.exs deleted file mode 100644 index 84985fc41..000000000 --- a/apps/emqx_prometheus/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXPrometheus.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_prometheus - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app, deps: deps()) - end - - defp deps() do - EMQX.Mix.Common.from_rebar_deps!() - end -end diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index c18bde7a0..8b7e0c67e 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -1,6 +1,8 @@ {deps, [ %% FIXME: tag this as v3.1.3 - {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}} + {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_psk/mix.exs b/apps/emqx_psk/mix.exs deleted file mode 100644 index 3395a5a33..000000000 --- a/apps/emqx_psk/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EmqxPSK.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_psk - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_psk/rebar.config b/apps/emqx_psk/rebar.config index 73696b033..3def1d00e 100644 --- a/apps/emqx_psk/rebar.config +++ b/apps/emqx_psk/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_resource/mix.exs b/apps/emqx_resource/mix.exs deleted file mode 100644 index 7520758e7..000000000 --- a/apps/emqx_resource/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXResource.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_resource - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_retainer/mix.exs b/apps/emqx_retainer/mix.exs deleted file mode 100644 index 57d1f9ddf..000000000 --- a/apps/emqx_retainer/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXRetainer.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_retainer - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index b49f979ac..8ab1ad7b0 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_rule_engine/mix.exs b/apps/emqx_rule_engine/mix.exs deleted file mode 100644 index bb5c0c895..000000000 --- a/apps/emqx_rule_engine/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXRuleEngine.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_rule_engine - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 097c18a3d..56a64b436 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {erl_opts, [warn_unused_vars, warn_shadow_vars, diff --git a/apps/emqx_slow_subs/mix.exs b/apps/emqx_slow_subs/mix.exs deleted file mode 100644 index 15f845837..000000000 --- a/apps/emqx_slow_subs/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXSlowSubs.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_slow_subs - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - [] - end -end diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_slow_subs/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_statsd/mix.exs b/apps/emqx_statsd/mix.exs deleted file mode 100644 index 4969eff3e..000000000 --- a/apps/emqx_statsd/mix.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule EMQXStatsd.MixProject do - use Mix.Project - Code.require_file("../../lib/emqx/mix/common.ex") - - @app :emqx_statsd - - def project() do - EMQX.Mix.Common.project( - @app, - deps: deps() - ) - end - - def application() do - EMQX.Mix.Common.application(@app) - end - - defp deps() do - EMQX.Mix.Common.from_rebar_deps!() - end -end diff --git a/lib/emqx/mix/common.ex b/lib/emqx/mix/common.ex deleted file mode 100644 index 5a276cb0c..000000000 --- a/lib/emqx/mix/common.ex +++ /dev/null @@ -1,107 +0,0 @@ -defmodule EMQX.Mix.Common do - @kernel_apps [:kernel, :stdlib, :sasl, :elixir] - - def project(app, overrides \\ []) when is_atom(app) and app != nil do - %{ - vsn: version, - description: description - } = - app - |> erl_app_props!() - |> Map.take([:vsn, :description]) - |> Map.new(fn {k, v} -> {k, to_string(v)} end) - - Keyword.merge( - [ - app: app, - version: version, - description: description, - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.13" - ], - overrides - ) - end - - def application(app, overrides \\ []) when app != nil do - {deps, overrides} = Keyword.pop(overrides, :deps, []) - # get only the dependency names - deps = Enum.map(deps, &elem(&1, 0)) - - app - |> erl_app_props!() - |> Map.take([:registered, :mod, :applications]) - |> Map.update!(:applications, - fn apps -> - deps ++ apps -- @kernel_apps - end) - |> Enum.reject(fn {_k, v} -> is_nil(v) end) - |> Keyword.new() - |> Keyword.merge(overrides) - end - - def erl_apps(app) when app != nil do - from_erl!(app, :applications) - end - - def erl_app_props!(app) do - path = Path.join("src", "#{app}.app.src") - {:ok, [{:application, ^app, props}]} = :file.consult(path) - Map.new(props) - end - - def from_erl!(app, key) when app != nil do - app - |> erl_app_props!() - |> Map.fetch!(key) - end - - def from_rebar_deps!() do - path = "rebar.config" - {:ok, props} = :file.consult(path) - - props - |> Keyword.fetch!(:deps) - |> Enum.map(&rebar_to_mix_dep/1) - end - - def rebar_to_mix_dep({name, {:git, url, {:tag, tag}}}), - do: {name, git: to_string(url), tag: to_string(tag)} - - def rebar_to_mix_dep({name, {:git, url, {:ref, ref}}}), - do: {name, git: to_string(url), ref: to_string(ref)} - - def rebar_to_mix_dep({name, {:git, url, {:branch, branch}}}), - do: {name, git: to_string(url), branch: to_string(branch)} - - def rebar_to_mix_dep({name, vsn}) when is_list(vsn), - do: {name, to_string(vsn)} - - def compile_protos(mix_filepath) do - app_path = Path.dirname(mix_filepath) - - config = [ - :use_packages, - :maps, - :strings_as_binaries, - rename: {:msg_name, :snake_case}, - rename: {:msg_fqname, :base_name}, - i: '.', - report_errors: false, - o: app_path |> Path.join("src") |> to_charlist(), - module_name_prefix: 'emqx_', - module_name_suffix: '_pb' - ] - - app_path - |> Path.join("priv/protos/*.proto") - |> Path.wildcard() - |> Enum.map(&to_charlist/1) - |> Enum.each(&:gpb_compile.file(&1, config)) - - :ok - end -end diff --git a/mix.exs b/mix.exs index 7525df5a1..c0c02f9e3 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule EMQXUmbrella.MixProject do def project do [ - apps_path: "apps", + app: :emqx_mix, version: pkg_vsn(), deps: deps(), releases: releases() @@ -31,19 +31,19 @@ defmodule EMQXUmbrella.MixProject do {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, {:ekka, github: "emqx/ekka", tag: "0.11.1", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.2.7"}, + {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, {:replayq, "0.3.3", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "klarna-incubator/system_monitor", tag: "2.2.0"}, # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.22.0"}, - {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1"}, + {:hocon, github: "emqx/hocon", tag: "0.22.0", override: true}, + {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, # in conflict by ehttpc and emqtt @@ -57,11 +57,41 @@ defmodule EMQXUmbrella.MixProject do # in conflict by cowboy_swagger and cowboy {:ranch, "1.8.0", override: true}, # in conflict by emqx and observer_cli - {:recon, github: "ferd/recon", tag: "2.5.1", override: true} - ] ++ bcrypt_dep() ++ quicer_dep() + {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, + {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true} + ] ++ + Enum.map( + [ + :emqx, + :emqx_conf, + :emqx_machine, + :emqx_plugin_libs, + :emqx_resource, + :emqx_connector, + :emqx_authn, + :emqx_authz, + :emqx_auto_subscribe, + :emqx_gateway, + :emqx_exhook, + :emqx_bridge, + :emqx_rule_engine, + :emqx_modules, + :emqx_management, + :emqx_dashboard, + :emqx_statsd, + :emqx_retainer, + :emqx_prometheus, + :emqx_psk, + :emqx_slow_subs, + :emqx_plugins + ], + &umbrella/1 + ) ++ bcrypt_dep() ++ quicer_dep() end - defp releases do + defp umbrella(app), do: {app, path: "apps/#{app}", manager: :rebar3, override: true} + + defp releases() do [ emqx: [ applications: [ @@ -102,7 +132,8 @@ defmodule EMQXUmbrella.MixProject do emqx_prometheus: :permanent, emqx_psk: :permanent, emqx_slow_subs: :permanent, - emqx_plugins: :permanent + emqx_plugins: :permanent, + emqx_mix: :none ], skip_mode_validation_for: [ :emqx_gateway, @@ -122,7 +153,7 @@ defmodule EMQXUmbrella.MixProject do :assemble, &create_RELEASES/1, ©_files/1, - ©_nodetool/1, + ©_nodetool/1 ] ] ] @@ -149,35 +180,40 @@ defmodule EMQXUmbrella.MixProject do Path.join(etc, "vm.args"), force: overwrite? ) + Mix.Generator.copy_file( "apps/emqx/etc/emqx_cloud/vm.args", Path.join(release.version_path, "vm.args"), force: overwrite? ) - release end # needed by nodetool and by release_handler def create_RELEASES(release) do - apps = Enum.map(release.applications, fn {app_name, app_props} -> - app_vsn = Keyword.fetch!(app_props, :vsn) - app_path = - "./lib" - |> Path.join("#{app_name}-#{app_vsn}") - |> to_charlist() - {app_name, app_vsn, app_path} - end) - release_entry = - [{ - :release, - to_charlist(release.name), - to_charlist(release.version), - release.erts_version, - apps, - :permanent - }] + apps = + Enum.map(release.applications, fn {app_name, app_props} -> + app_vsn = Keyword.fetch!(app_props, :vsn) + + app_path = + "./lib" + |> Path.join("#{app_name}-#{app_vsn}") + |> to_charlist() + + {app_name, app_vsn, app_path} + end) + + release_entry = [ + { + :release, + to_charlist(release.name), + to_charlist(release.version), + release.erts_version, + apps, + :permanent + } + ] release.path |> Path.join("releases") @@ -207,7 +243,7 @@ defmodule EMQXUmbrella.MixProject do def bcrypt_dep() do if enable_bcrypt?(), - do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0"}], + do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}], else: [] end diff --git a/mix.lock b/mix.lock index 54812891f..3213601fc 100644 --- a/mix.lock +++ b/mix.lock @@ -10,11 +10,12 @@ "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, "ekka": {:git, "https://github.com/emqx/ekka.git", "005fd6bb94199dc2ecb4ba03284f253b408e02d9", [tag: "0.11.1"]}, + "eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]}, "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, "epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]}, - "eredis": {:hex, :eredis, "1.2.0", "0b8e9cfc2c00fa1374cd107ea63b49be08d933df2cf175e6a89b73dd9c380de4", [:rebar3], [], "hexpm", "d9b5abef2c2c8aba8f32aa018203e0b3dc8b1157773b254ab1d4c2002317f1e1"}, - "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster.git", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, + "eredis": {:git, "https://github.com/emqx/eredis", "75f2b8eedbe631136326680225efbcd2684e93e7", [tag: "1.2.5"]}, + "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]}, "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]}, @@ -29,15 +30,15 @@ "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, "jiffy": {:git, "https://github.com/emqx/jiffy.git", "baa1f4e750ae3c5c9e54f9c2e52280b7fc24a8d9", [tag: "1.0.5"]}, "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, - "jsx": {:hex, :jsx, "2.9.0", "d2f6e5f069c00266cad52fb15d87c428579ea4d7d73a33669e12679e203329dd", [:mix, :rebar3], [], "hexpm", "8ee1db1cabafdd578a2776a6aaae87c2a8ce54b47b59e9ec7dab5d7eb71cd8dc"}, + "jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]}, "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, - "mongodb": {:git, "https://github.com/emqx/mongodb-erlang.git", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, + "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, - "mysql": {:git, "https://github.com/emqx/mysql-otp.git", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, + "mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]}, diff --git a/mix_release.sh b/mix_release.sh index 321f0fbe1..729647e84 100755 --- a/mix_release.sh +++ b/mix_release.sh @@ -22,5 +22,6 @@ sed -i -E 's#level => warning#level => debug#g' _build/dev/rel/emqx/releases/5.0 # cp _build/emqx/rel/emqx/releases/emqx_vars _build/dev/rel/emqx/releases/ cp _build/emqx/rel/emqx/etc/emqx.conf _build/dev/rel/emqx/etc/ +cp -r apps/emqx/etc/certs _build/dev/rel/emqx/etc/ echo "telemetry { enable = false }" >> _build/dev/rel/emqx/etc/emqx.conf diff --git a/rebar.config.erl b/rebar.config.erl index dbaf9d6a3..55004569c 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -248,6 +248,7 @@ relx_apps(ReleaseType, Edition) -> , inets , compiler , runtime_tools + , {hocon, load} , {emqx, load} % started by emqx_machine , {emqx_conf, load} , emqx_machine From 93d603e336d069c65c7289b3314f9a633c6b5dff Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Dec 2021 09:46:40 -0300 Subject: [PATCH 04/31] chore(mix): copy more required files to release This adds a few more required file copying/templating to the release build step. --- mix.exs | 55 +++++++++++++++++++++++++++++++++++++++++++++++++- mix_release.sh | 4 ++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index c0c02f9e3..4de8fa9e4 100644 --- a/mix.exs +++ b/mix.exs @@ -167,6 +167,7 @@ defmodule EMQXUmbrella.MixProject do Mix.Generator.create_directory(bin) Mix.Generator.create_directory(etc) + Mix.Generator.create_directory(Path.join(etc, "certs")) Mix.Generator.copy_file( "apps/emqx_authz/etc/acl.conf", @@ -174,19 +175,71 @@ defmodule EMQXUmbrella.MixProject do force: overwrite? ) - # FIXME: check if cloud/edge!! + # FIXME: check if cloud/edge??? Mix.Generator.copy_file( "apps/emqx/etc/emqx_cloud/vm.args", Path.join(etc, "vm.args"), force: overwrite? ) + # FIXME: check if cloud/edge!! Mix.Generator.copy_file( "apps/emqx/etc/emqx_cloud/vm.args", Path.join(release.version_path, "vm.args"), force: overwrite? ) + # required by emqx_authz + File.cp_r!( + "apps/emqx/etc/certs", + Path.join(etc, "certs") + ) + + # this is required by the produced escript / nodetool + Mix.Generator.copy_file( + Path.join(release.version_path, "start_clean.boot"), + Path.join(bin, "no_dot_erlang.boot"), + force: overwrite? + ) + + # This is generated by `scripts/merge-config.escript` or `make + # conf-segs`. So, this should be run before the release. + # TODO: run as a "compiler" step??? + + conf_rebar_template = File.read!("apps/emqx_conf/etc/emqx.conf.all") + # we must not consider surrounding space in the template var name + # because some help strings contain informative variables that + # should not be interpolated, and those have no spaces. + conf_eex_template = + Regex.replace( + ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/, + conf_rebar_template, + "<%= \\g{1} %>" + ) + + # FIXME: change variables by package type??? + conf_rendered = + EEx.eval_string(conf_eex_template, + platform_bin_dir: "bin", + platform_data_dir: "data", + platform_etc_dir: "etc", + platform_lib_dir: "lib", + platform_log_dir: "log", + platform_plugins_dir: "plugins", + runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)", + runner_bin_dir: "$RUNNER_ROOT_DIR/bin", + runner_etc_dir: "$RUNNER_ROOT_DIR/etc", + runner_lib_dir: "$RUNNER_ROOT_DIR/lib", + runner_log_dir: "$RUNNER_ROOT_DIR/log", + runner_data_dir: "$RUNNER_ROOT_DIR/data", + runner_user: "" + ) + + File.write!( + Path.join(etc, "emqx.conf"), + conf_rendered + ) + release end diff --git a/mix_release.sh b/mix_release.sh index 729647e84..4166e069a 100755 --- a/mix_release.sh +++ b/mix_release.sh @@ -21,7 +21,7 @@ sed -i -E 's#level => warning#level => debug#g' _build/dev/rel/emqx/releases/5.0 # cp _build/emqx/rel/emqx/releases/emqx_vars _build/dev/rel/emqx/releases/ -cp _build/emqx/rel/emqx/etc/emqx.conf _build/dev/rel/emqx/etc/ -cp -r apps/emqx/etc/certs _build/dev/rel/emqx/etc/ +# cp _build/emqx/rel/emqx/etc/emqx.conf _build/dev/rel/emqx/etc/ +# cp -r apps/emqx/etc/certs _build/dev/rel/emqx/etc/ echo "telemetry { enable = false }" >> _build/dev/rel/emqx/etc/emqx.conf From a5939790c5f7e4aa84c02e10107f2b707c39a0a6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Dec 2021 10:35:06 -0300 Subject: [PATCH 05/31] refactor(bin): extract common script functionalities into separate files In order for us to reuse most of the functionalities implemented in `bin/eqmx` also in the Elixir release script, common variable and function definitions are extracted in separated bash files that can be sourced by both Rebar and Elixir release scripts. --- bin/common_defs.sh | 60 +++++++ bin/common_defs2.sh | 65 ++++++++ bin/common_functions.sh | 217 +++++++++++++++++++++++++ bin/emqx | 339 +--------------------------------------- rebar.config.erl | 3 + 5 files changed, 352 insertions(+), 332 deletions(-) create mode 100644 bin/common_defs.sh create mode 100644 bin/common_defs2.sh create mode 100644 bin/common_functions.sh diff --git a/bin/common_defs.sh b/bin/common_defs.sh new file mode 100644 index 000000000..45d35eeb4 --- /dev/null +++ b/bin/common_defs.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + +ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +# shellcheck disable=SC1090 +. "$ROOT_DIR"/releases/emqx_vars + +# defined in emqx_vars +export RUNNER_ROOT_DIR +export RUNNER_ETC_DIR +export REL_VSN + +RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" +CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" +REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" +SCHEMA_MOD=emqx_conf_schema + +WHOAMI=$(whoami) + +# Make sure data/configs exists +CONFIGS_DIR="$RUNNER_DATA_DIR/configs" + +# hocon try to read environment variables starting with "EMQX_" +export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' + +export ROOTDIR="$RUNNER_ROOT_DIR" +export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN" +export BINDIR="$ERTS_DIR/bin" +export EMU="beam" +export PROGNAME="erl" +export ERTS_LIB_DIR="$ERTS_DIR/../lib" +DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs" + +## backward compatible +if [ -d "$ERTS_DIR/lib" ]; then + export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" +fi + +# EPMD_ARG="-start_epmd true $PROTO_DIST_ARG" +NO_EPMD="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka" +EPMD_ARG="${EPMD_ARG:-${NO_EPMD}}" + +# Warn the user if ulimit -n is less than 1024 +ULIMIT_F=$(ulimit -n) +if [ "$ULIMIT_F" -lt 1024 ]; then + echo "!!!!" + echo "!!!! WARNING: ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum." + echo "!!!!" +fi + +SED_REPLACE="sed -i " +case $(sed --help 2>&1) in + *GNU*) SED_REPLACE="sed -i ";; + *BusyBox*) SED_REPLACE="sed -i ";; + *) SED_REPLACE="sed -i '' ";; +esac diff --git a/bin/common_defs2.sh b/bin/common_defs2.sh new file mode 100644 index 000000000..f699add3c --- /dev/null +++ b/bin/common_defs2.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +## NOTE: those are defined separately from `common_defs.sh` because +## they require `common_functions.sh` to be sourced prior to sourcing +## this file. Basically, the definitions below depend on the function +## `call_hocon`, which is defined in `common_functions.sh`. Also, +## they require the variable `IS_BOOT_COMMAND` to be set to either +## `yes` or `no` for the name definition to be done properly. + +## make EMQX_NODE_COOKIE right +if [ -n "${EMQX_NODE_NAME:-}" ]; then + export EMQX_NODE__NAME="${EMQX_NODE_NAME}" + unset EMQX_NODE_NAME +fi + +## Possible ways to configure emqx node name: +## 1. configure node.name in emqx.conf +## 2. override with environment variable EMQX_NODE__NAME +## Node name is either short-name (without '@'), e.g. 'emqx' +## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1' +NAME="${EMQX_NODE__NAME:-}" +if [ -z "$NAME" ]; then + if [ "$IS_BOOT_COMMAND" = 'yes' ]; then + # for boot commands, inspect emqx.conf for node name + NAME="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")" + else + vm_args_file="$(latest_vm_args 'EMQX_NODE__NAME')" + NAME="$(grep -E '^-s?name' "${vm_args_file}" | awk '{print $2}')" + fi +fi + +# force to use 'emqx' short name +[ -z "$NAME" ] && NAME='emqx' +MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME" + +case "$NAME" in + *@*) + NAME_TYPE='-name' + ;; + *) + NAME_TYPE='-sname' +esac +SHORT_NAME="$(echo "$NAME" | awk -F'@' '{print $1}')" +export ESCRIPT_NAME="$SHORT_NAME" + +PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" + +## make EMQX_NODE_COOKIE right +if [ -n "${EMQX_NODE_COOKIE:-}" ]; then + export EMQX_NODE__COOKIE="${EMQX_NODE_COOKIE}" + unset EMQX_NODE_COOKIE +fi +COOKIE="${EMQX_NODE__COOKIE:-}" +if [ -z "$COOKIE" ]; then + if [ "$IS_BOOT_COMMAND" = 'yes' ]; then + COOKIE="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")" + else + vm_args_file="$(latest_vm_args 'EMQX_NODE__COOKIE')" + COOKIE="$(grep -E '^-setcookie' "${vm_args_file}" | awk '{print $2}')" + fi +fi + +if [ -z "$COOKIE" ]; then + die "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE" +fi diff --git a/bin/common_functions.sh b/bin/common_functions.sh new file mode 100644 index 000000000..cff9ed76a --- /dev/null +++ b/bin/common_functions.sh @@ -0,0 +1,217 @@ +#!/bin/bash + +# Echo to stderr on errors +echoerr() { echo "ERROR: $*" 1>&2; } + +die() { + echoerr "ERROR: $1" + errno=${2:-1} + exit "$errno" +} + +assert_node_alive() { + if ! relx_nodetool "ping" > /dev/null; then + die "node_is_not_running!" 1 + fi +} + +check_erlang_start() { + "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s erlang halt +} + +# Simple way to check the correct user and fail early +check_user() { + # Validate that the user running the script is the owner of the + # RUN_DIR. + if [ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]; then + if [ "x$WHOAMI" != "xroot" ]; then + echo "You need to be root or use sudo to run this command" + exit 1 + fi + CMD="DEBUG=$DEBUG \"$RUNNER_SCRIPT\" " + for ARG in "$@"; do + CMD="${CMD} \"$ARG\"" + done + # This will drop priviledges into the runner user + # It exec's in a new shell and the current shell will exit + exec su - "$RUNNER_USER" -c "$CMD" + fi +} + +# Get node pid +relx_get_pid() { + if output="$(relx_nodetool rpcterms os getpid)" + then + # shellcheck disable=SC2001 # Escaped quote taken as closing quote in editor + echo "$output" | sed -e 's/"//g' + return 0 + else + echo "$output" + return 1 + fi +} + +# Connect to a remote node +relx_rem_sh() { + # Generate a unique id used to allow multiple remsh to the same node + # transparently + id="remsh$(relx_gen_id)-${NAME}" + # Get the node's ticktime so that we use the same thing. + TICKTIME="$(relx_nodetool rpcterms net_kernel get_net_ticktime)" + + # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace + # Setup remote shell command to control node + exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot "$REL_DIR/start_clean" \ + -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ + -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" $EPMD_ARG +} + +# Generate a random id +relx_gen_id() { + od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}' +} + +# Control a node +relx_nodetool() { + command="$1"; shift + ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \ + "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ + -setcookie "$COOKIE" "$command" "$@" +} + +call_hocon() { + "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \ + || die "call_hocon_failed: $*" $? +} + +# Run an escript in the node's environment +relx_escript() { + shift; scriptpath="$1"; shift + "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@" +} + +# Output a start command for the last argument of run_erl +relx_start_command() { + printf "exec \"%s\" \"%s\"" "$RUNNER_SCRIPT" \ + "$START_OPTION" +} + +# Function to generate app.config and vm.args +generate_config() { + local name_type="$1" + local node_name="$2" + ## Delete the *.siz files first or it cann't start after + ## changing the config 'log.rotation.size' + rm -rf "${RUNNER_LOG_DIR}"/*.siz + + EMQX_LICENSE_CONF_OPTION="" + if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then + EMQX_LICENSE_CONF_OPTION="-c ${EMQX_LICENSE_CONF}" + fi + + ## timestamp for each generation + local NOW_TIME + NOW_TIME="$(call_hocon now_time)" + + ## ths command populates two files: app.