diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml new file mode 100644 index 000000000..f2aef28c9 --- /dev/null +++ b/.github/workflows/elixir_release.yml @@ -0,0 +1,39 @@ +# FIXME: temporary workflow for testing; remove later +name: Elixir Build (temporary) + +concurrency: + group: mix-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + - name: "[hack] make emqx with rebar3 to get configs" + run: | + make emqx + cd _build/emqx/rel/emqx + bin/emqx start + bin/emqx stop + - name: setup mix + run: | + mix local.hex --force + mix local.rebar --force + mix deps.get + - name: elixir release + run: ./mix_release.sh + - name: start release + run: | + cd _build/dev/rel/emqx + bin/emqx daemon_iex + - name: check if started + run: | + sleep 10 + nc -zv localhost 1883 diff --git a/.tool-versions b/.tool-versions index a6568713b..1c687075b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ erlang 24.1.5-3 +elixir 1.13.1-otp-24 diff --git a/apps/emqx/mix.exs b/apps/emqx/mix.exs new file mode 100644 index 000000000..847002cc2 --- /dev/null +++ b/apps/emqx/mix.exs @@ -0,0 +1,39 @@ +defmodule EMQX.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + [ + mod: EMQX.Mix.Common.from_erl!(:emqx, :mod), + applications: EMQX.Mix.Common.from_erl!(:emqx, :applications) + ] + end + + # since emqx app is more complicated than others, we manually set + # its dependencies here + defp deps() do + [ + {:lc, git: "https://github.com/qzhuyan/lc.git", tag: "0.1.2"}, + {:gproc, git: "https://github.com/uwiger/gproc", tag: "0.8.0"}, + {:typerefl, git: "https://github.com/k32/typerefl", tag: "0.8.5"}, + {:jiffy, git: "https://github.com/emqx/jiffy", tag: "1.0.5"}, + {:cowboy, git: "https://github.com/emqx/cowboy", tag: "2.9.0"}, + {:esockd, git: "https://github.com/emqx/esockd", tag: "5.9.0"}, + {:ekka, git: "https://github.com/emqx/ekka", tag: "0.11.1"}, + {:gen_rpc, git: "https://github.com/emqx/gen_rpc", tag: "2.5.1"}, + {:hocon, git: "https://github.com/emqx/hocon.git", tag: "0.22.0"}, + {:pbkdf2, git: "https://github.com/emqx/erlang-pbkdf2.git", tag: "2.0.4"}, + {:recon, git: "https://github.com/ferd/recon", tag: "2.5.1"}, + {:snabbkaffe, git: "https://github.com/kafka4beam/snabbkaffe.git", tag: "0.16.0"} + ] + end +end diff --git a/apps/emqx_authn/mix.exs b/apps/emqx_authn/mix.exs new file mode 100644 index 000000000..07af06495 --- /dev/null +++ b/apps/emqx_authn/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXAuthn.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_authn + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_authz/mix.exs b/apps/emqx_authz/mix.exs new file mode 100644 index 000000000..06a5792b4 --- /dev/null +++ b/apps/emqx_authz/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXAuthz.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_authz + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app, + extra_applications: [:crypto] + ) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_auto_subscribe/mix.exs b/apps/emqx_auto_subscribe/mix.exs new file mode 100644 index 000000000..d60e8f03e --- /dev/null +++ b/apps/emqx_auto_subscribe/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXAutoSubscribe.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_auto_subscribe + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_bridge/mix.exs b/apps/emqx_bridge/mix.exs new file mode 100644 index 000000000..629f1295c --- /dev/null +++ b/apps/emqx_bridge/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXBridge.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_bridge + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_conf/mix.exs b/apps/emqx_conf/mix.exs new file mode 100644 index 000000000..1ad98ee70 --- /dev/null +++ b/apps/emqx_conf/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXConf.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_conf + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs new file mode 100644 index 000000000..86c40a6d2 --- /dev/null +++ b/apps/emqx_connector/mix.exs @@ -0,0 +1,28 @@ +defmodule EMQXConnector.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_connector + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx_resource, in_umbrella: true, runtime: false}, + {:epgsql, github: "epgsql/epgsql", tag: "4.4.0"}, + {:mysql, github: "emqx/mysql-otp", tag: "1.7.1"}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, + {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.6.7"}, + {:mongodb, github: "emqx/mongodb-erlang", tag: "v3.0.10"} + ] + end +end diff --git a/apps/emqx_dashboard/mix.exs b/apps/emqx_dashboard/mix.exs new file mode 100644 index 000000000..e70feef50 --- /dev/null +++ b/apps/emqx_dashboard/mix.exs @@ -0,0 +1,23 @@ +defmodule EMQXDashboard.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_dashboard + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_exhook/mix.exs b/apps/emqx_exhook/mix.exs new file mode 100644 index 000000000..85f67b40c --- /dev/null +++ b/apps/emqx_exhook/mix.exs @@ -0,0 +1,29 @@ +defmodule EMQXExhook.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_exhook + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps(), + compilers: [:protos | Mix.compilers()], + aliases: ["compile.protos": &protos/1] + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end + + defp protos(_args) do + __ENV__.file + |> Path.dirname() + |> EMQX.Mix.Common.compile_protos() + end +end diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_gateway/src/coap/include/emqx_coap.hrl rename to apps/emqx_gateway/include/emqx_coap.hrl diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_gateway/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl rename to apps/emqx_gateway/include/emqx_exproto.hrl diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_gateway/include/emqx_lwm2m.hrl diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_gateway/include/emqx_sn.hrl similarity index 100% rename from apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl rename to apps/emqx_gateway/include/emqx_sn.hrl diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl rename to apps/emqx_gateway/include/emqx_stomp.hrl diff --git a/apps/emqx_gateway/mix.exs b/apps/emqx_gateway/mix.exs new file mode 100644 index 000000000..07267088d --- /dev/null +++ b/apps/emqx_gateway/mix.exs @@ -0,0 +1,32 @@ +defmodule EMQXGateway.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_gateway + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps(), + compilers: [:protos | Mix.compilers()], + aliases: ["compile.protos": &protos/1] + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp protos(_args) do + __ENV__.file + |> Path.dirname() + |> EMQX.Mix.Common.compile_protos() + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() ++ + [ + {:emqx, in_umbrella: true} + ] + end +end diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 0efb8043e..f07e2fbf8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -18,7 +18,7 @@ -behaviour(minirest_api). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([api_spec/0]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index ab079b587..da51def5d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -43,7 +43,7 @@ -export_type([channel/0]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index a567ea7f4..d8d88a4cc 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -28,8 +28,8 @@ , is_message/1 ]). --include("include/emqx_coap.hrl"). --include("apps/emqx/include/types.hrl"). +-include("emqx_coap.hrl"). +-include_lib("emqx/include/types.hrl"). -define(VERSION, 1). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl index 020e38496..d34b1f90c 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ empty/0, reset/1, reset/2 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index 3851b3428..93f15fb6d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -34,7 +34,7 @@ -export([ set/3, set_payload/2, get_option/2 , get_option/3, set_payload_block/3, set_payload_block/4]). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index 9c3a8c451..49d5fe68e 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -18,7 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ new/0 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index d6d05fd40..f6247b6bb 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -28,7 +28,7 @@ -export_type([manager/0, event_result/1]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -type direction() :: in | out. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 07e522309..bb09745b0 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -1,7 +1,7 @@ -module(emqx_coap_transport). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 47bf14d9b..98cb74a42 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 608eae92a..1ff3b9cbb 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index bb8072358..464dda263 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index 1b9afca62..95626abbe 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -19,7 +19,7 @@ % -behaviour(emqx_exproto_v_1_connection_adapter_bhvr). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index c01c6adc5..9d6eee5cc 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_channel). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). %% API -export([ info/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 1ee27228b..d7b056903 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_cmd). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). -export([ mqtt_to_coap/2 , coap_to_mqtt/4 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl index 70844e6d6..f14cf9d9c 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl @@ -23,7 +23,7 @@ , translate_json/1 ]). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index bee1bedcd..c797c5a08 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -18,8 +18,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). %% API -export([ new/0, init/4, update/3, parse_object_list/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl index 94ea31bf8..02c5ecd9f 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -define(TLV_TYPE_OBJECT_INSTANCE, 0). -define(TLV_TYPE_RESOURCE_INSTANCE, 1). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index d744a23f9..9549c62bb 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 3cef3c19e..92c0c6ebe 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl index 0b90d843c..1c188ce95 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ start_link/2 diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index 74c8d2c12..7a18f1236 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl index 59acc71ce..9bac188c8 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl @@ -20,7 +20,7 @@ -behaviour(emqx_gateway_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -export([ initial_parse_state/1 , serialize_opts/0 diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl index 411a34aec..25c7bd2b0 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl @@ -21,7 +21,7 @@ -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_sn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ start_link/2 diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index dfca57cf1..82cfdb142 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl index d75f77abe..18b5c8f55 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl @@ -70,7 +70,7 @@ -behaviour(emqx_gateway_frame). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ initial_parse_state/1 , parse/2 diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl index 99a1508e1..72217552b 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl @@ -17,7 +17,7 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ init/1 , check/3 diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 5e342ec76..3af3418f4 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index dd8fc2f7f..5f74c09c7 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 0782ab1b3..547775dbc 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index 1191d748e..56b8f1c1a 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl index 9e12a7bd4..b377864bb 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include_lib("src/mqttsn/include/emqx_sn.hrl"). +-include_lib("emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_machine/mix.exs b/apps/emqx_machine/mix.exs new file mode 100644 index 000000000..2943fe1fa --- /dev/null +++ b/apps/emqx_machine/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXMachine.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_machine + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_management/mix.exs b/apps/emqx_management/mix.exs new file mode 100644 index 000000000..d175bc801 --- /dev/null +++ b/apps/emqx_management/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXManagement.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_management + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_modules/mix.exs b/apps/emqx_modules/mix.exs new file mode 100644 index 000000000..e41d907cc --- /dev/null +++ b/apps/emqx_modules/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXModules.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_modules + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_plugin_libs/mix.exs b/apps/emqx_plugin_libs/mix.exs new file mode 100644 index 000000000..74ee89c02 --- /dev/null +++ b/apps/emqx_plugin_libs/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPluginLibs.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_plugin_libs + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_plugins/mix.exs b/apps/emqx_plugins/mix.exs new file mode 100644 index 000000000..4b5ffdda2 --- /dev/null +++ b/apps/emqx_plugins/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPlugins.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_plugins + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_prometheus/mix.exs b/apps/emqx_prometheus/mix.exs new file mode 100644 index 000000000..84985fc41 --- /dev/null +++ b/apps/emqx_prometheus/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXPrometheus.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_prometheus + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app, deps: deps()) + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() + end +end diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 02e81b295..c18bde7a0 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -1,5 +1,6 @@ {deps, - [{prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v3.1.1"}}} + [ %% FIXME: tag this as v3.1.3 + {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_psk/mix.exs b/apps/emqx_psk/mix.exs new file mode 100644 index 000000000..3395a5a33 --- /dev/null +++ b/apps/emqx_psk/mix.exs @@ -0,0 +1,21 @@ +defmodule EmqxPSK.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_psk + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_resource/mix.exs b/apps/emqx_resource/mix.exs new file mode 100644 index 000000000..7520758e7 --- /dev/null +++ b/apps/emqx_resource/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXResource.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_resource + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 1b93aa0de..8e5d12fea 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -8,9 +8,9 @@ [kernel, stdlib, gproc, - hocon, jsx, - emqx + emqx, + emqx_conf ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_retainer/mix.exs b/apps/emqx_retainer/mix.exs new file mode 100644 index 000000000..57d1f9ddf --- /dev/null +++ b/apps/emqx_retainer/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXRetainer.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_retainer + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_rule_engine/mix.exs b/apps/emqx_rule_engine/mix.exs new file mode 100644 index 000000000..bb5c0c895 --- /dev/null +++ b/apps/emqx_rule_engine/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXRuleEngine.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_rule_engine + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_slow_subs/mix.exs b/apps/emqx_slow_subs/mix.exs new file mode 100644 index 000000000..15f845837 --- /dev/null +++ b/apps/emqx_slow_subs/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXSlowSubs.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_slow_subs + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + [] + end +end diff --git a/apps/emqx_statsd/mix.exs b/apps/emqx_statsd/mix.exs new file mode 100644 index 000000000..4969eff3e --- /dev/null +++ b/apps/emqx_statsd/mix.exs @@ -0,0 +1,21 @@ +defmodule EMQXStatsd.MixProject do + use Mix.Project + Code.require_file("../../lib/emqx/mix/common.ex") + + @app :emqx_statsd + + def project() do + EMQX.Mix.Common.project( + @app, + deps: deps() + ) + end + + def application() do + EMQX.Mix.Common.application(@app) + end + + defp deps() do + EMQX.Mix.Common.from_rebar_deps!() + end +end diff --git a/lib/emqx/mix/common.ex b/lib/emqx/mix/common.ex new file mode 100644 index 000000000..5a276cb0c --- /dev/null +++ b/lib/emqx/mix/common.ex @@ -0,0 +1,107 @@ +defmodule EMQX.Mix.Common do + @kernel_apps [:kernel, :stdlib, :sasl, :elixir] + + def project(app, overrides \\ []) when is_atom(app) and app != nil do + %{ + vsn: version, + description: description + } = + app + |> erl_app_props!() + |> Map.take([:vsn, :description]) + |> Map.new(fn {k, v} -> {k, to_string(v)} end) + + Keyword.merge( + [ + app: app, + version: version, + description: description, + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.13" + ], + overrides + ) + end + + def application(app, overrides \\ []) when app != nil do + {deps, overrides} = Keyword.pop(overrides, :deps, []) + # get only the dependency names + deps = Enum.map(deps, &elem(&1, 0)) + + app + |> erl_app_props!() + |> Map.take([:registered, :mod, :applications]) + |> Map.update!(:applications, + fn apps -> + deps ++ apps -- @kernel_apps + end) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + |> Keyword.new() + |> Keyword.merge(overrides) + end + + def erl_apps(app) when app != nil do + from_erl!(app, :applications) + end + + def erl_app_props!(app) do + path = Path.join("src", "#{app}.app.src") + {:ok, [{:application, ^app, props}]} = :file.consult(path) + Map.new(props) + end + + def from_erl!(app, key) when app != nil do + app + |> erl_app_props!() + |> Map.fetch!(key) + end + + def from_rebar_deps!() do + path = "rebar.config" + {:ok, props} = :file.consult(path) + + props + |> Keyword.fetch!(:deps) + |> Enum.map(&rebar_to_mix_dep/1) + end + + def rebar_to_mix_dep({name, {:git, url, {:tag, tag}}}), + do: {name, git: to_string(url), tag: to_string(tag)} + + def rebar_to_mix_dep({name, {:git, url, {:ref, ref}}}), + do: {name, git: to_string(url), ref: to_string(ref)} + + def rebar_to_mix_dep({name, {:git, url, {:branch, branch}}}), + do: {name, git: to_string(url), branch: to_string(branch)} + + def rebar_to_mix_dep({name, vsn}) when is_list(vsn), + do: {name, to_string(vsn)} + + def compile_protos(mix_filepath) do + app_path = Path.dirname(mix_filepath) + + config = [ + :use_packages, + :maps, + :strings_as_binaries, + rename: {:msg_name, :snake_case}, + rename: {:msg_fqname, :base_name}, + i: '.', + report_errors: false, + o: app_path |> Path.join("src") |> to_charlist(), + module_name_prefix: 'emqx_', + module_name_suffix: '_pb' + ] + + app_path + |> Path.join("priv/protos/*.proto") + |> Path.wildcard() + |> Enum.map(&to_charlist/1) + |> Enum.each(&:gpb_compile.file(&1, config)) + + :ok + end +end diff --git a/lib/mix/release.exs b/lib/mix/release.exs new file mode 100644 index 000000000..d6c032877 --- /dev/null +++ b/lib/mix/release.exs @@ -0,0 +1,878 @@ +defmodule Mix.Release do + @moduledoc """ + Defines the release structure and convenience for assembling releases. + """ + + @doc """ + The Mix.Release struct has the following read-only fields: + + * `:name` - the name of the release as an atom + * `:version` - the version of the release as a string or + `{:from_app, app_name}` + * `:path` - the path to the release root + * `:version_path` - the path to the release version inside the release + * `:applications` - a map of application with their definitions + * `:erts_source` - the ERTS source as a charlist (or nil) + * `:erts_version` - the ERTS version as a charlist + + The following fields may be modified as long as they keep their defined types: + + * `:boot_scripts` - a map of boot scripts with the boot script name + as key and a keyword list with **all** applications that are part of + it and their modes as value + * `:config_providers` - a list of `{config_provider, term}` tuples where the + first element is a module that implements the `Config.Provider` behaviour + and `term` is the value given to it on `c:Config.Provider.init/1` + * `:options` - a keyword list with all other user supplied release options + * `:overlays` - a list of extra files added to the release. If you have a custom + step adding extra files to a release, you can add these files to the `:overlays` + field so they are also considered on further commands, such as tar/zip. Each entry + in overlays is the relative path to the release root of each file + * `:steps` - a list of functions that receive the release and returns a release. + Must also contain the atom `:assemble` which is the internal assembling step. + May also contain the atom `:tar` to create a tarball of the release. + + """ + defstruct [ + :name, + :version, + :path, + :version_path, + :applications, + :boot_scripts, + :erts_source, + :erts_version, + :config_providers, + :options, + :overlays, + :steps + ] + + @type mode :: :permanent | :transient | :temporary | :load | :none + @type application :: atom() + @type t :: %__MODULE__{ + name: atom(), + version: String.t(), + path: String.t(), + version_path: String.t() | {:from_app, application()}, + applications: %{application() => keyword()}, + boot_scripts: %{atom() => [{application(), mode()}]}, + erts_version: charlist(), + erts_source: charlist() | nil, + config_providers: [{module, term}], + options: keyword(), + overlays: list(String.t()), + steps: [(t -> t) | :assemble, ...] + } + + @default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent] + @safe_modes [:permanent, :temporary, :transient] + @unsafe_modes [:load, :none] + @significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c + @copy_app_dirs ["priv"] + + @doc false + @spec from_config!(atom, keyword, keyword) :: t + def from_config!(name, config, overrides) do + {name, apps, opts} = find_release(name, config) + + unless Atom.to_string(name) =~ ~r/^[a-z][a-z0-9_]*$/ do + Mix.raise( + "Invalid release name. A release name must start with a lowercase ASCII letter, " <> + "followed by lowercase ASCII letters, numbers, or underscores, got: #{inspect(name)}" + ) + end + + opts = + [overwrite: false, quiet: false, strip_beams: true] + |> Keyword.merge(opts) + |> Keyword.merge(overrides) + + {include_erts, opts} = Keyword.pop(opts, :include_erts, true) + {erts_source, erts_lib_dir, erts_version} = erts_data(include_erts) + + deps_apps = Mix.Project.deps_apps() + loaded_apps = apps |> Keyword.keys() |> load_apps(deps_apps, %{}, erts_lib_dir, [], :root) + + # Make sure IEx is either an active part of the release or add it as none. + {loaded_apps, apps} = + if Map.has_key?(loaded_apps, :iex) do + {loaded_apps, apps} + else + {load_apps([:iex], deps_apps, loaded_apps, erts_lib_dir, [], :root), apps ++ [iex: :none]} + end + + start_boot = build_start_boot(loaded_apps, apps) + start_clean_boot = build_start_clean_boot(start_boot) + + {path, opts} = + Keyword.pop_lazy(opts, :path, fn -> + Path.join([Mix.Project.build_path(config), "rel", Atom.to_string(name)]) + end) + + path = Path.absname(path) + + {version, opts} = + Keyword.pop_lazy(opts, :version, fn -> + config[:version] || + Mix.raise( + "No :version found. Please make sure a :version is set in your project definition " <> + "or inside the release the configuration" + ) + end) + + version = + case version do + {:from_app, app} -> + Application.load(app) + version = Application.spec(app, :vsn) + + if !version do + Mix.raise( + "Could not find version for #{inspect(app)}, please make sure the application exists" + ) + end + + to_string(version) + + "" -> + Mix.raise("The release :version cannot be an empty string") + + _ -> + version + end + + {config_providers, opts} = Keyword.pop(opts, :config_providers, []) + {steps, opts} = Keyword.pop(opts, :steps, [:assemble]) + validate_steps!(steps) + + %Mix.Release{ + name: name, + version: version, + path: path, + version_path: Path.join([path, "releases", version]), + erts_source: erts_source, + erts_version: erts_version, + applications: loaded_apps, + boot_scripts: %{start: start_boot, start_clean: start_clean_boot}, + config_providers: config_providers, + options: opts, + overlays: [], + steps: steps + } + end + + defp find_release(name, config) do + {name, opts_fun_or_list} = lookup_release(name, config) || infer_release(config) + opts = if is_function(opts_fun_or_list, 0), do: opts_fun_or_list.(), else: opts_fun_or_list + {apps, opts} = Keyword.pop(opts, :applications, []) + + if apps == [] and Mix.Project.umbrella?(config) do + bad_umbrella!() + end + + app = Keyword.get(config, :app) + apps = Keyword.merge(@default_apps, apps) + + if is_nil(app) or Keyword.has_key?(apps, app) do + {name, apps, opts} + else + {name, apps ++ [{app, :permanent}], opts} + end + end + + defp lookup_release(nil, config) do + case Keyword.get(config, :releases, []) do + [] -> + nil + + [{name, opts}] -> + {name, opts} + + [_ | _] -> + case Keyword.get(config, :default_release) do + nil -> + Mix.raise( + "\"mix release\" was invoked without a name but there are multiple releases. " <> + "Please call \"mix release NAME\" or set :default_release in your project configuration" + ) + + name -> + lookup_release(name, config) + end + end + end + + defp lookup_release(name, config) do + if opts = config[:releases][name] do + {name, opts} + else + found = Keyword.get(config, :releases, []) + + Mix.raise( + "Unknown release #{inspect(name)}. " <> + "The available releases are: #{inspect(Keyword.keys(found))}" + ) + end + end + + defp infer_release(config) do + if Mix.Project.umbrella?(config) do + bad_umbrella!() + else + {Keyword.fetch!(config, :app), []} + end + end + + defp bad_umbrella! do + Mix.raise(""" + Umbrella projects require releases to be explicitly defined with \ + a non-empty applications key that chooses which umbrella children \ + should be part of the releases: + + releases: [ + foo: [ + applications: [child_app_foo: :permanent] + ], + bar: [ + applications: [child_app_bar: :permanent] + ] + ] + + Alternatively you can perform the release from the children applications + """) + end + + defp erts_data(erts_data) when is_function(erts_data) do + erts_data(erts_data.()) + end + + defp erts_data(false) do + {nil, :code.lib_dir(), :erlang.system_info(:version)} + end + + defp erts_data(true) do + version = :erlang.system_info(:version) + {:filename.join(:code.root_dir(), 'erts-#{version}'), :code.lib_dir(), version} + end + + defp erts_data(erts_source) when is_binary(erts_source) do + if File.exists?(erts_source) do + [_, erts_version] = erts_source |> Path.basename() |> String.split("-") + erts_lib_dir = erts_source |> Path.dirname() |> Path.join("lib") |> to_charlist() + {to_charlist(erts_source), erts_lib_dir, to_charlist(erts_version)} + else + Mix.raise("Could not find ERTS system at #{inspect(erts_source)}") + end + end + + defp load_apps(apps, deps_apps, seen, otp_root, optional, type) do + for app <- apps, reduce: seen do + seen -> + if reentrant_seen = reentrant(seen, app, type) do + reentrant_seen + else + load_app(app, deps_apps, seen, otp_root, optional, type) + end + end + end + + defp reentrant(seen, app, type) do + properties = seen[app] + + cond do + is_nil(properties) -> + nil + + type != :root and properties[:type] != type -> + if properties[:type] == :root do + put_in(seen[app][:type], type) + else + Mix.raise( + "#{inspect(app)} is listed both as a regular application and as an included application" + ) + end + + true -> + seen + end + end + + defp load_app(app, deps_apps, seen, otp_root, optional, type) do + cond do + path = app not in deps_apps && otp_path(otp_root, app) -> + do_load_app(app, path, deps_apps, seen, otp_root, true, type) + + path = code_path(app) -> + do_load_app(app, path, deps_apps, seen, otp_root, false, type) + + app in optional -> + seen + + true -> + Mix.raise("Could not find application #{inspect(app)}") + end + end + + defp otp_path(otp_root, app) do + path = Path.join(otp_root, "#{app}-*") + + case Path.wildcard(path) do + [] -> nil + paths -> paths |> Enum.sort() |> List.last() |> to_charlist() + end + end + + defp code_path(app) do + case :code.lib_dir(app) do + {:error, :bad_name} -> nil + path -> path + end + end + + defp do_load_app(app, path, deps_apps, seen, otp_root, otp_app?, type) do + case :file.consult(Path.join(path, "ebin/#{app}.app")) do + {:ok, terms} -> + [{:application, ^app, properties}] = terms + value = [path: path, otp_app?: otp_app?, type: type] ++ properties + seen = Map.put(seen, app, value) + applications = Keyword.get(properties, :applications, []) + optional = Keyword.get(properties, :optional_applications, []) + seen = load_apps(applications, deps_apps, seen, otp_root, optional, :depended) + included_applications = Keyword.get(properties, :included_applications, []) + load_apps(included_applications, deps_apps, seen, otp_root, [], :included) + + {:error, reason} -> + Mix.raise("Could not load #{app}.app. Reason: #{inspect(reason)}") + end + end + + defp build_start_boot(all_apps, specified_apps) do + specified_apps ++ + Enum.sort( + for( + {app, props} <- all_apps, + not List.keymember?(specified_apps, app, 0), + do: {app, default_mode(props)} + ) + ) + end + + defp default_mode(props) do + if props[:type] == :included, do: :load, else: :permanent + end + + defp build_start_clean_boot(boot) do + for({app, _mode} <- boot, do: {app, :none}) + |> Keyword.put(:stdlib, :permanent) + |> Keyword.put(:kernel, :permanent) + end + + defp validate_steps!(steps) do + valid_atoms = [:assemble, :tar] + + if not is_list(steps) or Enum.any?(steps, &(&1 not in valid_atoms and not is_function(&1, 1))) do + Mix.raise(""" + The :steps option must be a list of: + + * anonymous function that receives one argument + * the atom :assemble or :tar + + Got: #{inspect(steps)} + """) + end + + if Enum.count(steps, &(&1 == :assemble)) != 1 do + Mix.raise("The :steps option must contain the atom :assemble once, got: #{inspect(steps)}") + end + + if :assemble in Enum.drop_while(steps, &(&1 != :tar)) do + Mix.raise("The :tar step must come after :assemble") + end + + if Enum.count(steps, &(&1 == :tar)) > 1 do + Mix.raise("The :steps option can only contain the atom :tar once") + end + + :ok + end + + @doc """ + Makes the `sys.config` structure. + + If there are config providers, then a value is injected into + the `:elixir` application configuration in `sys_config` to be + read during boot and trigger the providers. + + It uses the following release options to customize its behaviour: + + * `:reboot_system_after_config` + * `:start_distribution_during_config` + * `:prune_runtime_sys_config_after_boot` + + In case there are no config providers, it doesn't change `sys_config`. + """ + @spec make_sys_config(t, keyword(), Config.Provider.config_path()) :: + :ok | {:error, String.t()} + def make_sys_config(release, sys_config, config_provider_path) do + {sys_config, runtime_config?} = + merge_provider_config(release, sys_config, config_provider_path) + + path = Path.join(release.version_path, "sys.config") + + args = [runtime_config?, sys_config] + format = "%% coding: utf-8~n%% RUNTIME_CONFIG=~s~n~tw.~n" + File.mkdir_p!(Path.dirname(path)) + File.write!(path, IO.chardata_to_string(:io_lib.format(format, args))) + + case :file.consult(path) do + {:ok, _} -> + :ok + + {:error, reason} -> + invalid = + for {app, kv} <- sys_config, + {key, value} <- kv, + not valid_config?(value), + do: """ + + Application: #{inspect(app)} + Key: #{inspect(key)} + Value: #{inspect(value)} + """ + + message = + case invalid do + [] -> + "Could not read configuration file. Reason: #{inspect(reason)}" + + _ -> + "Could not read configuration file. It has invalid configuration terms " <> + "such as functions, references, and pids. Please make sure your configuration " <> + "is made of numbers, atoms, strings, maps, tuples and lists. The following entries " <> + "are wrong:\n#{Enum.join(invalid)}" + end + + {:error, message} + end + end + + defp valid_config?(m) when is_map(m), + do: Enum.all?(Map.delete(m, :__struct__), &valid_config?/1) + + defp valid_config?(l) when is_list(l), do: Enum.all?(l, &valid_config?/1) + defp valid_config?(t) when is_tuple(t), do: Enum.all?(Tuple.to_list(t), &valid_config?/1) + defp valid_config?(o), do: is_number(o) or is_atom(o) or is_binary(o) + + defp merge_provider_config(%{config_providers: []}, sys_config, _), do: {sys_config, false} + + defp merge_provider_config(release, sys_config, config_path) do + {reboot?, extra_config, initial_config} = start_distribution(release) + + prune_runtime_sys_config_after_boot = + Keyword.get(release.options, :prune_runtime_sys_config_after_boot, false) + + opts = [ + extra_config: initial_config, + prune_runtime_sys_config_after_boot: prune_runtime_sys_config_after_boot, + reboot_system_after_config: reboot?, + validate_compile_env: validate_compile_env(release) + ] + + init_config = Config.Provider.init(release.config_providers, config_path, opts) + {Config.Reader.merge(sys_config, init_config ++ extra_config), reboot?} + end + + defp validate_compile_env(release) do + with true <- Keyword.get(release.options, :validate_compile_env, true), + [_ | _] = compile_env <- compile_env(release) do + compile_env + else + _ -> false + end + end + + defp compile_env(release) do + for {_, properties} <- release.applications, + triplet <- Keyword.get(properties, :compile_env, []), + do: triplet + end + + defp start_distribution(%{options: opts}) do + reboot? = Keyword.get(opts, :reboot_system_after_config, false) + early_distribution? = Keyword.get(opts, :start_distribution_during_config, false) + + if not reboot? or early_distribution? do + {reboot?, [], []} + else + {true, [kernel: [start_distribution: false]], [kernel: [start_distribution: true]]} + end + end + + @doc """ + Copies the cookie to the given path. + + If a cookie option was given, we compare it with + the contents of the file (if any), and ask the user + if they want to override. + + If there is no option, we generate a random one + the first time. + """ + @spec make_cookie(t, Path.t()) :: :ok + def make_cookie(release, path) do + cond do + cookie = release.options[:cookie] -> + Mix.Generator.create_file(path, cookie, quiet: true) + :ok + + File.exists?(path) -> + :ok + + true -> + File.write!(path, random_cookie()) + :ok + end + end + + defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32)) + + @doc """ + Makes the start_erl.data file with the + ERTS version and release versions. + """ + @spec make_start_erl(t, Path.t()) :: :ok + def make_start_erl(release, path) do + File.write!(path, "#{release.erts_version} #{release.version}") + :ok + end + + @doc """ + Makes boot scripts. + + It receives a path to the boot file, without extension, such as + `releases/0.1.0/start` and this command will write `start.rel`, + `start.boot`, and `start.script` to the given path, returning + `{:ok, rel_path}` or `{:error, message}`. + + The boot script uses the RELEASE_LIB environment variable, which must + be accordingly set with `--boot-var` and point to the release lib dir. + """ + @spec make_boot_script(t, Path.t(), [{application(), mode()}], [String.t()]) :: + :ok | {:error, String.t()} + def make_boot_script(release, path, modes, prepend_paths \\ []) do + with {:ok, rel_spec} <- build_release_spec(release, modes) do + File.write!(path <> ".rel", consultable(rel_spec)) + + sys_path = String.to_charlist(path) + + sys_options = [ + :silent, + :no_dot_erlang, + :no_warn_sasl, + variables: build_variables(release), + path: build_paths(release) + ] + + case :systools.make_script(sys_path, sys_options) do + {:ok, _module, _warnings} -> + script_path = sys_path ++ '.script' + {:ok, [{:script, rel_info, instructions}]} = :file.consult(script_path) + + instructions = + instructions + |> post_stdlib_applies(release) + |> prepend_paths_to_script(prepend_paths) + + script = {:script, rel_info, instructions} + File.write!(script_path, consultable(script)) + :ok = :systools.script2boot(sys_path) + + {:error, module, info} -> + message = module.format_error(info) |> to_string() |> String.trim() + {:error, message} + end + end + end + + defp build_variables(release) do + for {_, properties} <- release.applications, + not Keyword.fetch!(properties, :otp_app?), + uniq: true, + do: {'RELEASE_LIB', properties |> Keyword.fetch!(:path) |> :filename.dirname()} + end + + defp build_paths(release) do + for {_, properties} <- release.applications, + Keyword.fetch!(properties, :otp_app?), + do: properties |> Keyword.fetch!(:path) |> Path.join("ebin") |> to_charlist() + end + + defp build_release_spec(release, modes) do + %{ + name: name, + version: version, + erts_version: erts_version, + applications: apps, + options: options + } = release + + skip_mode_validation_for = + options + |> Keyword.get(:skip_mode_validation_for, []) + |> MapSet.new() + + rel_apps = + for {app, mode} <- modes do + properties = Map.get(apps, app) || throw({:error, "Unknown application #{inspect(app)}"}) + children = Keyword.get(properties, :applications, []) + app in skip_mode_validation_for || validate_mode!(app, mode, modes, children) + build_app_for_release(app, mode, properties) + end + + {:ok, {:release, {to_charlist(name), to_charlist(version)}, {:erts, erts_version}, rel_apps}} + catch + {:error, message} -> {:error, message} + end + + defp validate_mode!(app, mode, modes, children) do + safe_mode? = mode in @safe_modes + + if not safe_mode? and mode not in @unsafe_modes do + throw( + {:error, + "Unknown mode #{inspect(mode)} for #{inspect(app)}. " <> + "Valid modes are: #{inspect(@safe_modes ++ @unsafe_modes)}"} + ) + end + + for child <- children do + child_mode = Keyword.get(modes, child) + + cond do + is_nil(child_mode) -> + throw( + {:error, + "Application #{inspect(app)} is listed in the release boot, " <> + "but it depends on #{inspect(child)}, which isn't"} + ) + + safe_mode? and child_mode in @unsafe_modes -> + throw( + {:error, + """ + Application #{inspect(app)} has mode #{inspect(mode)} but it depends on \ + #{inspect(child)} which is set to #{inspect(child_mode)}. If you really want \ + to set such mode for #{inspect(child)} make sure that all applications that depend \ + on it are also set to :load or :none, otherwise your release will fail to boot + """} + ) + + true -> + :ok + end + end + end + + defp build_app_for_release(app, mode, properties) do + vsn = Keyword.fetch!(properties, :vsn) + + case Keyword.get(properties, :included_applications, []) do + [] -> {app, vsn, mode} + included_apps -> {app, vsn, mode, included_apps} + end + end + + defp post_stdlib_applies(instructions, release) do + {pre, [stdlib | post]} = + Enum.split_while( + instructions, + &(not match?({:apply, {:application, :start_boot, [:stdlib, _]}}, &1)) + ) + + pre ++ [stdlib] ++ config_provider_apply(release) ++ post + end + + defp config_provider_apply(%{config_providers: []}), + do: [] + + defp config_provider_apply(_), + do: [{:apply, {Config.Provider, :boot, []}}] + + defp prepend_paths_to_script(instructions, []), do: instructions + + defp prepend_paths_to_script(instructions, prepend_paths) do + prepend_paths = Enum.map(prepend_paths, &String.to_charlist/1) + + Enum.map(instructions, fn + {:path, paths} -> + if Enum.any?(paths, &List.starts_with?(&1, '$RELEASE_LIB')) do + {:path, prepend_paths ++ paths} + else + {:path, paths} + end + + other -> + other + end) + end + + defp consultable(term) do + IO.chardata_to_string(:io_lib.format("%% coding: utf-8~n~tp.~n", [term])) + end + + @doc """ + Finds a template path for the release. + """ + def rel_templates_path(release, path) do + Path.join(release.options[:rel_templates_path] || "rel", path) + end + + @doc """ + Copies ERTS if the release is configured to do so. + + Returns true if the release was copied, false otherwise. + """ + @spec copy_erts(t) :: boolean() + def copy_erts(%{erts_source: nil}) do + false + end + + def copy_erts(release) do + destination = Path.join(release.path, "erts-#{release.erts_version}/bin") + File.mkdir_p!(destination) + + release.erts_source + |> Path.join("bin") + |> File.cp_r!(destination, fn _, _ -> false end) + + _ = File.rm(Path.join(destination, "erl")) + _ = File.rm(Path.join(destination, "erl.ini")) + + destination + |> Path.join("erl") + |> File.write!(~S""" + #!/bin/sh + SELF=$(readlink "$0" || true) + if [ -z "$SELF" ]; then SELF="$0"; fi + BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" + ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" + EMU=beam + PROGNAME=$(echo "$0" | sed 's/.*\///') + export EMU + export ROOTDIR + export BINDIR + export PROGNAME + exec "$BINDIR/erlexec" ${1+"$@"} + """) + + File.chmod!(Path.join(destination, "erl"), 0o755) + true + end + + @doc """ + Copies the given application specification into the release. + + It assumes the application exists in the release. + """ + @spec copy_app(t, application) :: boolean() + def copy_app(release, app) do + properties = Map.fetch!(release.applications, app) + vsn = Keyword.fetch!(properties, :vsn) + + source_app = Keyword.fetch!(properties, :path) + target_app = Path.join([release.path, "lib", "#{app}-#{vsn}"]) + + if is_nil(release.erts_source) and Keyword.fetch!(properties, :otp_app?) do + false + else + File.rm_rf!(target_app) + File.mkdir_p!(target_app) + + copy_ebin(release, Path.join(source_app, "ebin"), Path.join(target_app, "ebin")) + + for dir <- @copy_app_dirs do + source_dir = Path.join(source_app, dir) + target_dir = Path.join(target_app, dir) + + source_dir = + case File.read_link(source_dir) do + {:ok, link_target} -> Path.expand(link_target, source_app) + _ -> source_dir + end + + File.exists?(source_dir) && File.cp_r!(source_dir, target_dir) + end + + true + end + end + + @doc """ + Copies the ebin directory at `source` to `target` + respecting release options such a `:strip_beams`. + """ + @spec copy_ebin(t, Path.t(), Path.t()) :: boolean() + def copy_ebin(release, source, target) do + with {:ok, [_ | _] = files} <- File.ls(source) do + File.mkdir_p!(target) + + strip_options = + release.options + |> Keyword.get(:strip_beams, true) + |> parse_strip_beams_options() + + for file <- files do + source_file = Path.join(source, file) + target_file = Path.join(target, file) + + with true <- is_list(strip_options) and String.ends_with?(file, ".beam"), + {:ok, binary} <- strip_beam(File.read!(source_file), strip_options) do + File.write!(target_file, binary) + else + _ -> + # Use File.cp!/3 to preserve file mode for any executables stored + # in the ebin directory. + File.cp!(source_file, target_file) + end + end + + true + else + _ -> false + end + end + + @doc """ + Strips a beam file for a release. + + This keeps only significant chunks necessary for the VM operation, + discarding documentation, debug info, compile information and others. + + The exact chunks that are kept are not documented and may change in + future versions. + """ + @spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()} + def strip_beam(binary, options \\ []) when is_list(options) do + chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1) + all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep) + + case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do + {:ok, {_, chunks}} -> + chunks = for {name, chunk} <- chunks, is_binary(chunk), do: {name, chunk} + {:ok, binary} = :beam_lib.build_module(chunks) + {:ok, :zlib.gzip(binary)} + + {:error, _, _} = error -> + error + end + end + + defp parse_strip_beams_options(options) do + case options do + options when is_list(options) -> options + true -> [] + false -> nil + end + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 000000000..29278cf40 --- /dev/null +++ b/mix.exs @@ -0,0 +1,206 @@ +defmodule EMQXUmbrella.MixProject do + use Mix.Project + + # Temporary hack while 1.13.2 is not released + System.version() + |> Version.parse!() + |> Version.compare(Version.parse!("1.13.2")) + |> Kernel.==(:lt) + |> if(do: Code.require_file("lib/mix/release.exs")) + + def project do + [ + apps_path: "apps", + version: pkg_vsn(), + deps: deps(), + releases: releases() + ] + end + + defp deps do + # we need several overrides here because dependencies specify + # other exact versions, and not ranges. + [ + {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, + {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, + {:gproc, "0.8.0", override: true}, + {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, + {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true}, + {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.11.1", override: true}, + {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.2.7"}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, + {:replayq, "0.3.3", override: true}, + {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3"}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, + {:observer_cli, "1.7.1"}, + {:system_monitor, github: "klarna-incubator/system_monitor", tag: "2.2.0"}, + # in conflict by emqtt and hocon + {:getopt, "1.0.2", override: true}, + {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.22.0"}, + {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1"}, + {:esasl, github: "emqx/esasl", tag: "0.2.0"}, + {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, + # in conflict by ehttpc and emqtt + {:gun, github: "emqx/gun", tag: "1.3.6", override: true}, + # in conflict by emqx_connectior and system_monitor + {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, + # in conflict by mongodb and eredis_cluster + {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, + # in conflict by gun and emqtt + {:cowlib, "2.8.0", override: true}, + # in conflict by cowboy_swagger and cowboy + {:ranch, "1.8.0", override: true}, + # in conflict by emqx and observer_cli + {:recon, github: "ferd/recon", tag: "2.5.1", override: true} + ] ++ bcrypt_dep() ++ quicer_dep() + end + + defp releases do + [ + emqx: [ + applications: [ + logger: :permanent, + esasl: :load, + crypto: :permanent, + public_key: :permanent, + asn1: :permanent, + syntax_tools: :permanent, + ssl: :permanent, + os_mon: :permanent, + inets: :permanent, + compiler: :permanent, + runtime_tools: :permanent, + hocon: :load, + emqx: :load, + emqx_conf: :load, + emqx_machine: :permanent, + mria: :load, + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_statsd: :permanent, + emqx_retainer: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent + ], + skip_mode_validation_for: [ + :emqx_gateway, + :emqx_dashboard, + :emqx_resource, + :emqx_connector, + :emqx_exhook, + :emqx_bridge, + :emqx_modules, + :emqx_management, + :emqx_statsd, + :emqx_retainer, + :emqx_prometheus, + :emqx_plugins + ], + steps: [:assemble, ©_files/1] + ] + ] + end + + def copy_files(release) do + etc = Path.join(release.path, "etc") + + # FIXME: Remove?? + File.mkdir_p!(etc) + File.cp!("apps/emqx_authz/etc/acl.conf", Path.join(etc, "acl.conf")) + + release + end + + def bcrypt_dep() do + if enable_bcrypt?(), + do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0"}], + else: [] + end + + def quicer_dep() do + if enable_quicer?(), + # in conflict with emqx and emqtt + do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}], + else: [] + end + + def enable_bcrypt?() do + not win32?() + end + + def enable_quicer?() do + not Enum.any?([ + build_without_quic?(), + win32?(), + centos6?() + ]) + end + + def project_path() do + Path.expand("..", __ENV__.file) + end + + def pkg_vsn() do + project_path() + |> Path.join("pkg-vsn.sh") + |> System.cmd([]) + |> elem(0) + |> String.trim() + |> String.split("-") + |> Enum.reverse() + |> tl() + |> Enum.reverse() + |> fix_vsn() + |> Enum.join("-") + end + + # FIXME: remove hack + defp fix_vsn([vsn | extras]) do + if Version.parse(vsn) == :error do + [vsn <> ".0" | extras] + else + [vsn | extras] + end + end + + defp win32?(), + do: match?({:win_32, _}, :os.type()) + + defp centos6?() do + case File.read("/etc/centos-release") do + {:ok, "CentOS release 6" <> _} -> + true + + _ -> + false + end + end + + defp build_without_quic?() do + opt = System.get_env("BUILD_WITHOUT_QUIC", "false") + + String.downcase(opt) != "false" + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 000000000..54812891f --- /dev/null +++ b/mix.lock @@ -0,0 +1,59 @@ +%{ + "bcrypt": {:git, "https://github.com/emqx/erlang-bcrypt.git", "dc2ba66acf2332c111362d01137746eefecc5e90", [tag: "0.6.0"]}, + "bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]}, + "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]}, + "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]}, + "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, + "earmark": {:hex, :earmark, "1.4.19", "3854a17305c880cc46305af15fb1630568d23a709aba21aaa996ced082fc29d7", [:mix], [{:earmark_parser, ">= 1.4.18", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "d5a8c9f9e37159a8fdd3ea8437fb4e229eaf56d5129b9a011dc4780a4872079d"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"}, + "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]}, + "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, + "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, + "ekka": {:git, "https://github.com/emqx/ekka.git", "005fd6bb94199dc2ecb4ba03284f253b408e02d9", [tag: "0.11.1"]}, + "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, + "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, + "epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]}, + "eredis": {:hex, :eredis, "1.2.0", "0b8e9cfc2c00fa1374cd107ea63b49be08d933df2cf175e6a89b73dd9c380de4", [:rebar3], [], "hexpm", "d9b5abef2c2c8aba8f32aa018203e0b3dc8b1157773b254ab1d4c2002317f1e1"}, + "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster.git", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, + "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, + "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]}, + "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]}, + "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, + "gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]}, + "getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"}, + "gpb": {:hex, :gpb, "4.19.2", "e841a1c5d5b82b6fbaddd2b6968c66ad48b34adc3b0d974e55cca653bd28ccc5", [:make, :rebar3], [], "hexpm", "3f4cb7f263e94675a598501a6dc96ec17f93ecb28c29bea3bea3a71c53a3317a"}, + "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"}, + "grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]}, + "gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]}, + "hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]}, + "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, + "jiffy": {:git, "https://github.com/emqx/jiffy.git", "baa1f4e750ae3c5c9e54f9c2e52280b7fc24a8d9", [tag: "1.0.5"]}, + "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, + "jsx": {:hex, :jsx, "2.9.0", "d2f6e5f069c00266cad52fb15d87c428579ea4d7d73a33669e12679e203329dd", [:mix, :rebar3], [], "hexpm", "8ee1db1cabafdd578a2776a6aaae87c2a8ce54b47b59e9ec7dab5d7eb71cd8dc"}, + "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, + "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, + "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, + "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, + "mongodb": {:git, "https://github.com/emqx/mongodb-erlang.git", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, + "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, + "mysql": {:git, "https://github.com/emqx/mysql-otp.git", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, + "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, + "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]}, + "poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]}, + "prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]}, + "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]}, + "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"}, + "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]}, + "rulesql": {:git, "https://github.com/emqx/rulesql.git", "fec11b1a3cbf98480d19c06d3aca10442e1e02a9", [tag: "0.1.4"]}, + "sext": {:hex, :sext, "1.8.0", "90a95b889f5c781b70bbcf44278b763148e313c376b60d87ce664cb1c1dd29b5", [:rebar3], [], "hexpm", "bc6016cb8690baf677eacacfe6e7cadfec8dc7e286cbbed762f6cd55b0678e73"}, + "snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]}, + "ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]}, + "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"}, + "system_monitor": {:git, "https://github.com/klarna-incubator/system_monitor.git", "a106f72e8e2251d90e1d95944c139946dac1a07f", [tag: "2.2.0"]}, + "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"}, + "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]}, +} diff --git a/mix_release.sh b/mix_release.sh new file mode 100755 index 000000000..321f0fbe1 --- /dev/null +++ b/mix_release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +mix release --overwrite + +## FIXME: temporary hacks to get the needed configurations for the +## release to properly start. + +## Assumes that `make emqx` has been run before the mix build to +## generate the correct configs. + +mkdir -p _build/dev/rel/emqx/data/configs/ +LATEST_APP_CONFIG=$(ls -rt _build/emqx/rel/emqx/data/configs/app*.config | tail -n 1) + +# FIXME! +cp ${LATEST_APP_CONFIG} _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#_build/emqx/rel/emqx/etc/emqx.conf#_build/dev/rel/emqx/etc/emqx.conf#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#logger_level,warning#logger_level,debug#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config +sed -i -E 's#level => warning#level => debug#g' _build/dev/rel/emqx/releases/5.0.0-beta.2/sys.config + + +# cp _build/emqx/rel/emqx/releases/emqx_vars _build/dev/rel/emqx/releases/ +cp _build/emqx/rel/emqx/etc/emqx.conf _build/dev/rel/emqx/etc/ + +echo "telemetry { enable = false }" >> _build/dev/rel/emqx/etc/emqx.conf diff --git a/rel/env.bat.eex b/rel/env.bat.eex new file mode 100644 index 000000000..60beb808a --- /dev/null +++ b/rel/env.bat.eex @@ -0,0 +1,5 @@ +@echo off +rem Set the release to work across nodes. +rem RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +rem set RELEASE_DISTRIBUTION=name +rem set RELEASE_NODE=<%= @release.name %> diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 000000000..70cfce36a --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,17 @@ +#!/bin/sh + +# Sets and enables heart (recommended only in daemon mode) +# case $RELEASE_COMMAND in +# daemon*) +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# ;; +# *) +# ;; +# esac + +# Set the release to work across nodes. +# RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +# export RELEASE_DISTRIBUTION=name +# export RELEASE_NODE=<%= @release.name %> diff --git a/rel/remote.vm.args.eex b/rel/remote.vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/remote.vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/rel/vm.args.eex b/rel/vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10