diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 000000000..fbe8732d5 --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,7 @@ +[ + inputs: [ + "mix.exs", + "config/*.exs", + "scripts/*.exs", + ] +] diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml new file mode 100644 index 000000000..04bc8c024 --- /dev/null +++ b/.github/workflows/elixir_apps_check.yaml @@ -0,0 +1,48 @@ +--- + +name: Check Elixir Release Applications + +on: [pull_request] + +jobs: + elixir_apps_check: + runs-on: ubuntu-20.04 + container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + + strategy: + fail-fast: false + matrix: + release_type: + - cloud + - edge + package_type: + - bin + - pkg + edition_type: + - community + - enterprise + exclude: + - release_type: edge + package_type: bin + edition_type: enterprise + - release_type: edge + package_type: pkg + edition_type: enterprise + + steps: + - name: install + run: apk add make bash curl git + - name: Checkout + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - name: ensure rebar + run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1 + - name: check applications + run: ./scripts/check-elixir-applications.exs + env: + EMQX_RELEASE_TYPE: ${{ matrix.release_type }} + EMQX_PACKAGE_TYPE: ${{ matrix.package_type }} + EMQX_EDITION_TYPE: ${{ matrix.edition_type }} + +... diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml new file mode 100644 index 000000000..be637072f --- /dev/null +++ b/.github/workflows/elixir_deps_check.yaml @@ -0,0 +1,22 @@ +--- + +name: Elixir Dependency Version Check + +on: [pull_request] + +jobs: + elixir_deps_check: + runs-on: ubuntu-20.04 + container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + - name: install + run: apk add make bash curl git + - name: ensure rebar + run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1 + - name: check elixir deps + run: ./scripts/check-elixir-deps-discrepancies.exs + +... diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml new file mode 100644 index 000000000..dc887efbc --- /dev/null +++ b/.github/workflows/elixir_release.yml @@ -0,0 +1,38 @@ +# FIXME: temporary workflow for testing; remove later +name: Elixir Build (temporary) + +concurrency: + group: mix-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + - name: setup mix + run: | + mix local.hex --force + mix local.rebar --force + mix deps.get + - name: produce emqx.conf.all template + run: make conf-segs + - name: elixir release + run: mix release --overwrite + - name: start release + run: | + cd _build/dev/rel/emqx + bin/emqx start + - name: check if started + run: | + sleep 10 + nc -zv localhost 1883 + cd _build/dev/rel/emqx + bin/emqx ping + bin/emqx ctl status diff --git a/.tool-versions b/.tool-versions index a6568713b..1c687075b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ erlang 24.1.5-3 +elixir 1.13.1-otp-24 diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 1424eb8a5..09a1fdfb7 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -24,4 +24,8 @@ %% NOTE: This version number should be manually bumped for each release --define(EMQX_RELEASE, "5.0-beta.2"). +%% NOTE: This version number should have 3 numeric parts +%% (Major.Minor.Patch), and extra info can be added after a final +%% hyphen. + +-define(EMQX_RELEASE, "5.0.0-beta.2"). diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index f3ea37b67..0989cd7f1 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -5,7 +5,19 @@ {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, - {applications, [kernel,stdlib,gproc,gen_rpc,mria,esockd,cowboy,sasl,os_mon,jiffy,lc]}, + {applications, [ kernel + , stdlib + , gproc + , gen_rpc + , mria + , esockd + , cowboy + , sasl + , os_mon + , jiffy + , lc + , hocon + ]}, {mod, {emqx_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 73696b033..3def1d00e 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index ba38cc642..b1479d434 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -1,5 +1,5 @@ {erl_opts, [debug_info, nowarn_unused_import]}. -{deps, []}. +{deps, [{emqx, {path, "../emqx"}}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 88793f7ba..91cdd39d9 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 3fd6b41e0..d24d23f8c 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_conf/rebar.config b/apps/emqx_conf/rebar.config index e0456112b..35563d233 100644 --- a/apps/emqx_conf/rebar.config +++ b/apps/emqx_conf/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 563c9dc1d..e9fbbb4b9 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -3,7 +3,6 @@ {vsn, "0.1.0"}, {registered, []}, {mod, {emqx_conf_app, []}}, - {included_applications, [hocon]}, {applications, [kernel, stdlib]}, {env, []}, {modules, []} diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 58706e950..f919fc943 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -4,6 +4,7 @@ ]}. {deps, [ + {emqx, {path, "../emqx"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index fe8bb6c97..d83d16764 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -12,6 +12,7 @@ eredis_cluster, eredis, epgsql, + eldap2, mysql, mongodb, ehttpc, diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index d0a1fbde4..618fc203d 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,4 +1,6 @@ -{deps, []}. +{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} + , {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index afdaad084..8229c3a55 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} + [ {emqx, {path, "../emqx"}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 44f74eacf..c94a84de9 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. {deps, [ + {emqx, {path, "../emqx"}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 0efb8043e..d183319d3 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -18,7 +18,7 @@ -behaviour(minirest_api). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). %% API -export([api_spec/0]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 60f3d4837..db7f7d3c6 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -43,7 +43,7 @@ -export_type([channel/0]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index a567ea7f4..9ee190198 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -28,8 +28,8 @@ , is_message/1 ]). --include("include/emqx_coap.hrl"). --include("apps/emqx/include/types.hrl"). +-include("src/coap/include/emqx_coap.hrl"). +-include_lib("emqx/include/types.hrl"). -define(VERSION, 1). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl index 020e38496..e5c659320 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). %% API -export([ empty/0, reset/1, reset/2 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index 3851b3428..315612bbc 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -34,7 +34,7 @@ -export([ set/3, set_payload/2, get_option/2 , get_option/3, set_payload_block/3, set_payload_block/4]). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index 9c3a8c451..1c58a6d1b 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -18,7 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). %% API -export([ new/0 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index d6d05fd40..cb74ea2ab 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -28,7 +28,7 @@ -export_type([manager/0, event_result/1]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 07e522309..d3db897ae 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -1,7 +1,7 @@ -module(emqx_coap_transport). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 47bf14d9b..80cd7a77f 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index 608eae92a..beb5c8e13 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index c01c6adc5..12007f13b 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_channel). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API -export([ info/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 1ee27228b..88f3ed2e3 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -17,8 +17,8 @@ -module(emqx_lwm2m_cmd). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ mqtt_to_coap/2 , coap_to_mqtt/4 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index bee1bedcd..4c17546d2 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -18,8 +18,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API -export([ new/0, init/4, update/3, parse_object_list/1 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index d744a23f9..3359dc4e8 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 509971b15..9bf02a6ba 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 5e342ec76..7d0ac571d 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index dd8fc2f7f..14cc372f2 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 0782ab1b3..280a54398 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl index 335e00b20..baf4703e3 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). +-include("src/mqttsn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- @@ -181,4 +181,3 @@ gen_next(0, Acc) -> gen_next(N, Acc) -> Byte = rand:uniform(256) - 1, gen_next(N-1, <>). - diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index c637f44c2..49740e4d4 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). +-include("src/mqttsn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl index 6dd6860b7..dae4ab909 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -17,7 +17,7 @@ -module(emqx_stomp_SUITE). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_gateway/src/stomp/include/emqx_stomp.hrl"). +-include("src/stomp/include/emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -419,7 +419,7 @@ t_rest_clienit_info(_) -> %% %% TODO: Start/Stop, List Instace %% -%% TODO: RateLimit, OOM, +%% TODO: RateLimit, OOM, with_connection(DoFun) -> {ok, Sock} = gen_tcp:connect({127, 0, 0, 1}, diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index 1191d748e..08da2e511 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl index 95cfc6d20..2038e68e1 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl @@ -16,7 +16,7 @@ -module(emqx_sn_proper_types). --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). +-include("src/mqttsn/include/emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl index 9e12a7bd4..b8485edbd 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include_lib("src/mqttsn/include/emqx_sn.hrl"). +-include("src/mqttsn/include/emqx_sn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). @@ -76,4 +76,3 @@ mqtt_sn_message() -> , M:'WILLTOPICRESP'(), M:'WILLMSGUPD'() , M:'WILLMSGRESP'() ]). - diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_machine/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 24541990b..2084d3a05 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -135,6 +135,12 @@ add_app(G, App, undefined) -> ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), %% not loaded add_app(G, App, []); +% We ALWAYS want to add `emqx_conf', even if no other app declare a +% dependency on it. Otherwise, emqx may fail to load the config +% schemas, especially in the test profile. +add_app(G, App = emqx_conf, []) -> + digraph:add_vertex(G, App), + ok; add_app(_G, _App, []) -> ok; add_app(G, App, [Dep | Deps]) -> diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 03d9e6ba9..a760d2f5f 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -43,7 +43,7 @@ init_per_suite(Config) -> %% application:unload(emqx_authz), - emqx_common_test_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([emqx_conf]), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index e5e336e34..0cc5ace2a 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 7b30a8fd8..07646091a 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -1 +1,2 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_plugins/rebar.config b/apps/emqx_plugins/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_plugins/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 02e81b295..8b7e0c67e 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -1,5 +1,8 @@ {deps, - [{prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v3.1.1"}}} + [ %% FIXME: tag this as v3.1.3 + {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_psk/rebar.config b/apps/emqx_psk/rebar.config index 73696b033..3def1d00e 100644 --- a/apps/emqx_psk/rebar.config +++ b/apps/emqx_psk/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 1b93aa0de..9591c5718 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -8,7 +8,6 @@ [kernel, stdlib, gproc, - hocon, jsx, emqx ]}, diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index b49f979ac..e5b53f1be 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -19,6 +20,6 @@ [{test, [{deps, [ - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.0"}}}]} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}]} ]} ]}. diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 097c18a3d..56a64b436 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,4 +1,5 @@ -{deps, []}. +{deps, [ {emqx, {path, "../emqx"}} + ]}. {erl_opts, [warn_unused_vars, warn_shadow_vars, diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config new file mode 100644 index 000000000..07646091a --- /dev/null +++ b/apps/emqx_slow_subs/rebar.config @@ -0,0 +1,2 @@ +{deps, [ {emqx, {path, "../emqx"}} + ]}. diff --git a/bin/emqx b/bin/emqx index 9e7f710a1..bde1f0b51 100755 --- a/bin/emqx +++ b/bin/emqx @@ -66,7 +66,13 @@ assert_node_alive() { echoerr() { echo "$*" 1>&2; } check_erlang_start() { - "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s erlang halt + # RELEASE_LIB is used by Elixir + "$BINDIR/$PROGNAME" \ + -noshell \ + -boot_var RELEASE_LIB "$ERTS_LIB_DIR/lib" \ + -boot "$REL_DIR/start_clean" \ + -s crypto start \ + -s erlang halt } usage() { @@ -289,9 +295,24 @@ relx_rem_sh() { # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace # Setup remote shell command to control node - exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot "$REL_DIR/start_clean" \ - -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ - -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" $EPMD_ARG + if [ "$IS_ELIXIR" = "yes" ] + then + exec "$REL_DIR/iex" \ + --remsh "$NAME" \ + --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ + --cookie "$COOKIE" \ + --hidden \ + --erl "-kernel net_ticktime $TICKTIME" \ + --erl "$EPMD_ARG" \ + --erl "$NAME_TYPE $id" \ + --boot "$REL_DIR/start_clean" + else + exec "$BINDIR/erl" "$NAME_TYPE" "$id" \ + -remsh "$NAME" -boot "$REL_DIR/start_clean" \ + -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ + -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" \ + $EPMD_ARG + fi } # Generate a random id @@ -353,6 +374,11 @@ generate_config() { local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config" local HOCON_GEN_ARG_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args" + # This is needed by the Elixir scripts. + # Do NOT append `.config`. + RELEASE_SYS_CONFIG="$CONFIGS_DIR/app.$NOW_TIME" + export RELEASE_SYS_CONFIG + CONFIG_ARGS="-config $CONF_FILE -args_file $HOCON_GEN_ARG_FILE" ## Merge hocon generated *.args into the vm.args @@ -688,11 +714,23 @@ case "${COMMAND}" in # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace # Build an array of arguments to pass to exec later on # Build it here because this command will be used for logging. - set -- "$BINDIR/erlexec" \ - -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \ - -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ - -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ - $CONFIG_ARGS $EPMD_ARG + if [ "$IS_ELIXIR" = yes ] + then + set -- "$REL_DIR/iex" \ + --boot "$BOOTFILE" \ + --erl "-mode $CODE_LOADING_MODE" \ + --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ + --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \ + --erl "$CONFIG_ARGS" \ + --erl "$EPMD_ARG" \ + --werl + else + set -- "$BINDIR/erlexec" \ + -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \ + -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ + -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ + $CONFIG_ARGS $EPMD_ARG + fi # Log the startup logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}" @@ -727,11 +765,25 @@ case "${COMMAND}" in # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace # Build an array of arguments to pass to exec later on # Build it here because this command will be used for logging. - set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \ - -boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \ - -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ - -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ - $CONFIG_ARGS $EPMD_ARG + if [ "$IS_ELIXIR" = yes ] + then + set -- "$REL_DIR/elixir" \ + --boot "$REL_DIR/start" \ + --erl "$FOREGROUNDOPTIONS" \ + --erl "-mode $CODE_LOADING_MODE" \ + --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ + --boot-var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ + --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \ + --erl "$CONFIG_ARGS" \ + --erl "$EPMD_ARG" \ + --no-halt + else + set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \ + -boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \ + -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ + -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ + $CONFIG_ARGS $EPMD_ARG + fi # Log the startup logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}" @@ -773,7 +825,18 @@ case "${COMMAND}" in assert_node_alive shift - relx_nodetool "eval" "$@" + if [ "$IS_ELIXIR" = "yes" ] + then + "$REL_DIR/elixir" \ + --hidden \ + --cookie "$COOKIE" \ + --boot "$REL_DIR/start_clean" \ + --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ + --vm-args "$(latest_vm_args 'EMQX_NODE__NAME')"\ + --rpc-eval "$NAME" "$@" + else + relx_nodetool "eval" "$@" + fi ;; *) usage "$COMMAND" diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 501ba5487..74b6da6e2 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -304,8 +304,8 @@ permafy(TargetNode, RelName, Vsn) -> make_permanent, [Vsn], ?TIMEOUT), ?INFO("Made release permanent: ~p", [Vsn]), %% upgrade/downgrade the scripts by replacing them - Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool", - "install_upgrade.escript"], + Scripts = [RelNameStr, RelNameStr ++ "_ctl", + "nodetool", "install_upgrade.escript"], [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]), filename:join(["bin", File])) || File <- Scripts], diff --git a/bin/node_dump b/bin/node_dump index c8f76841b..3640fada9 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -1,10 +1,11 @@ #!/bin/sh set -eu +# shellcheck disable=SC1090,SC1091 ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" echo "Running node dump in ${ROOT_DIR}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 . "$ROOT_DIR"/releases/emqx_vars cd "${ROOT_DIR}" diff --git a/data/BUILT_ON b/data/BUILT_ON index 2997223fa..2dfa56a16 100644 --- a/data/BUILT_ON +++ b/data/BUILT_ON @@ -1 +1 @@ -{{built_on_arch}} +{{ built_on_arch }} diff --git a/data/emqx_vars b/data/emqx_vars index 453bd618f..6687ca881 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -13,6 +13,7 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" +IS_ELIXIR="{{ is_elixir }}" EMQX_LICENSE_CONF='' export EMQX_DESCRIPTION='{{ emqx_description }}' diff --git a/lib/mix/release.exs b/lib/mix/release.exs new file mode 100644 index 000000000..d6c032877 --- /dev/null +++ b/lib/mix/release.exs @@ -0,0 +1,878 @@ +defmodule Mix.Release do + @moduledoc """ + Defines the release structure and convenience for assembling releases. + """ + + @doc """ + The Mix.Release struct has the following read-only fields: + + * `:name` - the name of the release as an atom + * `:version` - the version of the release as a string or + `{:from_app, app_name}` + * `:path` - the path to the release root + * `:version_path` - the path to the release version inside the release + * `:applications` - a map of application with their definitions + * `:erts_source` - the ERTS source as a charlist (or nil) + * `:erts_version` - the ERTS version as a charlist + + The following fields may be modified as long as they keep their defined types: + + * `:boot_scripts` - a map of boot scripts with the boot script name + as key and a keyword list with **all** applications that are part of + it and their modes as value + * `:config_providers` - a list of `{config_provider, term}` tuples where the + first element is a module that implements the `Config.Provider` behaviour + and `term` is the value given to it on `c:Config.Provider.init/1` + * `:options` - a keyword list with all other user supplied release options + * `:overlays` - a list of extra files added to the release. If you have a custom + step adding extra files to a release, you can add these files to the `:overlays` + field so they are also considered on further commands, such as tar/zip. Each entry + in overlays is the relative path to the release root of each file + * `:steps` - a list of functions that receive the release and returns a release. + Must also contain the atom `:assemble` which is the internal assembling step. + May also contain the atom `:tar` to create a tarball of the release. + + """ + defstruct [ + :name, + :version, + :path, + :version_path, + :applications, + :boot_scripts, + :erts_source, + :erts_version, + :config_providers, + :options, + :overlays, + :steps + ] + + @type mode :: :permanent | :transient | :temporary | :load | :none + @type application :: atom() + @type t :: %__MODULE__{ + name: atom(), + version: String.t(), + path: String.t(), + version_path: String.t() | {:from_app, application()}, + applications: %{application() => keyword()}, + boot_scripts: %{atom() => [{application(), mode()}]}, + erts_version: charlist(), + erts_source: charlist() | nil, + config_providers: [{module, term}], + options: keyword(), + overlays: list(String.t()), + steps: [(t -> t) | :assemble, ...] + } + + @default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent] + @safe_modes [:permanent, :temporary, :transient] + @unsafe_modes [:load, :none] + @significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c + @copy_app_dirs ["priv"] + + @doc false + @spec from_config!(atom, keyword, keyword) :: t + def from_config!(name, config, overrides) do + {name, apps, opts} = find_release(name, config) + + unless Atom.to_string(name) =~ ~r/^[a-z][a-z0-9_]*$/ do + Mix.raise( + "Invalid release name. A release name must start with a lowercase ASCII letter, " <> + "followed by lowercase ASCII letters, numbers, or underscores, got: #{inspect(name)}" + ) + end + + opts = + [overwrite: false, quiet: false, strip_beams: true] + |> Keyword.merge(opts) + |> Keyword.merge(overrides) + + {include_erts, opts} = Keyword.pop(opts, :include_erts, true) + {erts_source, erts_lib_dir, erts_version} = erts_data(include_erts) + + deps_apps = Mix.Project.deps_apps() + loaded_apps = apps |> Keyword.keys() |> load_apps(deps_apps, %{}, erts_lib_dir, [], :root) + + # Make sure IEx is either an active part of the release or add it as none. + {loaded_apps, apps} = + if Map.has_key?(loaded_apps, :iex) do + {loaded_apps, apps} + else + {load_apps([:iex], deps_apps, loaded_apps, erts_lib_dir, [], :root), apps ++ [iex: :none]} + end + + start_boot = build_start_boot(loaded_apps, apps) + start_clean_boot = build_start_clean_boot(start_boot) + + {path, opts} = + Keyword.pop_lazy(opts, :path, fn -> + Path.join([Mix.Project.build_path(config), "rel", Atom.to_string(name)]) + end) + + path = Path.absname(path) + + {version, opts} = + Keyword.pop_lazy(opts, :version, fn -> + config[:version] || + Mix.raise( + "No :version found. Please make sure a :version is set in your project definition " <> + "or inside the release the configuration" + ) + end) + + version = + case version do + {:from_app, app} -> + Application.load(app) + version = Application.spec(app, :vsn) + + if !version do + Mix.raise( + "Could not find version for #{inspect(app)}, please make sure the application exists" + ) + end + + to_string(version) + + "" -> + Mix.raise("The release :version cannot be an empty string") + + _ -> + version + end + + {config_providers, opts} = Keyword.pop(opts, :config_providers, []) + {steps, opts} = Keyword.pop(opts, :steps, [:assemble]) + validate_steps!(steps) + + %Mix.Release{ + name: name, + version: version, + path: path, + version_path: Path.join([path, "releases", version]), + erts_source: erts_source, + erts_version: erts_version, + applications: loaded_apps, + boot_scripts: %{start: start_boot, start_clean: start_clean_boot}, + config_providers: config_providers, + options: opts, + overlays: [], + steps: steps + } + end + + defp find_release(name, config) do + {name, opts_fun_or_list} = lookup_release(name, config) || infer_release(config) + opts = if is_function(opts_fun_or_list, 0), do: opts_fun_or_list.(), else: opts_fun_or_list + {apps, opts} = Keyword.pop(opts, :applications, []) + + if apps == [] and Mix.Project.umbrella?(config) do + bad_umbrella!() + end + + app = Keyword.get(config, :app) + apps = Keyword.merge(@default_apps, apps) + + if is_nil(app) or Keyword.has_key?(apps, app) do + {name, apps, opts} + else + {name, apps ++ [{app, :permanent}], opts} + end + end + + defp lookup_release(nil, config) do + case Keyword.get(config, :releases, []) do + [] -> + nil + + [{name, opts}] -> + {name, opts} + + [_ | _] -> + case Keyword.get(config, :default_release) do + nil -> + Mix.raise( + "\"mix release\" was invoked without a name but there are multiple releases. " <> + "Please call \"mix release NAME\" or set :default_release in your project configuration" + ) + + name -> + lookup_release(name, config) + end + end + end + + defp lookup_release(name, config) do + if opts = config[:releases][name] do + {name, opts} + else + found = Keyword.get(config, :releases, []) + + Mix.raise( + "Unknown release #{inspect(name)}. " <> + "The available releases are: #{inspect(Keyword.keys(found))}" + ) + end + end + + defp infer_release(config) do + if Mix.Project.umbrella?(config) do + bad_umbrella!() + else + {Keyword.fetch!(config, :app), []} + end + end + + defp bad_umbrella! do + Mix.raise(""" + Umbrella projects require releases to be explicitly defined with \ + a non-empty applications key that chooses which umbrella children \ + should be part of the releases: + + releases: [ + foo: [ + applications: [child_app_foo: :permanent] + ], + bar: [ + applications: [child_app_bar: :permanent] + ] + ] + + Alternatively you can perform the release from the children applications + """) + end + + defp erts_data(erts_data) when is_function(erts_data) do + erts_data(erts_data.()) + end + + defp erts_data(false) do + {nil, :code.lib_dir(), :erlang.system_info(:version)} + end + + defp erts_data(true) do + version = :erlang.system_info(:version) + {:filename.join(:code.root_dir(), 'erts-#{version}'), :code.lib_dir(), version} + end + + defp erts_data(erts_source) when is_binary(erts_source) do + if File.exists?(erts_source) do + [_, erts_version] = erts_source |> Path.basename() |> String.split("-") + erts_lib_dir = erts_source |> Path.dirname() |> Path.join("lib") |> to_charlist() + {to_charlist(erts_source), erts_lib_dir, to_charlist(erts_version)} + else + Mix.raise("Could not find ERTS system at #{inspect(erts_source)}") + end + end + + defp load_apps(apps, deps_apps, seen, otp_root, optional, type) do + for app <- apps, reduce: seen do + seen -> + if reentrant_seen = reentrant(seen, app, type) do + reentrant_seen + else + load_app(app, deps_apps, seen, otp_root, optional, type) + end + end + end + + defp reentrant(seen, app, type) do + properties = seen[app] + + cond do + is_nil(properties) -> + nil + + type != :root and properties[:type] != type -> + if properties[:type] == :root do + put_in(seen[app][:type], type) + else + Mix.raise( + "#{inspect(app)} is listed both as a regular application and as an included application" + ) + end + + true -> + seen + end + end + + defp load_app(app, deps_apps, seen, otp_root, optional, type) do + cond do + path = app not in deps_apps && otp_path(otp_root, app) -> + do_load_app(app, path, deps_apps, seen, otp_root, true, type) + + path = code_path(app) -> + do_load_app(app, path, deps_apps, seen, otp_root, false, type) + + app in optional -> + seen + + true -> + Mix.raise("Could not find application #{inspect(app)}") + end + end + + defp otp_path(otp_root, app) do + path = Path.join(otp_root, "#{app}-*") + + case Path.wildcard(path) do + [] -> nil + paths -> paths |> Enum.sort() |> List.last() |> to_charlist() + end + end + + defp code_path(app) do + case :code.lib_dir(app) do + {:error, :bad_name} -> nil + path -> path + end + end + + defp do_load_app(app, path, deps_apps, seen, otp_root, otp_app?, type) do + case :file.consult(Path.join(path, "ebin/#{app}.app")) do + {:ok, terms} -> + [{:application, ^app, properties}] = terms + value = [path: path, otp_app?: otp_app?, type: type] ++ properties + seen = Map.put(seen, app, value) + applications = Keyword.get(properties, :applications, []) + optional = Keyword.get(properties, :optional_applications, []) + seen = load_apps(applications, deps_apps, seen, otp_root, optional, :depended) + included_applications = Keyword.get(properties, :included_applications, []) + load_apps(included_applications, deps_apps, seen, otp_root, [], :included) + + {:error, reason} -> + Mix.raise("Could not load #{app}.app. Reason: #{inspect(reason)}") + end + end + + defp build_start_boot(all_apps, specified_apps) do + specified_apps ++ + Enum.sort( + for( + {app, props} <- all_apps, + not List.keymember?(specified_apps, app, 0), + do: {app, default_mode(props)} + ) + ) + end + + defp default_mode(props) do + if props[:type] == :included, do: :load, else: :permanent + end + + defp build_start_clean_boot(boot) do + for({app, _mode} <- boot, do: {app, :none}) + |> Keyword.put(:stdlib, :permanent) + |> Keyword.put(:kernel, :permanent) + end + + defp validate_steps!(steps) do + valid_atoms = [:assemble, :tar] + + if not is_list(steps) or Enum.any?(steps, &(&1 not in valid_atoms and not is_function(&1, 1))) do + Mix.raise(""" + The :steps option must be a list of: + + * anonymous function that receives one argument + * the atom :assemble or :tar + + Got: #{inspect(steps)} + """) + end + + if Enum.count(steps, &(&1 == :assemble)) != 1 do + Mix.raise("The :steps option must contain the atom :assemble once, got: #{inspect(steps)}") + end + + if :assemble in Enum.drop_while(steps, &(&1 != :tar)) do + Mix.raise("The :tar step must come after :assemble") + end + + if Enum.count(steps, &(&1 == :tar)) > 1 do + Mix.raise("The :steps option can only contain the atom :tar once") + end + + :ok + end + + @doc """ + Makes the `sys.config` structure. + + If there are config providers, then a value is injected into + the `:elixir` application configuration in `sys_config` to be + read during boot and trigger the providers. + + It uses the following release options to customize its behaviour: + + * `:reboot_system_after_config` + * `:start_distribution_during_config` + * `:prune_runtime_sys_config_after_boot` + + In case there are no config providers, it doesn't change `sys_config`. + """ + @spec make_sys_config(t, keyword(), Config.Provider.config_path()) :: + :ok | {:error, String.t()} + def make_sys_config(release, sys_config, config_provider_path) do + {sys_config, runtime_config?} = + merge_provider_config(release, sys_config, config_provider_path) + + path = Path.join(release.version_path, "sys.config") + + args = [runtime_config?, sys_config] + format = "%% coding: utf-8~n%% RUNTIME_CONFIG=~s~n~tw.~n" + File.mkdir_p!(Path.dirname(path)) + File.write!(path, IO.chardata_to_string(:io_lib.format(format, args))) + + case :file.consult(path) do + {:ok, _} -> + :ok + + {:error, reason} -> + invalid = + for {app, kv} <- sys_config, + {key, value} <- kv, + not valid_config?(value), + do: """ + + Application: #{inspect(app)} + Key: #{inspect(key)} + Value: #{inspect(value)} + """ + + message = + case invalid do + [] -> + "Could not read configuration file. Reason: #{inspect(reason)}" + + _ -> + "Could not read configuration file. It has invalid configuration terms " <> + "such as functions, references, and pids. Please make sure your configuration " <> + "is made of numbers, atoms, strings, maps, tuples and lists. The following entries " <> + "are wrong:\n#{Enum.join(invalid)}" + end + + {:error, message} + end + end + + defp valid_config?(m) when is_map(m), + do: Enum.all?(Map.delete(m, :__struct__), &valid_config?/1) + + defp valid_config?(l) when is_list(l), do: Enum.all?(l, &valid_config?/1) + defp valid_config?(t) when is_tuple(t), do: Enum.all?(Tuple.to_list(t), &valid_config?/1) + defp valid_config?(o), do: is_number(o) or is_atom(o) or is_binary(o) + + defp merge_provider_config(%{config_providers: []}, sys_config, _), do: {sys_config, false} + + defp merge_provider_config(release, sys_config, config_path) do + {reboot?, extra_config, initial_config} = start_distribution(release) + + prune_runtime_sys_config_after_boot = + Keyword.get(release.options, :prune_runtime_sys_config_after_boot, false) + + opts = [ + extra_config: initial_config, + prune_runtime_sys_config_after_boot: prune_runtime_sys_config_after_boot, + reboot_system_after_config: reboot?, + validate_compile_env: validate_compile_env(release) + ] + + init_config = Config.Provider.init(release.config_providers, config_path, opts) + {Config.Reader.merge(sys_config, init_config ++ extra_config), reboot?} + end + + defp validate_compile_env(release) do + with true <- Keyword.get(release.options, :validate_compile_env, true), + [_ | _] = compile_env <- compile_env(release) do + compile_env + else + _ -> false + end + end + + defp compile_env(release) do + for {_, properties} <- release.applications, + triplet <- Keyword.get(properties, :compile_env, []), + do: triplet + end + + defp start_distribution(%{options: opts}) do + reboot? = Keyword.get(opts, :reboot_system_after_config, false) + early_distribution? = Keyword.get(opts, :start_distribution_during_config, false) + + if not reboot? or early_distribution? do + {reboot?, [], []} + else + {true, [kernel: [start_distribution: false]], [kernel: [start_distribution: true]]} + end + end + + @doc """ + Copies the cookie to the given path. + + If a cookie option was given, we compare it with + the contents of the file (if any), and ask the user + if they want to override. + + If there is no option, we generate a random one + the first time. + """ + @spec make_cookie(t, Path.t()) :: :ok + def make_cookie(release, path) do + cond do + cookie = release.options[:cookie] -> + Mix.Generator.create_file(path, cookie, quiet: true) + :ok + + File.exists?(path) -> + :ok + + true -> + File.write!(path, random_cookie()) + :ok + end + end + + defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32)) + + @doc """ + Makes the start_erl.data file with the + ERTS version and release versions. + """ + @spec make_start_erl(t, Path.t()) :: :ok + def make_start_erl(release, path) do + File.write!(path, "#{release.erts_version} #{release.version}") + :ok + end + + @doc """ + Makes boot scripts. + + It receives a path to the boot file, without extension, such as + `releases/0.1.0/start` and this command will write `start.rel`, + `start.boot`, and `start.script` to the given path, returning + `{:ok, rel_path}` or `{:error, message}`. + + The boot script uses the RELEASE_LIB environment variable, which must + be accordingly set with `--boot-var` and point to the release lib dir. + """ + @spec make_boot_script(t, Path.t(), [{application(), mode()}], [String.t()]) :: + :ok | {:error, String.t()} + def make_boot_script(release, path, modes, prepend_paths \\ []) do + with {:ok, rel_spec} <- build_release_spec(release, modes) do + File.write!(path <> ".rel", consultable(rel_spec)) + + sys_path = String.to_charlist(path) + + sys_options = [ + :silent, + :no_dot_erlang, + :no_warn_sasl, + variables: build_variables(release), + path: build_paths(release) + ] + + case :systools.make_script(sys_path, sys_options) do + {:ok, _module, _warnings} -> + script_path = sys_path ++ '.script' + {:ok, [{:script, rel_info, instructions}]} = :file.consult(script_path) + + instructions = + instructions + |> post_stdlib_applies(release) + |> prepend_paths_to_script(prepend_paths) + + script = {:script, rel_info, instructions} + File.write!(script_path, consultable(script)) + :ok = :systools.script2boot(sys_path) + + {:error, module, info} -> + message = module.format_error(info) |> to_string() |> String.trim() + {:error, message} + end + end + end + + defp build_variables(release) do + for {_, properties} <- release.applications, + not Keyword.fetch!(properties, :otp_app?), + uniq: true, + do: {'RELEASE_LIB', properties |> Keyword.fetch!(:path) |> :filename.dirname()} + end + + defp build_paths(release) do + for {_, properties} <- release.applications, + Keyword.fetch!(properties, :otp_app?), + do: properties |> Keyword.fetch!(:path) |> Path.join("ebin") |> to_charlist() + end + + defp build_release_spec(release, modes) do + %{ + name: name, + version: version, + erts_version: erts_version, + applications: apps, + options: options + } = release + + skip_mode_validation_for = + options + |> Keyword.get(:skip_mode_validation_for, []) + |> MapSet.new() + + rel_apps = + for {app, mode} <- modes do + properties = Map.get(apps, app) || throw({:error, "Unknown application #{inspect(app)}"}) + children = Keyword.get(properties, :applications, []) + app in skip_mode_validation_for || validate_mode!(app, mode, modes, children) + build_app_for_release(app, mode, properties) + end + + {:ok, {:release, {to_charlist(name), to_charlist(version)}, {:erts, erts_version}, rel_apps}} + catch + {:error, message} -> {:error, message} + end + + defp validate_mode!(app, mode, modes, children) do + safe_mode? = mode in @safe_modes + + if not safe_mode? and mode not in @unsafe_modes do + throw( + {:error, + "Unknown mode #{inspect(mode)} for #{inspect(app)}. " <> + "Valid modes are: #{inspect(@safe_modes ++ @unsafe_modes)}"} + ) + end + + for child <- children do + child_mode = Keyword.get(modes, child) + + cond do + is_nil(child_mode) -> + throw( + {:error, + "Application #{inspect(app)} is listed in the release boot, " <> + "but it depends on #{inspect(child)}, which isn't"} + ) + + safe_mode? and child_mode in @unsafe_modes -> + throw( + {:error, + """ + Application #{inspect(app)} has mode #{inspect(mode)} but it depends on \ + #{inspect(child)} which is set to #{inspect(child_mode)}. If you really want \ + to set such mode for #{inspect(child)} make sure that all applications that depend \ + on it are also set to :load or :none, otherwise your release will fail to boot + """} + ) + + true -> + :ok + end + end + end + + defp build_app_for_release(app, mode, properties) do + vsn = Keyword.fetch!(properties, :vsn) + + case Keyword.get(properties, :included_applications, []) do + [] -> {app, vsn, mode} + included_apps -> {app, vsn, mode, included_apps} + end + end + + defp post_stdlib_applies(instructions, release) do + {pre, [stdlib | post]} = + Enum.split_while( + instructions, + &(not match?({:apply, {:application, :start_boot, [:stdlib, _]}}, &1)) + ) + + pre ++ [stdlib] ++ config_provider_apply(release) ++ post + end + + defp config_provider_apply(%{config_providers: []}), + do: [] + + defp config_provider_apply(_), + do: [{:apply, {Config.Provider, :boot, []}}] + + defp prepend_paths_to_script(instructions, []), do: instructions + + defp prepend_paths_to_script(instructions, prepend_paths) do + prepend_paths = Enum.map(prepend_paths, &String.to_charlist/1) + + Enum.map(instructions, fn + {:path, paths} -> + if Enum.any?(paths, &List.starts_with?(&1, '$RELEASE_LIB')) do + {:path, prepend_paths ++ paths} + else + {:path, paths} + end + + other -> + other + end) + end + + defp consultable(term) do + IO.chardata_to_string(:io_lib.format("%% coding: utf-8~n~tp.~n", [term])) + end + + @doc """ + Finds a template path for the release. + """ + def rel_templates_path(release, path) do + Path.join(release.options[:rel_templates_path] || "rel", path) + end + + @doc """ + Copies ERTS if the release is configured to do so. + + Returns true if the release was copied, false otherwise. + """ + @spec copy_erts(t) :: boolean() + def copy_erts(%{erts_source: nil}) do + false + end + + def copy_erts(release) do + destination = Path.join(release.path, "erts-#{release.erts_version}/bin") + File.mkdir_p!(destination) + + release.erts_source + |> Path.join("bin") + |> File.cp_r!(destination, fn _, _ -> false end) + + _ = File.rm(Path.join(destination, "erl")) + _ = File.rm(Path.join(destination, "erl.ini")) + + destination + |> Path.join("erl") + |> File.write!(~S""" + #!/bin/sh + SELF=$(readlink "$0" || true) + if [ -z "$SELF" ]; then SELF="$0"; fi + BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" + ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" + EMU=beam + PROGNAME=$(echo "$0" | sed 's/.*\///') + export EMU + export ROOTDIR + export BINDIR + export PROGNAME + exec "$BINDIR/erlexec" ${1+"$@"} + """) + + File.chmod!(Path.join(destination, "erl"), 0o755) + true + end + + @doc """ + Copies the given application specification into the release. + + It assumes the application exists in the release. + """ + @spec copy_app(t, application) :: boolean() + def copy_app(release, app) do + properties = Map.fetch!(release.applications, app) + vsn = Keyword.fetch!(properties, :vsn) + + source_app = Keyword.fetch!(properties, :path) + target_app = Path.join([release.path, "lib", "#{app}-#{vsn}"]) + + if is_nil(release.erts_source) and Keyword.fetch!(properties, :otp_app?) do + false + else + File.rm_rf!(target_app) + File.mkdir_p!(target_app) + + copy_ebin(release, Path.join(source_app, "ebin"), Path.join(target_app, "ebin")) + + for dir <- @copy_app_dirs do + source_dir = Path.join(source_app, dir) + target_dir = Path.join(target_app, dir) + + source_dir = + case File.read_link(source_dir) do + {:ok, link_target} -> Path.expand(link_target, source_app) + _ -> source_dir + end + + File.exists?(source_dir) && File.cp_r!(source_dir, target_dir) + end + + true + end + end + + @doc """ + Copies the ebin directory at `source` to `target` + respecting release options such a `:strip_beams`. + """ + @spec copy_ebin(t, Path.t(), Path.t()) :: boolean() + def copy_ebin(release, source, target) do + with {:ok, [_ | _] = files} <- File.ls(source) do + File.mkdir_p!(target) + + strip_options = + release.options + |> Keyword.get(:strip_beams, true) + |> parse_strip_beams_options() + + for file <- files do + source_file = Path.join(source, file) + target_file = Path.join(target, file) + + with true <- is_list(strip_options) and String.ends_with?(file, ".beam"), + {:ok, binary} <- strip_beam(File.read!(source_file), strip_options) do + File.write!(target_file, binary) + else + _ -> + # Use File.cp!/3 to preserve file mode for any executables stored + # in the ebin directory. + File.cp!(source_file, target_file) + end + end + + true + else + _ -> false + end + end + + @doc """ + Strips a beam file for a release. + + This keeps only significant chunks necessary for the VM operation, + discarding documentation, debug info, compile information and others. + + The exact chunks that are kept are not documented and may change in + future versions. + """ + @spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()} + def strip_beam(binary, options \\ []) when is_list(options) do + chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1) + all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep) + + case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do + {:ok, {_, chunks}} -> + chunks = for {name, chunk} <- chunks, is_binary(chunk), do: {name, chunk} + {:ok, binary} = :beam_lib.build_module(chunks) + {:ok, :zlib.gzip(binary)} + + {:error, _, _} = error -> + error + end + end + + defp parse_strip_beams_options(options) do + case options do + options when is_list(options) -> options + true -> [] + false -> nil + end + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 000000000..a6936c85c --- /dev/null +++ b/mix.exs @@ -0,0 +1,602 @@ +defmodule EMQXUmbrella.MixProject do + use Mix.Project + + @moduledoc """ + + The purpose of this file is to configure the release of EMQX under + Mix. Since EMQX uses its own configuration conventions and startup + procedures, one cannot simply use `iex -S mix`. Instead, it's + recommendd to build and use the release. + + ## Release Environment Variables + + The release build is controlled by a few environment variables. + + * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz` + tarball along with the release. + * `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`. Controls a + few dependencies and the `vm.args` to be used. Defaults to + `cloud`. + * `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`. Controls + whether the build is intended for direct usage or for packaging. + Defaults to `bin`. + * `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`. + Defaults to `community`. + """ + + # Temporary hack while 1.13.2 is not released + System.version() + |> Version.parse!() + |> Version.compare(Version.parse!("1.13.2")) + |> Kernel.==(:lt) + |> if(do: Code.require_file("lib/mix/release.exs")) + + def project() do + [ + app: :emqx_mix, + version: pkg_vsn(), + deps: deps(), + releases: releases() + ] + end + + defp deps() do + # we need several overrides here because dependencies specify + # other exact versions, and not ranges. + [ + {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, + {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, + {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, + {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, + {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true}, + {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.11.2", override: true}, + {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, + {:replayq, "0.3.3", override: true}, + {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, + {:observer_cli, "1.7.1"}, + {:system_monitor, github: "k32/system_monitor", tag: "2.2.1"}, + # in conflict by emqtt and hocon + {:getopt, "1.0.2", override: true}, + {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.22.0", override: true}, + {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true}, + {:esasl, github: "emqx/esasl", tag: "0.2.0"}, + {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, + # in conflict by ehttpc and emqtt + {:gun, github: "emqx/gun", tag: "1.3.6", override: true}, + # in conflict by emqx_connectior and system_monitor + {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, + # in conflict by mongodb and eredis_cluster + {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, + # in conflict by emqx and observer_cli + {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, + {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, + # dependencies of dependencies; we choose specific refs to match + # what rebar3 chooses. + # in conflict by gun and emqtt + {:cowlib, + github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true}, + # in conflict by cowboy_swagger and cowboy + {:ranch, + github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, + # in conflict by grpc and eetcd + {:gpb, "4.11.2", override: true} + ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep() + end + + defp umbrella_apps() do + "apps/*" + |> Path.wildcard() + |> Enum.map(fn path -> + app = + path + |> String.trim_leading("apps/") + |> String.to_atom() + + {app, path: path, manager: :rebar3, override: true} + end) + end + + defp releases() do + [ + emqx: fn -> + %{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + } = read_inputs() + + base_steps = [ + :assemble, + &create_RELEASES/1, + ©_files(&1, release_type, package_type, edition_type), + ©_escript(&1, "nodetool"), + ©_escript(&1, "install_upgrade.escript") + ] + + steps = + if System.get_env("ELIXIR_MAKE_TAR") == "yes" do + base_steps ++ [:tar] + else + base_steps + end + + [ + applications: applications(release_type), + skip_mode_validation_for: [ + :emqx_gateway, + :emqx_dashboard, + :emqx_resource, + :emqx_connector, + :emqx_exhook, + :emqx_bridge, + :emqx_modules, + :emqx_management, + :emqx_statsd, + :emqx_retainer, + :emqx_prometheus, + :emqx_plugins + ], + steps: steps, + strip_beams: false + ] + end + ] + end + + def applications(release_type) do + [ + logger: :permanent, + crypto: :permanent, + public_key: :permanent, + asn1: :permanent, + syntax_tools: :permanent, + ssl: :permanent, + os_mon: :permanent, + inets: :permanent, + compiler: :permanent, + runtime_tools: :permanent, + hocon: :load, + emqx: :load, + emqx_conf: :load, + emqx_machine: :permanent, + mria: :load, + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + esasl: :load, + observer_cli: :permanent, + system_monitor: :permanent, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_retainer: :permanent, + emqx_statsd: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent, + emqx_mix: :none + ] ++ + if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ + if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ + if(release_type == :cloud, + do: [xmerl: :permanent, observer: :load], + else: [] + ) + end + + defp read_inputs() do + release_type = + read_enum_env_var( + "EMQX_RELEASE_TYPE", + [:cloud, :edge], + :cloud + ) + + package_type = + read_enum_env_var( + "EMQX_PACKAGE_TYPE", + [:bin, :pkg], + :bin + ) + + edition_type = + read_enum_env_var( + "EMQX_EDITION_TYPE", + [:community, :enterprise], + :community + ) + + %{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + } + end + + defp copy_files(release, release_type, package_type, edition_type) do + overwrite? = Keyword.get(release.options, :overwrite, false) + + bin = Path.join(release.path, "bin") + etc = Path.join(release.path, "etc") + + Mix.Generator.create_directory(bin) + Mix.Generator.create_directory(etc) + Mix.Generator.create_directory(Path.join(etc, "certs")) + + Mix.Generator.copy_file( + "apps/emqx_authz/etc/acl.conf", + Path.join(etc, "acl.conf"), + force: overwrite? + ) + + # required by emqx_authz + File.cp_r!( + "apps/emqx/etc/certs", + Path.join(etc, "certs") + ) + + # this is required by the produced escript / nodetool + Mix.Generator.copy_file( + Path.join(release.version_path, "start_clean.boot"), + Path.join(bin, "no_dot_erlang.boot"), + force: overwrite? + ) + + assigns = template_vars(release, release_type, package_type, edition_type) + + # This is generated by `scripts/merge-config.escript` or `make + # conf-segs`. So, this should be run before the release. + # TODO: run as a "compiler" step??? + conf_rendered = + File.read!("apps/emqx_conf/etc/emqx.conf.all") + |> from_rebar_to_eex_template() + |> EEx.eval_string(assigns) + + File.write!( + Path.join(etc, "emqx.conf"), + conf_rendered + ) + + vars_rendered = + File.read!("data/emqx_vars") + |> from_rebar_to_eex_template() + |> EEx.eval_string(assigns) + + File.write!( + Path.join([release.path, "releases", "emqx_vars"]), + vars_rendered + ) + + vm_args_template_path = + case release_type do + :cloud -> + "apps/emqx/etc/emqx_cloud/vm.args" + + :edge -> + "apps/emqx/etc/emqx_edge/vm.args" + end + + vm_args_rendered = + File.read!(vm_args_template_path) + |> from_rebar_to_eex_template() + |> EEx.eval_string(assigns) + + File.write!( + Path.join(etc, "vm.args"), + vm_args_rendered + ) + + File.write!( + Path.join(release.version_path, "vm.args"), + vm_args_rendered + ) + + for name <- [ + "emqx", + "emqx_ctl" + ] do + Mix.Generator.copy_file( + "bin/#{name}", + Path.join(bin, name), + force: overwrite? + ) + + # Files with the version appended are expected by the release + # upgrade script `install_upgrade.escript` + Mix.Generator.copy_file( + Path.join(bin, name), + Path.join(bin, name <> "-#{release.version}"), + force: overwrite? + ) + end + + for base_name <- ["emqx", "emqx_ctl"], + suffix <- ["", "-#{release.version}"] do + name = base_name <> suffix + File.chmod!(Path.join(bin, name), 0o755) + end + + built_on_rendered = + File.read!("data/BUILT_ON") + |> from_rebar_to_eex_template() + |> EEx.eval_string(assigns) + + File.write!( + Path.join([release.version_path, "BUILT_ON"]), + built_on_rendered + ) + + release + end + + # needed by nodetool and by release_handler + defp create_RELEASES(release) do + apps = + Enum.map(release.applications, fn {app_name, app_props} -> + app_vsn = Keyword.fetch!(app_props, :vsn) + + app_path = + "./lib" + |> Path.join("#{app_name}-#{app_vsn}") + |> to_charlist() + + {app_name, app_vsn, app_path} + end) + + release_entry = [ + { + :release, + to_charlist(release.name), + to_charlist(release.version), + release.erts_version, + apps, + :permanent + } + ] + + release.path + |> Path.join("releases") + |> Path.join("RELEASES") + |> File.open!([:write, :utf8], fn handle -> + IO.puts(handle, "%% coding: utf-8") + :io.format(handle, '~tp.~n', [release_entry]) + end) + + release + end + + defp copy_escript(release, escript_name) do + [shebang, rest] = + "bin/#{escript_name}" + |> File.read!() + |> String.split("\n", parts: 2) + + # the elixir version of escript + start.boot required the boot_var + # RELEASE_LIB to be defined. + boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib" + + # Files with the version appended are expected by the release + # upgrade script `install_upgrade.escript` + Enum.each( + [escript_name, escript_name <> "-" <> release.version], + fn name -> + path = Path.join([release.path, "bin", name]) + File.write!(path, [shebang, "\n", boot_var, "\n", rest]) + end + ) + + release + end + + defp template_vars(release, release_type, :bin = _package_type, edition_type) do + [ + platform_bin_dir: "bin", + platform_data_dir: "data", + platform_etc_dir: "etc", + platform_lib_dir: "lib", + platform_log_dir: "log", + platform_plugins_dir: "plugins", + runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)", + runner_bin_dir: "$RUNNER_ROOT_DIR/bin", + runner_etc_dir: "$RUNNER_ROOT_DIR/etc", + runner_lib_dir: "$RUNNER_ROOT_DIR/lib", + runner_log_dir: "$RUNNER_ROOT_DIR/log", + runner_data_dir: "$RUNNER_ROOT_DIR/data", + runner_user: "", + release_version: release.version, + erts_vsn: release.erts_version, + # FIXME: this is empty in `make emqx` ??? + erl_opts: "", + emqx_description: emqx_description(release_type, edition_type), + built_on_arch: built_on(), + is_elixir: "yes" + ] + end + + defp template_vars(release, release_type, :pkg = _package_type, edition_type) do + [ + platform_bin_dir: "", + platform_data_dir: "/var/lib/emqx", + platform_etc_dir: "/etc/emqx", + platform_lib_dir: "", + platform_log_dir: "/var/log/emqx", + platform_plugins_dir: "/var/lib/emqx/plugins", + runner_root_dir: "/usr/lib/emqx", + runner_bin_dir: "/usr/bin", + runner_etc_dir: "/etc/emqx", + runner_lib_dir: "$RUNNER_ROOT_DIR/lib", + runner_log_dir: "/var/log/emqx", + runner_data_dir: "/var/lib/emqx", + runner_user: "emqx", + release_version: release.version, + erts_vsn: release.erts_version, + # FIXME: this is empty in `make emqx` ??? + erl_opts: "", + emqx_description: emqx_description(release_type, edition_type), + built_on: built_on(), + is_elixir: "yes" + ] + end + + defp read_enum_env_var(env_var, allowed_values, default_value) do + case System.fetch_env(env_var) do + :error -> + default_value + + {:ok, raw_value} -> + value = + raw_value + |> String.downcase() + |> String.to_atom() + + if value not in allowed_values do + Mix.raise(""" + Invalid value #{raw_value} for variable #{env_var}. + Allowed values are: #{inspect(allowed_values)} + """) + end + + value + end + end + + defp emqx_description(release_type, edition_type) do + case {release_type, edition_type} do + {:cloud, :enterprise} -> + "EMQ X Enterprise Edition" + + {:cloud, :community} -> + "EMQ X Community Edition" + + {:edge, :community} -> + "EMQ X Edge Edition" + end + end + + defp bcrypt_dep() do + if enable_bcrypt?(), + do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}], + else: [] + end + + defp quicer_dep() do + if enable_quicer?(), + # in conflict with emqx and emqtt + do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}], + else: [] + end + + defp enable_bcrypt?() do + not win32?() + end + + defp enable_quicer?() do + not Enum.any?([ + build_without_quic?(), + win32?(), + centos6?() + ]) + end + + defp pkg_vsn() do + basedir = Path.dirname(__ENV__.file) + script = Path.join(basedir, "pkg-vsn.sh") + {str_vsn, 0} = System.cmd(script, []) + + String.trim(str_vsn) + end + + defp win32?(), + do: match?({:win_32, _}, :os.type()) + + defp centos6?() do + case File.read("/etc/centos-release") do + {:ok, "CentOS release 6" <> _} -> + true + + _ -> + false + end + end + + defp build_without_quic?() do + opt = System.get_env("BUILD_WITHOUT_QUIC", "false") + + String.downcase(opt) != "false" + end + + defp from_rebar_to_eex_template(str) do + # we must not consider surrounding space in the template var name + # because some help strings contain informative variables that + # should not be interpolated, and those have no spaces. + Regex.replace( + ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/, + str, + "<%= \\g{1} %>" + ) + end + + defp built_on() do + system_architecture = to_string(:erlang.system_info(:system_architecture)) + elixir_version = System.version() + words = wordsize() + + "#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}" + end + + # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142 + defp wordsize() do + size = + try do + :erlang.system_info({:wordsize, :external}) + rescue + ErlangError -> + :erlang.system_info(:wordsize) + end + + to_string(8 * size) + end + + # As from Erlang/OTP 17, the OTP release number corresponds to the + # major OTP version number. No erlang:system_info() argument gives + # the exact OTP version. + # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release + # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577 + defp otp_release() do + major_version = System.otp_release() + root_dir = to_string(:code.root_dir()) + + [root_dir, "releases", major_version, "OTP_VERSION"] + |> Path.join() + |> File.read() + |> case do + {:error, _} -> + major_version + + {:ok, version} -> + version + |> String.trim() + |> String.split("**") + |> List.first() + end + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 000000000..c619b1036 --- /dev/null +++ b/mix.lock @@ -0,0 +1,54 @@ +%{ + "bcrypt": {:git, "https://github.com/emqx/erlang-bcrypt.git", "dc2ba66acf2332c111362d01137746eefecc5e90", [tag: "0.6.0"]}, + "bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]}, + "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]}, + "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]}, + "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363"]}, + "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]}, + "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, + "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, + "ekka": {:git, "https://github.com/emqx/ekka.git", "70f2250e5e968e0c1da64e5b4733c5eb0eb402de", [tag: "0.11.2"]}, + "eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]}, + "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, + "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, + "epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]}, + "eredis": {:git, "https://github.com/emqx/eredis", "75f2b8eedbe631136326680225efbcd2684e93e7", [tag: "1.2.5"]}, + "eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]}, + "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, + "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]}, + "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]}, + "gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]}, + "getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"}, + "gpb": {:hex, :gpb, "4.11.2", "a2c05241408310b8bd8dbdfd5e1a419799e45fc1408371eaa0f595023c3b21aa", [:make, :rebar], [], "hexpm", "b355a5982b604d6c044ebb6013e5fe65d30c30d0700d35317e7066691eb6bd61"}, + "gproc": {:git, "https://github.com/uwiger/gproc.git", "ce7397809aca0d6eb3aac6db65953752e47fb511", [tag: "0.8.0"]}, + "grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]}, + "gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]}, + "hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]}, + "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, + "jiffy": {:git, "https://github.com/emqx/jiffy.git", "baa1f4e750ae3c5c9e54f9c2e52280b7fc24a8d9", [tag: "1.0.5"]}, + "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, + "jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]}, + "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, + "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, + "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, + "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, + "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, + "mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, + "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, + "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]}, + "poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]}, + "prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]}, + "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]}, + "ranch": {:git, "https://github.com/ninenines/ranch.git", "a692f44567034dacf5efcaa24a24183788594eb7", [ref: "a692f44567034dacf5efcaa24a24183788594eb7"]}, + "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]}, + "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"}, + "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]}, + "rulesql": {:git, "https://github.com/emqx/rulesql.git", "fec11b1a3cbf98480d19c06d3aca10442e1e02a9", [tag: "0.1.4"]}, + "sext": {:hex, :sext, "1.8.0", "90a95b889f5c781b70bbcf44278b763148e313c376b60d87ce664cb1c1dd29b5", [:rebar3], [], "hexpm", "bc6016cb8690baf677eacacfe6e7cadfec8dc7e286cbbed762f6cd55b0678e73"}, + "snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]}, + "ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]}, + "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"}, + "system_monitor": {:git, "https://github.com/k32/system_monitor.git", "3b4b381bf9503695cd764ecf22067ac6542cee89", [tag: "2.2.1"]}, + "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"}, + "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]}, +} diff --git a/rebar.config.erl b/rebar.config.erl index dbaf9d6a3..982f5c3b9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -219,6 +219,7 @@ overlay_vars_pkg(bin) -> , {runner_log_dir, "$RUNNER_ROOT_DIR/log"} , {runner_data_dir, "$RUNNER_ROOT_DIR/data"} , {runner_user, ""} + , {is_elixir, "no"} ]; overlay_vars_pkg(pkg) -> [ {platform_bin_dir, ""} @@ -234,6 +235,7 @@ overlay_vars_pkg(pkg) -> , {runner_log_dir, "/var/log/emqx"} , {runner_data_dir, "/var/lib/emqx"} , {runner_user, "emqx"} + , {is_elixir, "no"} ]. relx_apps(ReleaseType, Edition) -> @@ -248,6 +250,7 @@ relx_apps(ReleaseType, Edition) -> , inets , compiler , runtime_tools + , {hocon, load} , {emqx, load} % started by emqx_machine , {emqx_conf, load} , emqx_machine diff --git a/rel/env.bat.eex b/rel/env.bat.eex new file mode 100644 index 000000000..60beb808a --- /dev/null +++ b/rel/env.bat.eex @@ -0,0 +1,5 @@ +@echo off +rem Set the release to work across nodes. +rem RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +rem set RELEASE_DISTRIBUTION=name +rem set RELEASE_NODE=<%= @release.name %> diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 000000000..70cfce36a --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,17 @@ +#!/bin/sh + +# Sets and enables heart (recommended only in daemon mode) +# case $RELEASE_COMMAND in +# daemon*) +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# ;; +# *) +# ;; +# esac + +# Set the release to work across nodes. +# RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none". +# export RELEASE_DISTRIBUTION=name +# export RELEASE_NODE=<%= @release.name %> diff --git a/rel/remote.vm.args.eex b/rel/remote.vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/remote.vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/rel/vm.args.eex b/rel/vm.args.eex new file mode 100644 index 000000000..5886aa87e --- /dev/null +++ b/rel/vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: https://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, and others) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/scripts/check-elixir-applications.exs b/scripts/check-elixir-applications.exs new file mode 100755 index 000000000..3ada6d779 --- /dev/null +++ b/scripts/check-elixir-applications.exs @@ -0,0 +1,282 @@ +#!/usr/bin/env elixir + +defmodule CheckElixirApplications do + @default_applications [:kernel, :stdlib, :sasl] + + def main() do + {:ok, _} = Application.ensure_all_started(:mix) + inputs = read_inputs() + # produce `rebar.config.rendered` to consult + profile = profile_of(inputs) + + File.cwd!() + |> Path.join("rebar3") + |> System.cmd(["as", to_string(profile)], + env: [{"DEBUG", "1"}] + ) + + File.cwd!() + |> Path.join("mix.exs") + |> Code.compile_file() + + mix_apps = mix_applications(inputs.release_type) + rebar_apps = rebar_applications(profile) + results = diff_apps(mix_apps, rebar_apps) + + report_discrepancy( + results[:missing_apps], + "* There are missing applications in the Elixir release", + fn %{app: app, mode: mode, after: last_app} -> + IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}") + end + ) + + report_discrepancy( + results[:different_modes], + "* There are applications with different application modes in the Elixir release", + fn %{app: app, rebar_mode: rebar_mode, mix_mode: mix_mode} -> + IO.puts( + " * #{inspect(app)} should have mode #{inspect(rebar_mode)}, but it has mode #{inspect(mix_mode)}" + ) + end + ) + + report_discrepancy( + results[:different_positions], + "* There are applications in the Elixir release in the wrong order", + fn %{app: app, mode: mode, after: last_app} -> + IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}") + end + ) + + success? = + results + |> Map.take([:missing_apps, :different_modes, :different_positions]) + |> Map.values() + |> Enum.concat() + |> Enum.empty?() + + if not success? do + System.halt(1) + else + IO.puts( + IO.ANSI.green() <> + "Mix and Rebar applications OK!" <> + IO.ANSI.reset() + ) + end + end + + defp mix_applications(release_type) do + EMQXUmbrella.MixProject.applications(release_type) + end + + defp rebar_applications(profile) do + {:ok, props} = + File.cwd!() + |> Path.join("rebar.config.rendered") + |> :file.consult() + + props[:profiles][profile][:relx] + |> Enum.find(&(elem(&1, 0) == :release)) + |> elem(2) + |> Enum.map(fn + app when is_atom(app) -> + {app, :permanent} + + {app, mode} -> + {app, mode} + end) + |> Enum.reject(fn {app, _mode} -> + # Elixir already includes those implicitly + app in @default_applications + end) + end + + defp profile_of(%{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + }) do + case {release_type, package_type, edition_type} do + {:cloud, :bin, :community} -> + :emqx + + {:cloud, :pkg, :community} -> + :"emqx-pkg" + + {:cloud, :bin, :enterprise} -> + :"emqx-enterprise" + + {:cloud, :pkg, :enterprise} -> + :"emqx-enterprise-pkg" + + {:edge, :bin, :community} -> + :"emqx-edge" + + {:edge, :pkg, :community} -> + :"emqx-edge-pkg" + end + end + + defp read_inputs() do + release_type = + read_enum_env_var( + "EMQX_RELEASE_TYPE", + [:cloud, :edge], + :cloud + ) + + package_type = + read_enum_env_var( + "EMQX_PACKAGE_TYPE", + [:bin, :pkg], + :bin + ) + + edition_type = + read_enum_env_var( + "EMQX_EDITION_TYPE", + [:community, :enterprise], + :community + ) + + %{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + } + end + + defp read_enum_env_var(env_var, allowed_values, default_value) do + case System.fetch_env(env_var) do + :error -> + default_value + + {:ok, raw_value} -> + value = + raw_value + |> String.downcase() + |> String.to_atom() + + if value not in allowed_values do + Mix.raise(""" + Invalid value #{raw_value} for variable #{env_var}. + Allowed values are: #{inspect(allowed_values)} + """) + end + + value + end + end + + defp diff_apps(mix_apps, rebar_apps) do + app_names = Keyword.keys(rebar_apps) + mix_apps = Keyword.filter(mix_apps, fn {app, _mode} -> app in app_names end) + + acc = %{ + mix_apps: mix_apps, + missing_apps: [], + different_positions: [], + different_modes: [], + last_app: nil + } + + Enum.reduce( + rebar_apps, + acc, + fn + {rebar_app, rebar_mode}, acc = %{mix_apps: [], last_app: last_app} -> + missing_app = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:missing_apps, &[missing_app | &1]) + |> Map.put(:last_app, rebar_app) + + {rebar_app, rebar_mode}, + acc = %{mix_apps: [{mix_app, mix_mode} | rest], last_app: last_app} -> + case {rebar_app, rebar_mode} do + {^mix_app, ^mix_mode} -> + acc + |> Map.put(:mix_apps, rest) + |> Map.put(:last_app, rebar_app) + + {^mix_app, _mode} -> + different_mode = %{ + app: rebar_app, + rebar_mode: rebar_mode, + mix_mode: mix_mode + } + + acc + |> Map.put(:mix_apps, rest) + |> Map.update!(:different_modes, &[different_mode | &1]) + |> Map.put(:last_app, rebar_app) + + {_app, _mode} -> + case Keyword.pop(rest, rebar_app) do + {nil, _} -> + missing_app = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:missing_apps, &[missing_app | &1]) + |> Map.put(:last_app, rebar_app) + + {^rebar_mode, rest} -> + different_position = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:different_positions, &[different_position | &1]) + |> Map.put(:last_app, rebar_app) + |> Map.put(:mix_apps, [{mix_app, mix_mode} | rest]) + + {mode, rest} -> + different_mode = %{ + app: rebar_app, + rebar_mode: rebar_mode, + mix_mode: mode + } + + different_position = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.put(:mix_apps, [{mix_app, mix_mode} | rest]) + |> Map.update!(:different_modes, &[different_mode | &1]) + |> Map.update!(:different_positions, &[different_position | &1]) + |> Map.put(:last_app, rebar_app) + end + end + end + ) + end + + defp report_discrepancy(diffs, header, line_fn) do + unless Enum.empty?(diffs) do + IO.puts(IO.ANSI.red() <> header) + + diffs + |> Enum.reverse() + |> Enum.each(line_fn) + + IO.puts(IO.ANSI.reset()) + end + end +end + +CheckElixirApplications.main() diff --git a/scripts/check-elixir-deps-discrepancies.exs b/scripts/check-elixir-deps-discrepancies.exs new file mode 100755 index 000000000..21a2d18ef --- /dev/null +++ b/scripts/check-elixir-deps-discrepancies.exs @@ -0,0 +1,108 @@ +#!/usr/bin/env elixir + +# ensure we have a fresh rebar.lock + +case File.stat("rebar.lock") do + {:ok, _} -> + File.rm!("rebar.lock") + + _ -> + :ok +end + +{_, 0} = + File.cwd!() + |> Path.join("rebar3") + |> System.cmd(["tree"], into: IO.stream()) + +{:ok, props} = :file.consult("rebar.lock") + +{[{_, rebar_deps}], [props]} = Enum.split_with(props, &is_tuple/1) + +# dpendencies declared as package versions have a "secondary index" +pkg_idx = + props + |> Keyword.fetch!(:pkg_hash) + |> Map.new() + +rebar_deps = + Map.new(rebar_deps, fn {name, ref, _} -> + ref = + case ref do + {:pkg, _, _} -> + pkg_idx + |> Map.fetch!(name) + |> String.downcase() + + {:git, _, {:ref, ref}} -> + to_string(ref) + end + + {name, ref} + end) + +{mix_deps, []} = Code.eval_file("mix.lock") + +mix_deps = + Map.new(mix_deps, fn {name, ref} -> + ref = + case ref do + {:git, _, ref, _} -> + ref + + {:hex, _, _, ref, _, _, _, _} -> + ref + end + + {to_string(name), ref} + end) + +diffs = + Enum.reduce(rebar_deps, %{}, fn {name, rebar_ref}, acc -> + mix_ref = mix_deps[name] + + cond do + mix_ref && mix_ref != rebar_ref -> + Map.put(acc, name, {rebar_ref, mix_ref}) + + is_nil(mix_ref) -> + Map.put(acc, name, {rebar_ref, nil}) + + :otherwise -> + acc + end + end) + +if diffs == %{} do + IO.puts( + IO.ANSI.green() <> + "* Mix and Rebar3 dependencies OK!" <> + IO.ANSI.reset() + ) + + System.halt(0) +else + IO.puts( + IO.ANSI.red() <> + "* Discrepancies between Elixir and Rebar3 dependencies found!" <> + IO.ANSI.reset() + ) + + Enum.each(diffs, fn {name, {rebar_ref, mix_ref}} -> + IO.puts( + IO.ANSI.red() <> + " * #{name}\n" <> + " * Rebar3 ref: #{rebar_ref}\n" <> + " * Mix ref: #{mix_ref}\n" <> + IO.ANSI.reset() + ) + end) + + IO.puts( + IO.ANSI.red() <> + "Update `mix.exs` to match Rebar3's references (use `overwrite: true` if necessary) and try again" <> + IO.ANSI.reset() + ) + + System.halt(1) +end diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh index 5f8cdfd51..c00a87caa 100755 --- a/scripts/shellcheck.sh +++ b/scripts/shellcheck.sh @@ -3,7 +3,10 @@ set -euo pipefail target_files=() -while IFS='' read -r line; do target_files+=("$line"); done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .) +while IFS='' read -r line; +do + target_files+=("$line"); +done < <(git grep -r -l "^#!/bin/" .) return_code=0 for i in "${target_files[@]}"; do echo checking "$i" ...