Merge pull request #6507 from emqx/build-with-mix-mkII

Build with Elixir Mix Release
This commit is contained in:
Thales Macedo Garitezi 2021-12-30 15:00:38 -03:00 committed by GitHub
commit c29bc126ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
70 changed files with 2270 additions and 73 deletions

7
.formatter.exs Normal file
View File

@ -0,0 +1,7 @@
[
inputs: [
"mix.exs",
"config/*.exs",
"scripts/*.exs",
]
]

View File

@ -0,0 +1,48 @@
---
name: Check Elixir Release Applications
on: [pull_request]
jobs:
elixir_apps_check:
runs-on: ubuntu-20.04
container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0
strategy:
fail-fast: false
matrix:
release_type:
- cloud
- edge
package_type:
- bin
- pkg
edition_type:
- community
- enterprise
exclude:
- release_type: edge
package_type: bin
edition_type: enterprise
- release_type: edge
package_type: pkg
edition_type: enterprise
steps:
- name: install
run: apk add make bash curl git
- name: Checkout
uses: actions/checkout@v2.4.0
with:
fetch-depth: 0
- name: ensure rebar
run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1
- name: check applications
run: ./scripts/check-elixir-applications.exs
env:
EMQX_RELEASE_TYPE: ${{ matrix.release_type }}
EMQX_PACKAGE_TYPE: ${{ matrix.package_type }}
EMQX_EDITION_TYPE: ${{ matrix.edition_type }}
...

View File

@ -0,0 +1,22 @@
---
name: Elixir Dependency Version Check
on: [pull_request]
jobs:
elixir_deps_check:
runs-on: ubuntu-20.04
container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0
steps:
- name: Checkout
uses: actions/checkout@v2.4.0
- name: install
run: apk add make bash curl git
- name: ensure rebar
run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1
- name: check elixir deps
run: ./scripts/check-elixir-deps-discrepancies.exs
...

38
.github/workflows/elixir_release.yml vendored Normal file
View File

@ -0,0 +1,38 @@
# FIXME: temporary workflow for testing; remove later
name: Elixir Build (temporary)
concurrency:
group: mix-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
jobs:
build:
runs-on: ubuntu-latest
container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
steps:
- name: Checkout
uses: actions/checkout@v2.4.0
- name: setup mix
run: |
mix local.hex --force
mix local.rebar --force
mix deps.get
- name: produce emqx.conf.all template
run: make conf-segs
- name: elixir release
run: mix release --overwrite
- name: start release
run: |
cd _build/dev/rel/emqx
bin/emqx start
- name: check if started
run: |
sleep 10
nc -zv localhost 1883
cd _build/dev/rel/emqx
bin/emqx ping
bin/emqx ctl status

View File

@ -1 +1,2 @@
erlang 24.1.5-3
elixir 1.13.1-otp-24

View File

@ -24,4 +24,8 @@
%% NOTE: This version number should be manually bumped for each release
-define(EMQX_RELEASE, "5.0-beta.2").
%% NOTE: This version number should have 3 numeric parts
%% (Major.Minor.Patch), and extra info can be added after a final
%% hyphen.
-define(EMQX_RELEASE, "5.0.0-beta.2").

View File

@ -5,7 +5,19 @@
{vsn, "5.0.0"}, % strict semver, bump manually!
{modules, []},
{registered, []},
{applications, [kernel,stdlib,gproc,gen_rpc,mria,esockd,cowboy,sasl,os_mon,jiffy,lc]},
{applications, [ kernel
, stdlib
, gproc
, gen_rpc
, mria
, esockd
, cowboy
, sasl
, os_mon
, jiffy
, lc
, hocon
]},
{mod, {emqx_app,[]}},
{env, []},
{licenses, ["Apache-2.0"]},

View File

@ -1,4 +1,5 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,

View File

@ -1,5 +1,5 @@
{erl_opts, [debug_info, nowarn_unused_import]}.
{deps, []}.
{deps, [{emqx, {path, "../emqx"}}]}.
{shell, [
% {config, "config/sys.config"},

View File

@ -1,5 +1,6 @@
{erl_opts, [debug_info]}.
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{shell, [
{apps, [emqx_auto_subscribe]}

View File

@ -1,5 +1,6 @@
{erl_opts, [debug_info]}.
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{shell, [
% {config, "config/sys.config"},

View File

@ -1,5 +1,6 @@
{erl_opts, [debug_info]}.
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{shell, [
% {config, "config/sys.config"},

View File

@ -3,7 +3,6 @@
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_conf_app, []}},
{included_applications, [hocon]},
{applications, [kernel, stdlib]},
{env, []},
{modules, []}

View File

@ -4,6 +4,7 @@
]}.
{deps, [
{emqx, {path, "../emqx"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}},

View File

@ -12,6 +12,7 @@
eredis_cluster,
eredis,
epgsql,
eldap2,
mysql,
mongodb,
ehttpc,

View File

@ -1,4 +1,6 @@
{deps, []}.
{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}}
, {emqx, {path, "../emqx"}}
]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,

View File

@ -5,7 +5,8 @@
]}.
{deps,
[{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}}
[ {emqx, {path, "../emqx"}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}}
]}.
{grpc,

View File

@ -1,5 +1,6 @@
{erl_opts, [debug_info]}.
{deps, [
{emqx, {path, "../emqx"}},
{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}}
]}.

View File

@ -18,7 +18,7 @@
-behaviour(minirest_api).
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
%% API
-export([api_spec/0]).

View File

@ -43,7 +43,7 @@
-export_type([channel/0]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).

View File

@ -28,8 +28,8 @@
, is_message/1
]).
-include("include/emqx_coap.hrl").
-include("apps/emqx/include/types.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("emqx/include/types.hrl").
-define(VERSION, 1).

View File

@ -20,7 +20,7 @@
-module(emqx_coap_medium).
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
%% API
-export([ empty/0, reset/1, reset/2

View File

@ -34,7 +34,7 @@
-export([ set/3, set_payload/2, get_option/2
, get_option/3, set_payload_block/3, set_payload_block/4]).
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
request(Type, Method) ->
request(Type, Method, <<>>, []).

View File

@ -18,7 +18,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
%% API
-export([ new/0

View File

@ -28,7 +28,7 @@
-export_type([manager/0, event_result/1]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-type direction() :: in | out.

View File

@ -1,7 +1,7 @@
-module(emqx_coap_transport).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-define(ACK_TIMEOUT, 2000).
-define(ACK_RANDOM_FACTOR, 1000).

View File

@ -16,7 +16,7 @@
-module(emqx_coap_mqtt_handler).
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-export([handle_request/4]).
-import(emqx_coap_message, [response/2, response/3]).

View File

@ -18,7 +18,7 @@
-module(emqx_coap_pubsub_handler).
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-export([handle_request/4]).

View File

@ -17,8 +17,8 @@
-module(emqx_lwm2m_channel).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
%% API
-export([ info/1

View File

@ -17,8 +17,8 @@
-module(emqx_lwm2m_cmd).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-export([ mqtt_to_coap/2
, coap_to_mqtt/4

View File

@ -18,8 +18,8 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
%% API
-export([ new/0, init/4, update/3, parse_object_list/1

View File

@ -16,7 +16,7 @@
-module(emqx_lwm2m_xml_object).
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-include_lib("xmerl/include/xmerl.hrl").
-export([ get_obj_def/2

View File

@ -16,7 +16,7 @@
-module(emqx_lwm2m_xml_object_db).
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-include_lib("xmerl/include/xmerl.hrl").
-include_lib("emqx/include/logger.hrl").

View File

@ -19,7 +19,7 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

View File

@ -23,8 +23,8 @@
-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)).
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

View File

@ -23,8 +23,8 @@
-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)).
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

View File

@ -19,7 +19,7 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl").
-include("src/mqttsn/include/emqx_sn.hrl").
-include_lib("eunit/include/eunit.hrl").
%%--------------------------------------------------------------------
@ -181,4 +181,3 @@ gen_next(0, Acc) ->
gen_next(N, Acc) ->
Byte = rand:uniform(256) - 1,
gen_next(N-1, <<Acc/binary, Byte:8>>).

View File

@ -19,7 +19,7 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl").
-include("src/mqttsn/include/emqx_sn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

View File

@ -17,7 +17,7 @@
-module(emqx_stomp_SUITE).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx_gateway/src/stomp/include/emqx_stomp.hrl").
-include("src/stomp/include/emqx_stomp.hrl").
-compile(export_all).
-compile(nowarn_export_all).
@ -419,7 +419,7 @@ t_rest_clienit_info(_) ->
%%
%% TODO: Start/Stop, List Instace
%%
%% TODO: RateLimit, OOM,
%% TODO: RateLimit, OOM,
with_connection(DoFun) ->
{ok, Sock} = gen_tcp:connect({127, 0, 0, 1},

View File

@ -21,8 +21,8 @@
-define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)).
-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include("src/lwm2m/include/emqx_lwm2m.hrl").
-include("src/coap/include/emqx_coap.hrl").
-include_lib("eunit/include/eunit.hrl").
%%--------------------------------------------------------------------

View File

@ -16,7 +16,7 @@
-module(emqx_sn_proper_types).
-include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl").
-include("src/mqttsn/include/emqx_sn.hrl").
-include_lib("proper/include/proper.hrl").
-compile({no_auto_import, [register/1]}).

View File

@ -16,7 +16,7 @@
-module(prop_emqx_sn_frame).
-include_lib("src/mqttsn/include/emqx_sn.hrl").
-include("src/mqttsn/include/emqx_sn.hrl").
-include_lib("proper/include/proper.hrl").
-compile({no_auto_import, [register/1]}).
@ -76,4 +76,3 @@ mqtt_sn_message() ->
, M:'WILLTOPICRESP'(), M:'WILLMSGUPD'()
, M:'WILLMSGRESP'()
]).

View File

@ -0,0 +1,2 @@
{deps, [ {emqx, {path, "../emqx"}}
]}.

View File

@ -135,6 +135,12 @@ add_app(G, App, undefined) ->
?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
%% not loaded
add_app(G, App, []);
% We ALWAYS want to add `emqx_conf', even if no other app declare a
% dependency on it. Otherwise, emqx may fail to load the config
% schemas, especially in the test profile.
add_app(G, App = emqx_conf, []) ->
digraph:add_vertex(G, App),
ok;
add_app(_G, _App, []) ->
ok;
add_app(G, App, [Dep | Deps]) ->

View File

@ -43,7 +43,7 @@ init_per_suite(Config) ->
%%
application:unload(emqx_authz),
emqx_common_test_helpers:start_apps([]),
emqx_common_test_helpers:start_apps([emqx_conf]),
Config.
end_per_suite(_Config) ->

View File

@ -1,4 +1,5 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,

View File

@ -1 +1,2 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.

View File

@ -0,0 +1,2 @@
{deps, [ {emqx, {path, "../emqx"}}
]}.

View File

@ -1,5 +1,8 @@
{deps,
[{prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v3.1.1"}}}
[ %% FIXME: tag this as v3.1.3
{prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}},
{minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}}
]}.
{edoc_opts, [{preprocess, true}]}.

View File

@ -1,4 +1,5 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,

View File

@ -8,7 +8,6 @@
[kernel,
stdlib,
gproc,
hocon,
jsx,
emqx
]},

View File

@ -1,4 +1,5 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,
@ -19,6 +20,6 @@
[{test,
[{deps,
[
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.0"}}}]}
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}]}
]}
]}.

View File

@ -1,4 +1,5 @@
{deps, []}.
{deps, [ {emqx, {path, "../emqx"}}
]}.
{erl_opts, [warn_unused_vars,
warn_shadow_vars,

View File

@ -0,0 +1,2 @@
{deps, [ {emqx, {path, "../emqx"}}
]}.

View File

@ -66,7 +66,13 @@ assert_node_alive() {
echoerr() { echo "$*" 1>&2; }
check_erlang_start() {
"$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s erlang halt
# RELEASE_LIB is used by Elixir
"$BINDIR/$PROGNAME" \
-noshell \
-boot_var RELEASE_LIB "$ERTS_LIB_DIR/lib" \
-boot "$REL_DIR/start_clean" \
-s crypto start \
-s erlang halt
}
usage() {
@ -289,9 +295,24 @@ relx_rem_sh() {
# shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
# Setup remote shell command to control node
exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot "$REL_DIR/start_clean" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" $EPMD_ARG
if [ "$IS_ELIXIR" = "yes" ]
then
exec "$REL_DIR/iex" \
--remsh "$NAME" \
--boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
--cookie "$COOKIE" \
--hidden \
--erl "-kernel net_ticktime $TICKTIME" \
--erl "$EPMD_ARG" \
--erl "$NAME_TYPE $id" \
--boot "$REL_DIR/start_clean"
else
exec "$BINDIR/erl" "$NAME_TYPE" "$id" \
-remsh "$NAME" -boot "$REL_DIR/start_clean" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" \
$EPMD_ARG
fi
}
# Generate a random id
@ -353,6 +374,11 @@ generate_config() {
local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
local HOCON_GEN_ARG_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args"
# This is needed by the Elixir scripts.
# Do NOT append `.config`.
RELEASE_SYS_CONFIG="$CONFIGS_DIR/app.$NOW_TIME"
export RELEASE_SYS_CONFIG
CONFIG_ARGS="-config $CONF_FILE -args_file $HOCON_GEN_ARG_FILE"
## Merge hocon generated *.args into the vm.args
@ -688,11 +714,23 @@ case "${COMMAND}" in
# shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
# Build an array of arguments to pass to exec later on
# Build it here because this command will be used for logging.
set -- "$BINDIR/erlexec" \
-boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$CONFIG_ARGS $EPMD_ARG
if [ "$IS_ELIXIR" = yes ]
then
set -- "$REL_DIR/iex" \
--boot "$BOOTFILE" \
--erl "-mode $CODE_LOADING_MODE" \
--boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
--erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
--erl "$CONFIG_ARGS" \
--erl "$EPMD_ARG" \
--werl
else
set -- "$BINDIR/erlexec" \
-boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$CONFIG_ARGS $EPMD_ARG
fi
# Log the startup
logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"
@ -727,11 +765,25 @@ case "${COMMAND}" in
# shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
# Build an array of arguments to pass to exec later on
# Build it here because this command will be used for logging.
set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
-boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$CONFIG_ARGS $EPMD_ARG
if [ "$IS_ELIXIR" = yes ]
then
set -- "$REL_DIR/elixir" \
--boot "$REL_DIR/start" \
--erl "$FOREGROUNDOPTIONS" \
--erl "-mode $CODE_LOADING_MODE" \
--boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
--boot-var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
--erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
--erl "$CONFIG_ARGS" \
--erl "$EPMD_ARG" \
--no-halt
else
set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
-boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \
-boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-mnesia dir "\"${MNESIA_DATA_DIR}\"" \
$CONFIG_ARGS $EPMD_ARG
fi
# Log the startup
logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"
@ -773,7 +825,18 @@ case "${COMMAND}" in
assert_node_alive
shift
relx_nodetool "eval" "$@"
if [ "$IS_ELIXIR" = "yes" ]
then
"$REL_DIR/elixir" \
--hidden \
--cookie "$COOKIE" \
--boot "$REL_DIR/start_clean" \
--boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
--vm-args "$(latest_vm_args 'EMQX_NODE__NAME')"\
--rpc-eval "$NAME" "$@"
else
relx_nodetool "eval" "$@"
fi
;;
*)
usage "$COMMAND"

View File

@ -304,8 +304,8 @@ permafy(TargetNode, RelName, Vsn) ->
make_permanent, [Vsn], ?TIMEOUT),
?INFO("Made release permanent: ~p", [Vsn]),
%% upgrade/downgrade the scripts by replacing them
Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool",
"install_upgrade.escript"],
Scripts = [RelNameStr, RelNameStr ++ "_ctl",
"nodetool", "install_upgrade.escript"],
[{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]),
filename:join(["bin", File]))
|| File <- Scripts],

View File

@ -1,10 +1,11 @@
#!/bin/sh
set -eu
# shellcheck disable=SC1090,SC1091
ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
echo "Running node dump in ${ROOT_DIR}"
# shellcheck disable=SC1090
# shellcheck disable=SC1090,SC1091
. "$ROOT_DIR"/releases/emqx_vars
cd "${ROOT_DIR}"

View File

@ -1 +1 @@
{{built_on_arch}}
{{ built_on_arch }}

View File

@ -13,6 +13,7 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}"
RUNNER_ETC_DIR="{{ runner_etc_dir }}"
RUNNER_DATA_DIR="{{ runner_data_dir }}"
RUNNER_USER="{{ runner_user }}"
IS_ELIXIR="{{ is_elixir }}"
EMQX_LICENSE_CONF=''
export EMQX_DESCRIPTION='{{ emqx_description }}'

878
lib/mix/release.exs Normal file
View File

@ -0,0 +1,878 @@
defmodule Mix.Release do
@moduledoc """
Defines the release structure and convenience for assembling releases.
"""
@doc """
The Mix.Release struct has the following read-only fields:
* `:name` - the name of the release as an atom
* `:version` - the version of the release as a string or
`{:from_app, app_name}`
* `:path` - the path to the release root
* `:version_path` - the path to the release version inside the release
* `:applications` - a map of application with their definitions
* `:erts_source` - the ERTS source as a charlist (or nil)
* `:erts_version` - the ERTS version as a charlist
The following fields may be modified as long as they keep their defined types:
* `:boot_scripts` - a map of boot scripts with the boot script name
as key and a keyword list with **all** applications that are part of
it and their modes as value
* `:config_providers` - a list of `{config_provider, term}` tuples where the
first element is a module that implements the `Config.Provider` behaviour
and `term` is the value given to it on `c:Config.Provider.init/1`
* `:options` - a keyword list with all other user supplied release options
* `:overlays` - a list of extra files added to the release. If you have a custom
step adding extra files to a release, you can add these files to the `:overlays`
field so they are also considered on further commands, such as tar/zip. Each entry
in overlays is the relative path to the release root of each file
* `:steps` - a list of functions that receive the release and returns a release.
Must also contain the atom `:assemble` which is the internal assembling step.
May also contain the atom `:tar` to create a tarball of the release.
"""
defstruct [
:name,
:version,
:path,
:version_path,
:applications,
:boot_scripts,
:erts_source,
:erts_version,
:config_providers,
:options,
:overlays,
:steps
]
@type mode :: :permanent | :transient | :temporary | :load | :none
@type application :: atom()
@type t :: %__MODULE__{
name: atom(),
version: String.t(),
path: String.t(),
version_path: String.t() | {:from_app, application()},
applications: %{application() => keyword()},
boot_scripts: %{atom() => [{application(), mode()}]},
erts_version: charlist(),
erts_source: charlist() | nil,
config_providers: [{module, term}],
options: keyword(),
overlays: list(String.t()),
steps: [(t -> t) | :assemble, ...]
}
@default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent]
@safe_modes [:permanent, :temporary, :transient]
@unsafe_modes [:load, :none]
@significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c
@copy_app_dirs ["priv"]
@doc false
@spec from_config!(atom, keyword, keyword) :: t
def from_config!(name, config, overrides) do
{name, apps, opts} = find_release(name, config)
unless Atom.to_string(name) =~ ~r/^[a-z][a-z0-9_]*$/ do
Mix.raise(
"Invalid release name. A release name must start with a lowercase ASCII letter, " <>
"followed by lowercase ASCII letters, numbers, or underscores, got: #{inspect(name)}"
)
end
opts =
[overwrite: false, quiet: false, strip_beams: true]
|> Keyword.merge(opts)
|> Keyword.merge(overrides)
{include_erts, opts} = Keyword.pop(opts, :include_erts, true)
{erts_source, erts_lib_dir, erts_version} = erts_data(include_erts)
deps_apps = Mix.Project.deps_apps()
loaded_apps = apps |> Keyword.keys() |> load_apps(deps_apps, %{}, erts_lib_dir, [], :root)
# Make sure IEx is either an active part of the release or add it as none.
{loaded_apps, apps} =
if Map.has_key?(loaded_apps, :iex) do
{loaded_apps, apps}
else
{load_apps([:iex], deps_apps, loaded_apps, erts_lib_dir, [], :root), apps ++ [iex: :none]}
end
start_boot = build_start_boot(loaded_apps, apps)
start_clean_boot = build_start_clean_boot(start_boot)
{path, opts} =
Keyword.pop_lazy(opts, :path, fn ->
Path.join([Mix.Project.build_path(config), "rel", Atom.to_string(name)])
end)
path = Path.absname(path)
{version, opts} =
Keyword.pop_lazy(opts, :version, fn ->
config[:version] ||
Mix.raise(
"No :version found. Please make sure a :version is set in your project definition " <>
"or inside the release the configuration"
)
end)
version =
case version do
{:from_app, app} ->
Application.load(app)
version = Application.spec(app, :vsn)
if !version do
Mix.raise(
"Could not find version for #{inspect(app)}, please make sure the application exists"
)
end
to_string(version)
"" ->
Mix.raise("The release :version cannot be an empty string")
_ ->
version
end
{config_providers, opts} = Keyword.pop(opts, :config_providers, [])
{steps, opts} = Keyword.pop(opts, :steps, [:assemble])
validate_steps!(steps)
%Mix.Release{
name: name,
version: version,
path: path,
version_path: Path.join([path, "releases", version]),
erts_source: erts_source,
erts_version: erts_version,
applications: loaded_apps,
boot_scripts: %{start: start_boot, start_clean: start_clean_boot},
config_providers: config_providers,
options: opts,
overlays: [],
steps: steps
}
end
defp find_release(name, config) do
{name, opts_fun_or_list} = lookup_release(name, config) || infer_release(config)
opts = if is_function(opts_fun_or_list, 0), do: opts_fun_or_list.(), else: opts_fun_or_list
{apps, opts} = Keyword.pop(opts, :applications, [])
if apps == [] and Mix.Project.umbrella?(config) do
bad_umbrella!()
end
app = Keyword.get(config, :app)
apps = Keyword.merge(@default_apps, apps)
if is_nil(app) or Keyword.has_key?(apps, app) do
{name, apps, opts}
else
{name, apps ++ [{app, :permanent}], opts}
end
end
defp lookup_release(nil, config) do
case Keyword.get(config, :releases, []) do
[] ->
nil
[{name, opts}] ->
{name, opts}
[_ | _] ->
case Keyword.get(config, :default_release) do
nil ->
Mix.raise(
"\"mix release\" was invoked without a name but there are multiple releases. " <>
"Please call \"mix release NAME\" or set :default_release in your project configuration"
)
name ->
lookup_release(name, config)
end
end
end
defp lookup_release(name, config) do
if opts = config[:releases][name] do
{name, opts}
else
found = Keyword.get(config, :releases, [])
Mix.raise(
"Unknown release #{inspect(name)}. " <>
"The available releases are: #{inspect(Keyword.keys(found))}"
)
end
end
defp infer_release(config) do
if Mix.Project.umbrella?(config) do
bad_umbrella!()
else
{Keyword.fetch!(config, :app), []}
end
end
defp bad_umbrella! do
Mix.raise("""
Umbrella projects require releases to be explicitly defined with \
a non-empty applications key that chooses which umbrella children \
should be part of the releases:
releases: [
foo: [
applications: [child_app_foo: :permanent]
],
bar: [
applications: [child_app_bar: :permanent]
]
]
Alternatively you can perform the release from the children applications
""")
end
defp erts_data(erts_data) when is_function(erts_data) do
erts_data(erts_data.())
end
defp erts_data(false) do
{nil, :code.lib_dir(), :erlang.system_info(:version)}
end
defp erts_data(true) do
version = :erlang.system_info(:version)
{:filename.join(:code.root_dir(), 'erts-#{version}'), :code.lib_dir(), version}
end
defp erts_data(erts_source) when is_binary(erts_source) do
if File.exists?(erts_source) do
[_, erts_version] = erts_source |> Path.basename() |> String.split("-")
erts_lib_dir = erts_source |> Path.dirname() |> Path.join("lib") |> to_charlist()
{to_charlist(erts_source), erts_lib_dir, to_charlist(erts_version)}
else
Mix.raise("Could not find ERTS system at #{inspect(erts_source)}")
end
end
defp load_apps(apps, deps_apps, seen, otp_root, optional, type) do
for app <- apps, reduce: seen do
seen ->
if reentrant_seen = reentrant(seen, app, type) do
reentrant_seen
else
load_app(app, deps_apps, seen, otp_root, optional, type)
end
end
end
defp reentrant(seen, app, type) do
properties = seen[app]
cond do
is_nil(properties) ->
nil
type != :root and properties[:type] != type ->
if properties[:type] == :root do
put_in(seen[app][:type], type)
else
Mix.raise(
"#{inspect(app)} is listed both as a regular application and as an included application"
)
end
true ->
seen
end
end
defp load_app(app, deps_apps, seen, otp_root, optional, type) do
cond do
path = app not in deps_apps && otp_path(otp_root, app) ->
do_load_app(app, path, deps_apps, seen, otp_root, true, type)
path = code_path(app) ->
do_load_app(app, path, deps_apps, seen, otp_root, false, type)
app in optional ->
seen
true ->
Mix.raise("Could not find application #{inspect(app)}")
end
end
defp otp_path(otp_root, app) do
path = Path.join(otp_root, "#{app}-*")
case Path.wildcard(path) do
[] -> nil
paths -> paths |> Enum.sort() |> List.last() |> to_charlist()
end
end
defp code_path(app) do
case :code.lib_dir(app) do
{:error, :bad_name} -> nil
path -> path
end
end
defp do_load_app(app, path, deps_apps, seen, otp_root, otp_app?, type) do
case :file.consult(Path.join(path, "ebin/#{app}.app")) do
{:ok, terms} ->
[{:application, ^app, properties}] = terms
value = [path: path, otp_app?: otp_app?, type: type] ++ properties
seen = Map.put(seen, app, value)
applications = Keyword.get(properties, :applications, [])
optional = Keyword.get(properties, :optional_applications, [])
seen = load_apps(applications, deps_apps, seen, otp_root, optional, :depended)
included_applications = Keyword.get(properties, :included_applications, [])
load_apps(included_applications, deps_apps, seen, otp_root, [], :included)
{:error, reason} ->
Mix.raise("Could not load #{app}.app. Reason: #{inspect(reason)}")
end
end
defp build_start_boot(all_apps, specified_apps) do
specified_apps ++
Enum.sort(
for(
{app, props} <- all_apps,
not List.keymember?(specified_apps, app, 0),
do: {app, default_mode(props)}
)
)
end
defp default_mode(props) do
if props[:type] == :included, do: :load, else: :permanent
end
defp build_start_clean_boot(boot) do
for({app, _mode} <- boot, do: {app, :none})
|> Keyword.put(:stdlib, :permanent)
|> Keyword.put(:kernel, :permanent)
end
defp validate_steps!(steps) do
valid_atoms = [:assemble, :tar]
if not is_list(steps) or Enum.any?(steps, &(&1 not in valid_atoms and not is_function(&1, 1))) do
Mix.raise("""
The :steps option must be a list of:
* anonymous function that receives one argument
* the atom :assemble or :tar
Got: #{inspect(steps)}
""")
end
if Enum.count(steps, &(&1 == :assemble)) != 1 do
Mix.raise("The :steps option must contain the atom :assemble once, got: #{inspect(steps)}")
end
if :assemble in Enum.drop_while(steps, &(&1 != :tar)) do
Mix.raise("The :tar step must come after :assemble")
end
if Enum.count(steps, &(&1 == :tar)) > 1 do
Mix.raise("The :steps option can only contain the atom :tar once")
end
:ok
end
@doc """
Makes the `sys.config` structure.
If there are config providers, then a value is injected into
the `:elixir` application configuration in `sys_config` to be
read during boot and trigger the providers.
It uses the following release options to customize its behaviour:
* `:reboot_system_after_config`
* `:start_distribution_during_config`
* `:prune_runtime_sys_config_after_boot`
In case there are no config providers, it doesn't change `sys_config`.
"""
@spec make_sys_config(t, keyword(), Config.Provider.config_path()) ::
:ok | {:error, String.t()}
def make_sys_config(release, sys_config, config_provider_path) do
{sys_config, runtime_config?} =
merge_provider_config(release, sys_config, config_provider_path)
path = Path.join(release.version_path, "sys.config")
args = [runtime_config?, sys_config]
format = "%% coding: utf-8~n%% RUNTIME_CONFIG=~s~n~tw.~n"
File.mkdir_p!(Path.dirname(path))
File.write!(path, IO.chardata_to_string(:io_lib.format(format, args)))
case :file.consult(path) do
{:ok, _} ->
:ok
{:error, reason} ->
invalid =
for {app, kv} <- sys_config,
{key, value} <- kv,
not valid_config?(value),
do: """
Application: #{inspect(app)}
Key: #{inspect(key)}
Value: #{inspect(value)}
"""
message =
case invalid do
[] ->
"Could not read configuration file. Reason: #{inspect(reason)}"
_ ->
"Could not read configuration file. It has invalid configuration terms " <>
"such as functions, references, and pids. Please make sure your configuration " <>
"is made of numbers, atoms, strings, maps, tuples and lists. The following entries " <>
"are wrong:\n#{Enum.join(invalid)}"
end
{:error, message}
end
end
defp valid_config?(m) when is_map(m),
do: Enum.all?(Map.delete(m, :__struct__), &valid_config?/1)
defp valid_config?(l) when is_list(l), do: Enum.all?(l, &valid_config?/1)
defp valid_config?(t) when is_tuple(t), do: Enum.all?(Tuple.to_list(t), &valid_config?/1)
defp valid_config?(o), do: is_number(o) or is_atom(o) or is_binary(o)
defp merge_provider_config(%{config_providers: []}, sys_config, _), do: {sys_config, false}
defp merge_provider_config(release, sys_config, config_path) do
{reboot?, extra_config, initial_config} = start_distribution(release)
prune_runtime_sys_config_after_boot =
Keyword.get(release.options, :prune_runtime_sys_config_after_boot, false)
opts = [
extra_config: initial_config,
prune_runtime_sys_config_after_boot: prune_runtime_sys_config_after_boot,
reboot_system_after_config: reboot?,
validate_compile_env: validate_compile_env(release)
]
init_config = Config.Provider.init(release.config_providers, config_path, opts)
{Config.Reader.merge(sys_config, init_config ++ extra_config), reboot?}
end
defp validate_compile_env(release) do
with true <- Keyword.get(release.options, :validate_compile_env, true),
[_ | _] = compile_env <- compile_env(release) do
compile_env
else
_ -> false
end
end
defp compile_env(release) do
for {_, properties} <- release.applications,
triplet <- Keyword.get(properties, :compile_env, []),
do: triplet
end
defp start_distribution(%{options: opts}) do
reboot? = Keyword.get(opts, :reboot_system_after_config, false)
early_distribution? = Keyword.get(opts, :start_distribution_during_config, false)
if not reboot? or early_distribution? do
{reboot?, [], []}
else
{true, [kernel: [start_distribution: false]], [kernel: [start_distribution: true]]}
end
end
@doc """
Copies the cookie to the given path.
If a cookie option was given, we compare it with
the contents of the file (if any), and ask the user
if they want to override.
If there is no option, we generate a random one
the first time.
"""
@spec make_cookie(t, Path.t()) :: :ok
def make_cookie(release, path) do
cond do
cookie = release.options[:cookie] ->
Mix.Generator.create_file(path, cookie, quiet: true)
:ok
File.exists?(path) ->
:ok
true ->
File.write!(path, random_cookie())
:ok
end
end
defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32))
@doc """
Makes the start_erl.data file with the
ERTS version and release versions.
"""
@spec make_start_erl(t, Path.t()) :: :ok
def make_start_erl(release, path) do
File.write!(path, "#{release.erts_version} #{release.version}")
:ok
end
@doc """
Makes boot scripts.
It receives a path to the boot file, without extension, such as
`releases/0.1.0/start` and this command will write `start.rel`,
`start.boot`, and `start.script` to the given path, returning
`{:ok, rel_path}` or `{:error, message}`.
The boot script uses the RELEASE_LIB environment variable, which must
be accordingly set with `--boot-var` and point to the release lib dir.
"""
@spec make_boot_script(t, Path.t(), [{application(), mode()}], [String.t()]) ::
:ok | {:error, String.t()}
def make_boot_script(release, path, modes, prepend_paths \\ []) do
with {:ok, rel_spec} <- build_release_spec(release, modes) do
File.write!(path <> ".rel", consultable(rel_spec))
sys_path = String.to_charlist(path)
sys_options = [
:silent,
:no_dot_erlang,
:no_warn_sasl,
variables: build_variables(release),
path: build_paths(release)
]
case :systools.make_script(sys_path, sys_options) do
{:ok, _module, _warnings} ->
script_path = sys_path ++ '.script'
{:ok, [{:script, rel_info, instructions}]} = :file.consult(script_path)
instructions =
instructions
|> post_stdlib_applies(release)
|> prepend_paths_to_script(prepend_paths)
script = {:script, rel_info, instructions}
File.write!(script_path, consultable(script))
:ok = :systools.script2boot(sys_path)
{:error, module, info} ->
message = module.format_error(info) |> to_string() |> String.trim()
{:error, message}
end
end
end
defp build_variables(release) do
for {_, properties} <- release.applications,
not Keyword.fetch!(properties, :otp_app?),
uniq: true,
do: {'RELEASE_LIB', properties |> Keyword.fetch!(:path) |> :filename.dirname()}
end
defp build_paths(release) do
for {_, properties} <- release.applications,
Keyword.fetch!(properties, :otp_app?),
do: properties |> Keyword.fetch!(:path) |> Path.join("ebin") |> to_charlist()
end
defp build_release_spec(release, modes) do
%{
name: name,
version: version,
erts_version: erts_version,
applications: apps,
options: options
} = release
skip_mode_validation_for =
options
|> Keyword.get(:skip_mode_validation_for, [])
|> MapSet.new()
rel_apps =
for {app, mode} <- modes do
properties = Map.get(apps, app) || throw({:error, "Unknown application #{inspect(app)}"})
children = Keyword.get(properties, :applications, [])
app in skip_mode_validation_for || validate_mode!(app, mode, modes, children)
build_app_for_release(app, mode, properties)
end
{:ok, {:release, {to_charlist(name), to_charlist(version)}, {:erts, erts_version}, rel_apps}}
catch
{:error, message} -> {:error, message}
end
defp validate_mode!(app, mode, modes, children) do
safe_mode? = mode in @safe_modes
if not safe_mode? and mode not in @unsafe_modes do
throw(
{:error,
"Unknown mode #{inspect(mode)} for #{inspect(app)}. " <>
"Valid modes are: #{inspect(@safe_modes ++ @unsafe_modes)}"}
)
end
for child <- children do
child_mode = Keyword.get(modes, child)
cond do
is_nil(child_mode) ->
throw(
{:error,
"Application #{inspect(app)} is listed in the release boot, " <>
"but it depends on #{inspect(child)}, which isn't"}
)
safe_mode? and child_mode in @unsafe_modes ->
throw(
{:error,
"""
Application #{inspect(app)} has mode #{inspect(mode)} but it depends on \
#{inspect(child)} which is set to #{inspect(child_mode)}. If you really want \
to set such mode for #{inspect(child)} make sure that all applications that depend \
on it are also set to :load or :none, otherwise your release will fail to boot
"""}
)
true ->
:ok
end
end
end
defp build_app_for_release(app, mode, properties) do
vsn = Keyword.fetch!(properties, :vsn)
case Keyword.get(properties, :included_applications, []) do
[] -> {app, vsn, mode}
included_apps -> {app, vsn, mode, included_apps}
end
end
defp post_stdlib_applies(instructions, release) do
{pre, [stdlib | post]} =
Enum.split_while(
instructions,
&(not match?({:apply, {:application, :start_boot, [:stdlib, _]}}, &1))
)
pre ++ [stdlib] ++ config_provider_apply(release) ++ post
end
defp config_provider_apply(%{config_providers: []}),
do: []
defp config_provider_apply(_),
do: [{:apply, {Config.Provider, :boot, []}}]
defp prepend_paths_to_script(instructions, []), do: instructions
defp prepend_paths_to_script(instructions, prepend_paths) do
prepend_paths = Enum.map(prepend_paths, &String.to_charlist/1)
Enum.map(instructions, fn
{:path, paths} ->
if Enum.any?(paths, &List.starts_with?(&1, '$RELEASE_LIB')) do
{:path, prepend_paths ++ paths}
else
{:path, paths}
end
other ->
other
end)
end
defp consultable(term) do
IO.chardata_to_string(:io_lib.format("%% coding: utf-8~n~tp.~n", [term]))
end
@doc """
Finds a template path for the release.
"""
def rel_templates_path(release, path) do
Path.join(release.options[:rel_templates_path] || "rel", path)
end
@doc """
Copies ERTS if the release is configured to do so.
Returns true if the release was copied, false otherwise.
"""
@spec copy_erts(t) :: boolean()
def copy_erts(%{erts_source: nil}) do
false
end
def copy_erts(release) do
destination = Path.join(release.path, "erts-#{release.erts_version}/bin")
File.mkdir_p!(destination)
release.erts_source
|> Path.join("bin")
|> File.cp_r!(destination, fn _, _ -> false end)
_ = File.rm(Path.join(destination, "erl"))
_ = File.rm(Path.join(destination, "erl.ini"))
destination
|> Path.join("erl")
|> File.write!(~S"""
#!/bin/sh
SELF=$(readlink "$0" || true)
if [ -z "$SELF" ]; then SELF="$0"; fi
BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)"
ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}"
EMU=beam
PROGNAME=$(echo "$0" | sed 's/.*\///')
export EMU
export ROOTDIR
export BINDIR
export PROGNAME
exec "$BINDIR/erlexec" ${1+"$@"}
""")
File.chmod!(Path.join(destination, "erl"), 0o755)
true
end
@doc """
Copies the given application specification into the release.
It assumes the application exists in the release.
"""
@spec copy_app(t, application) :: boolean()
def copy_app(release, app) do
properties = Map.fetch!(release.applications, app)
vsn = Keyword.fetch!(properties, :vsn)
source_app = Keyword.fetch!(properties, :path)
target_app = Path.join([release.path, "lib", "#{app}-#{vsn}"])
if is_nil(release.erts_source) and Keyword.fetch!(properties, :otp_app?) do
false
else
File.rm_rf!(target_app)
File.mkdir_p!(target_app)
copy_ebin(release, Path.join(source_app, "ebin"), Path.join(target_app, "ebin"))
for dir <- @copy_app_dirs do
source_dir = Path.join(source_app, dir)
target_dir = Path.join(target_app, dir)
source_dir =
case File.read_link(source_dir) do
{:ok, link_target} -> Path.expand(link_target, source_app)
_ -> source_dir
end
File.exists?(source_dir) && File.cp_r!(source_dir, target_dir)
end
true
end
end
@doc """
Copies the ebin directory at `source` to `target`
respecting release options such a `:strip_beams`.
"""
@spec copy_ebin(t, Path.t(), Path.t()) :: boolean()
def copy_ebin(release, source, target) do
with {:ok, [_ | _] = files} <- File.ls(source) do
File.mkdir_p!(target)
strip_options =
release.options
|> Keyword.get(:strip_beams, true)
|> parse_strip_beams_options()
for file <- files do
source_file = Path.join(source, file)
target_file = Path.join(target, file)
with true <- is_list(strip_options) and String.ends_with?(file, ".beam"),
{:ok, binary} <- strip_beam(File.read!(source_file), strip_options) do
File.write!(target_file, binary)
else
_ ->
# Use File.cp!/3 to preserve file mode for any executables stored
# in the ebin directory.
File.cp!(source_file, target_file)
end
end
true
else
_ -> false
end
end
@doc """
Strips a beam file for a release.
This keeps only significant chunks necessary for the VM operation,
discarding documentation, debug info, compile information and others.
The exact chunks that are kept are not documented and may change in
future versions.
"""
@spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()}
def strip_beam(binary, options \\ []) when is_list(options) do
chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1)
all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep)
case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do
{:ok, {_, chunks}} ->
chunks = for {name, chunk} <- chunks, is_binary(chunk), do: {name, chunk}
{:ok, binary} = :beam_lib.build_module(chunks)
{:ok, :zlib.gzip(binary)}
{:error, _, _} = error ->
error
end
end
defp parse_strip_beams_options(options) do
case options do
options when is_list(options) -> options
true -> []
false -> nil
end
end
end

602
mix.exs Normal file
View File

@ -0,0 +1,602 @@
defmodule EMQXUmbrella.MixProject do
use Mix.Project
@moduledoc """
The purpose of this file is to configure the release of EMQX under
Mix. Since EMQX uses its own configuration conventions and startup
procedures, one cannot simply use `iex -S mix`. Instead, it's
recommendd to build and use the release.
## Release Environment Variables
The release build is controlled by a few environment variables.
* `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
tarball along with the release.
* `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`. Controls a
few dependencies and the `vm.args` to be used. Defaults to
`cloud`.
* `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`. Controls
whether the build is intended for direct usage or for packaging.
Defaults to `bin`.
* `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`.
Defaults to `community`.
"""
# Temporary hack while 1.13.2 is not released
System.version()
|> Version.parse!()
|> Version.compare(Version.parse!("1.13.2"))
|> Kernel.==(:lt)
|> if(do: Code.require_file("lib/mix/release.exs"))
def project() do
[
app: :emqx_mix,
version: pkg_vsn(),
deps: deps(),
releases: releases()
]
end
defp deps() do
# we need several overrides here because dependencies specify
# other exact versions, and not ranges.
[
{:lc, github: "qzhuyan/lc", tag: "0.1.2"},
{:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true},
{:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"},
{:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.0", override: true},
{:mria, github: "emqx/mria", tag: "0.1.5", override: true},
{:ekka, github: "emqx/ekka", tag: "0.11.2", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true},
{:minirest, github: "emqx/minirest", tag: "1.2.7", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.1"},
{:replayq, "0.3.3", override: true},
{:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
{:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true},
{:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
{:observer_cli, "1.7.1"},
{:system_monitor, github: "k32/system_monitor", tag: "2.2.1"},
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.22.0", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
# in conflict by ehttpc and emqtt
{:gun, github: "emqx/gun", tag: "1.3.6", override: true},
# in conflict by emqx_connectior and system_monitor
{:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true},
# in conflict by mongodb and eredis_cluster
{:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true},
# in conflict by emqx and observer_cli
{:recon, github: "ferd/recon", tag: "2.5.1", override: true},
{:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
# dependencies of dependencies; we choose specific refs to match
# what rebar3 chooses.
# in conflict by gun and emqtt
{:cowlib,
github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
# in conflict by cowboy_swagger and cowboy
{:ranch,
github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
# in conflict by grpc and eetcd
{:gpb, "4.11.2", override: true}
] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
end
defp umbrella_apps() do
"apps/*"
|> Path.wildcard()
|> Enum.map(fn path ->
app =
path
|> String.trim_leading("apps/")
|> String.to_atom()
{app, path: path, manager: :rebar3, override: true}
end)
end
defp releases() do
[
emqx: fn ->
%{
release_type: release_type,
package_type: package_type,
edition_type: edition_type
} = read_inputs()
base_steps = [
:assemble,
&create_RELEASES/1,
&copy_files(&1, release_type, package_type, edition_type),
&copy_escript(&1, "nodetool"),
&copy_escript(&1, "install_upgrade.escript")
]
steps =
if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
base_steps ++ [:tar]
else
base_steps
end
[
applications: applications(release_type),
skip_mode_validation_for: [
:emqx_gateway,
:emqx_dashboard,
:emqx_resource,
:emqx_connector,
:emqx_exhook,
:emqx_bridge,
:emqx_modules,
:emqx_management,
:emqx_statsd,
:emqx_retainer,
:emqx_prometheus,
:emqx_plugins
],
steps: steps,
strip_beams: false
]
end
]
end
def applications(release_type) do
[
logger: :permanent,
crypto: :permanent,
public_key: :permanent,
asn1: :permanent,
syntax_tools: :permanent,
ssl: :permanent,
os_mon: :permanent,
inets: :permanent,
compiler: :permanent,
runtime_tools: :permanent,
hocon: :load,
emqx: :load,
emqx_conf: :load,
emqx_machine: :permanent,
mria: :load,
mnesia: :load,
ekka: :load,
emqx_plugin_libs: :load,
esasl: :load,
observer_cli: :permanent,
system_monitor: :permanent,
emqx_http_lib: :permanent,
emqx_resource: :permanent,
emqx_connector: :permanent,
emqx_authn: :permanent,
emqx_authz: :permanent,
emqx_auto_subscribe: :permanent,
emqx_gateway: :permanent,
emqx_exhook: :permanent,
emqx_bridge: :permanent,
emqx_rule_engine: :permanent,
emqx_modules: :permanent,
emqx_management: :permanent,
emqx_dashboard: :permanent,
emqx_retainer: :permanent,
emqx_statsd: :permanent,
emqx_prometheus: :permanent,
emqx_psk: :permanent,
emqx_slow_subs: :permanent,
emqx_plugins: :permanent,
emqx_mix: :none
] ++
if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
if(release_type == :cloud,
do: [xmerl: :permanent, observer: :load],
else: []
)
end
defp read_inputs() do
release_type =
read_enum_env_var(
"EMQX_RELEASE_TYPE",
[:cloud, :edge],
:cloud
)
package_type =
read_enum_env_var(
"EMQX_PACKAGE_TYPE",
[:bin, :pkg],
:bin
)
edition_type =
read_enum_env_var(
"EMQX_EDITION_TYPE",
[:community, :enterprise],
:community
)
%{
release_type: release_type,
package_type: package_type,
edition_type: edition_type
}
end
defp copy_files(release, release_type, package_type, edition_type) do
overwrite? = Keyword.get(release.options, :overwrite, false)
bin = Path.join(release.path, "bin")
etc = Path.join(release.path, "etc")
Mix.Generator.create_directory(bin)
Mix.Generator.create_directory(etc)
Mix.Generator.create_directory(Path.join(etc, "certs"))
Mix.Generator.copy_file(
"apps/emqx_authz/etc/acl.conf",
Path.join(etc, "acl.conf"),
force: overwrite?
)
# required by emqx_authz
File.cp_r!(
"apps/emqx/etc/certs",
Path.join(etc, "certs")
)
# this is required by the produced escript / nodetool
Mix.Generator.copy_file(
Path.join(release.version_path, "start_clean.boot"),
Path.join(bin, "no_dot_erlang.boot"),
force: overwrite?
)
assigns = template_vars(release, release_type, package_type, edition_type)
# This is generated by `scripts/merge-config.escript` or `make
# conf-segs`. So, this should be run before the release.
# TODO: run as a "compiler" step???
conf_rendered =
File.read!("apps/emqx_conf/etc/emqx.conf.all")
|> from_rebar_to_eex_template()
|> EEx.eval_string(assigns)
File.write!(
Path.join(etc, "emqx.conf"),
conf_rendered
)
vars_rendered =
File.read!("data/emqx_vars")
|> from_rebar_to_eex_template()
|> EEx.eval_string(assigns)
File.write!(
Path.join([release.path, "releases", "emqx_vars"]),
vars_rendered
)
vm_args_template_path =
case release_type do
:cloud ->
"apps/emqx/etc/emqx_cloud/vm.args"
:edge ->
"apps/emqx/etc/emqx_edge/vm.args"
end
vm_args_rendered =
File.read!(vm_args_template_path)
|> from_rebar_to_eex_template()
|> EEx.eval_string(assigns)
File.write!(
Path.join(etc, "vm.args"),
vm_args_rendered
)
File.write!(
Path.join(release.version_path, "vm.args"),
vm_args_rendered
)
for name <- [
"emqx",
"emqx_ctl"
] do
Mix.Generator.copy_file(
"bin/#{name}",
Path.join(bin, name),
force: overwrite?
)
# Files with the version appended are expected by the release
# upgrade script `install_upgrade.escript`
Mix.Generator.copy_file(
Path.join(bin, name),
Path.join(bin, name <> "-#{release.version}"),
force: overwrite?
)
end
for base_name <- ["emqx", "emqx_ctl"],
suffix <- ["", "-#{release.version}"] do
name = base_name <> suffix
File.chmod!(Path.join(bin, name), 0o755)
end
built_on_rendered =
File.read!("data/BUILT_ON")
|> from_rebar_to_eex_template()
|> EEx.eval_string(assigns)
File.write!(
Path.join([release.version_path, "BUILT_ON"]),
built_on_rendered
)
release
end
# needed by nodetool and by release_handler
defp create_RELEASES(release) do
apps =
Enum.map(release.applications, fn {app_name, app_props} ->
app_vsn = Keyword.fetch!(app_props, :vsn)
app_path =
"./lib"
|> Path.join("#{app_name}-#{app_vsn}")
|> to_charlist()
{app_name, app_vsn, app_path}
end)
release_entry = [
{
:release,
to_charlist(release.name),
to_charlist(release.version),
release.erts_version,
apps,
:permanent
}
]
release.path
|> Path.join("releases")
|> Path.join("RELEASES")
|> File.open!([:write, :utf8], fn handle ->
IO.puts(handle, "%% coding: utf-8")
:io.format(handle, '~tp.~n', [release_entry])
end)
release
end
defp copy_escript(release, escript_name) do
[shebang, rest] =
"bin/#{escript_name}"
|> File.read!()
|> String.split("\n", parts: 2)
# the elixir version of escript + start.boot required the boot_var
# RELEASE_LIB to be defined.
boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"
# Files with the version appended are expected by the release
# upgrade script `install_upgrade.escript`
Enum.each(
[escript_name, escript_name <> "-" <> release.version],
fn name ->
path = Path.join([release.path, "bin", name])
File.write!(path, [shebang, "\n", boot_var, "\n", rest])
end
)
release
end
defp template_vars(release, release_type, :bin = _package_type, edition_type) do
[
platform_bin_dir: "bin",
platform_data_dir: "data",
platform_etc_dir: "etc",
platform_lib_dir: "lib",
platform_log_dir: "log",
platform_plugins_dir: "plugins",
runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)",
runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
runner_etc_dir: "$RUNNER_ROOT_DIR/etc",
runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
runner_log_dir: "$RUNNER_ROOT_DIR/log",
runner_data_dir: "$RUNNER_ROOT_DIR/data",
runner_user: "",
release_version: release.version,
erts_vsn: release.erts_version,
# FIXME: this is empty in `make emqx` ???
erl_opts: "",
emqx_description: emqx_description(release_type, edition_type),
built_on_arch: built_on(),
is_elixir: "yes"
]
end
defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
[
platform_bin_dir: "",
platform_data_dir: "/var/lib/emqx",
platform_etc_dir: "/etc/emqx",
platform_lib_dir: "",
platform_log_dir: "/var/log/emqx",
platform_plugins_dir: "/var/lib/emqx/plugins",
runner_root_dir: "/usr/lib/emqx",
runner_bin_dir: "/usr/bin",
runner_etc_dir: "/etc/emqx",
runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
runner_log_dir: "/var/log/emqx",
runner_data_dir: "/var/lib/emqx",
runner_user: "emqx",
release_version: release.version,
erts_vsn: release.erts_version,
# FIXME: this is empty in `make emqx` ???
erl_opts: "",
emqx_description: emqx_description(release_type, edition_type),
built_on: built_on(),
is_elixir: "yes"
]
end
defp read_enum_env_var(env_var, allowed_values, default_value) do
case System.fetch_env(env_var) do
:error ->
default_value
{:ok, raw_value} ->
value =
raw_value
|> String.downcase()
|> String.to_atom()
if value not in allowed_values do
Mix.raise("""
Invalid value #{raw_value} for variable #{env_var}.
Allowed values are: #{inspect(allowed_values)}
""")
end
value
end
end
defp emqx_description(release_type, edition_type) do
case {release_type, edition_type} do
{:cloud, :enterprise} ->
"EMQ X Enterprise Edition"
{:cloud, :community} ->
"EMQ X Community Edition"
{:edge, :community} ->
"EMQ X Edge Edition"
end
end
defp bcrypt_dep() do
if enable_bcrypt?(),
do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}],
else: []
end
defp quicer_dep() do
if enable_quicer?(),
# in conflict with emqx and emqtt
do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}],
else: []
end
defp enable_bcrypt?() do
not win32?()
end
defp enable_quicer?() do
not Enum.any?([
build_without_quic?(),
win32?(),
centos6?()
])
end
defp pkg_vsn() do
basedir = Path.dirname(__ENV__.file)
script = Path.join(basedir, "pkg-vsn.sh")
{str_vsn, 0} = System.cmd(script, [])
String.trim(str_vsn)
end
defp win32?(),
do: match?({:win_32, _}, :os.type())
defp centos6?() do
case File.read("/etc/centos-release") do
{:ok, "CentOS release 6" <> _} ->
true
_ ->
false
end
end
defp build_without_quic?() do
opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
String.downcase(opt) != "false"
end
defp from_rebar_to_eex_template(str) do
# we must not consider surrounding space in the template var name
# because some help strings contain informative variables that
# should not be interpolated, and those have no spaces.
Regex.replace(
~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
str,
"<%= \\g{1} %>"
)
end
defp built_on() do
system_architecture = to_string(:erlang.system_info(:system_architecture))
elixir_version = System.version()
words = wordsize()
"#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}"
end
# https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
defp wordsize() do
size =
try do
:erlang.system_info({:wordsize, :external})
rescue
ErlangError ->
:erlang.system_info(:wordsize)
end
to_string(8 * size)
end
# As from Erlang/OTP 17, the OTP release number corresponds to the
# major OTP version number. No erlang:system_info() argument gives
# the exact OTP version.
# https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
# https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
defp otp_release() do
major_version = System.otp_release()
root_dir = to_string(:code.root_dir())
[root_dir, "releases", major_version, "OTP_VERSION"]
|> Path.join()
|> File.read()
|> case do
{:error, _} ->
major_version
{:ok, version} ->
version
|> String.trim()
|> String.split("**")
|> List.first()
end
end
end

54
mix.lock Normal file
View File

@ -0,0 +1,54 @@
%{
"bcrypt": {:git, "https://github.com/emqx/erlang-bcrypt.git", "dc2ba66acf2332c111362d01137746eefecc5e90", [tag: "0.6.0"]},
"bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]},
"cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]},
"cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]},
"cowlib": {:git, "https://github.com/ninenines/cowlib.git", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363"]},
"ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]},
"eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]},
"ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]},
"ekka": {:git, "https://github.com/emqx/ekka.git", "70f2250e5e968e0c1da64e5b4733c5eb0eb402de", [tag: "0.11.2"]},
"eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]},
"emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]},
"emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]},
"epgsql": {:git, "https://github.com/epgsql/epgsql.git", "f7530f63ae40ea2b81bae7d4a33292212349b761", [tag: "4.6.0"]},
"eredis": {:git, "https://github.com/emqx/eredis", "75f2b8eedbe631136326680225efbcd2684e93e7", [tag: "1.2.5"]},
"eredis_cluster": {:git, "https://github.com/emqx/eredis_cluster", "624749b4aef25668e9c7a545427fdc663a04faef", [tag: "0.6.7"]},
"esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]},
"esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]},
"estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]},
"gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]},
"getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"},
"gpb": {:hex, :gpb, "4.11.2", "a2c05241408310b8bd8dbdfd5e1a419799e45fc1408371eaa0f595023c3b21aa", [:make, :rebar], [], "hexpm", "b355a5982b604d6c044ebb6013e5fe65d30c30d0700d35317e7066691eb6bd61"},
"gproc": {:git, "https://github.com/uwiger/gproc.git", "ce7397809aca0d6eb3aac6db65953752e47fb511", [tag: "0.8.0"]},
"grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]},
"gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]},
"hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]},
"hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},
"jiffy": {:git, "https://github.com/emqx/jiffy.git", "baa1f4e750ae3c5c9e54f9c2e52280b7fc24a8d9", [tag: "1.0.5"]},
"jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]},
"jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]},
"lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]},
"minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]},
"mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]},
"mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]},
"mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]},
"mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]},
"observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"},
"pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]},
"poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]},
"prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]},
"quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]},
"ranch": {:git, "https://github.com/ninenines/ranch.git", "a692f44567034dacf5efcaa24a24183788594eb7", [ref: "a692f44567034dacf5efcaa24a24183788594eb7"]},
"recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]},
"replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"},
"rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]},
"rulesql": {:git, "https://github.com/emqx/rulesql.git", "fec11b1a3cbf98480d19c06d3aca10442e1e02a9", [tag: "0.1.4"]},
"sext": {:hex, :sext, "1.8.0", "90a95b889f5c781b70bbcf44278b763148e313c376b60d87ce664cb1c1dd29b5", [:rebar3], [], "hexpm", "bc6016cb8690baf677eacacfe6e7cadfec8dc7e286cbbed762f6cd55b0678e73"},
"snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]},
"ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]},
"supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"},
"system_monitor": {:git, "https://github.com/k32/system_monitor.git", "3b4b381bf9503695cd764ecf22067ac6542cee89", [tag: "2.2.1"]},
"trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"},
"typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]},
}

View File

@ -219,6 +219,7 @@ overlay_vars_pkg(bin) ->
, {runner_log_dir, "$RUNNER_ROOT_DIR/log"}
, {runner_data_dir, "$RUNNER_ROOT_DIR/data"}
, {runner_user, ""}
, {is_elixir, "no"}
];
overlay_vars_pkg(pkg) ->
[ {platform_bin_dir, ""}
@ -234,6 +235,7 @@ overlay_vars_pkg(pkg) ->
, {runner_log_dir, "/var/log/emqx"}
, {runner_data_dir, "/var/lib/emqx"}
, {runner_user, "emqx"}
, {is_elixir, "no"}
].
relx_apps(ReleaseType, Edition) ->
@ -248,6 +250,7 @@ relx_apps(ReleaseType, Edition) ->
, inets
, compiler
, runtime_tools
, {hocon, load}
, {emqx, load} % started by emqx_machine
, {emqx_conf, load}
, emqx_machine

5
rel/env.bat.eex Normal file
View File

@ -0,0 +1,5 @@
@echo off
rem Set the release to work across nodes.
rem RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none".
rem set RELEASE_DISTRIBUTION=name
rem set RELEASE_NODE=<%= @release.name %>

17
rel/env.sh.eex Normal file
View File

@ -0,0 +1,17 @@
#!/bin/sh
# Sets and enables heart (recommended only in daemon mode)
# case $RELEASE_COMMAND in
# daemon*)
# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND"
# export HEART_COMMAND
# export ELIXIR_ERL_OPTIONS="-heart"
# ;;
# *)
# ;;
# esac
# Set the release to work across nodes.
# RELEASE_DISTRIBUTION must be "sname" (local), "name" (distributed) or "none".
# export RELEASE_DISTRIBUTION=name
# export RELEASE_NODE=<%= @release.name %>

11
rel/remote.vm.args.eex Normal file
View File

@ -0,0 +1,11 @@
## Customize flags given to the VM: https://erlang.org/doc/man/erl.html
## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
## Number of dirty schedulers doing IO work (file, sockets, and others)
##+SDio 5
## Increase number of concurrent ports/sockets
##+Q 65536
## Tweak GC to run more often
##-env ERL_FULLSWEEP_AFTER 10

11
rel/vm.args.eex Normal file
View File

@ -0,0 +1,11 @@
## Customize flags given to the VM: https://erlang.org/doc/man/erl.html
## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
## Number of dirty schedulers doing IO work (file, sockets, and others)
##+SDio 5
## Increase number of concurrent ports/sockets
##+Q 65536
## Tweak GC to run more often
##-env ERL_FULLSWEEP_AFTER 10

View File

@ -0,0 +1,282 @@
#!/usr/bin/env elixir
defmodule CheckElixirApplications do
@default_applications [:kernel, :stdlib, :sasl]
def main() do
{:ok, _} = Application.ensure_all_started(:mix)
inputs = read_inputs()
# produce `rebar.config.rendered` to consult
profile = profile_of(inputs)
File.cwd!()
|> Path.join("rebar3")
|> System.cmd(["as", to_string(profile)],
env: [{"DEBUG", "1"}]
)
File.cwd!()
|> Path.join("mix.exs")
|> Code.compile_file()
mix_apps = mix_applications(inputs.release_type)
rebar_apps = rebar_applications(profile)
results = diff_apps(mix_apps, rebar_apps)
report_discrepancy(
results[:missing_apps],
"* There are missing applications in the Elixir release",
fn %{app: app, mode: mode, after: last_app} ->
IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}")
end
)
report_discrepancy(
results[:different_modes],
"* There are applications with different application modes in the Elixir release",
fn %{app: app, rebar_mode: rebar_mode, mix_mode: mix_mode} ->
IO.puts(
" * #{inspect(app)} should have mode #{inspect(rebar_mode)}, but it has mode #{inspect(mix_mode)}"
)
end
)
report_discrepancy(
results[:different_positions],
"* There are applications in the Elixir release in the wrong order",
fn %{app: app, mode: mode, after: last_app} ->
IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}")
end
)
success? =
results
|> Map.take([:missing_apps, :different_modes, :different_positions])
|> Map.values()
|> Enum.concat()
|> Enum.empty?()
if not success? do
System.halt(1)
else
IO.puts(
IO.ANSI.green() <>
"Mix and Rebar applications OK!" <>
IO.ANSI.reset()
)
end
end
defp mix_applications(release_type) do
EMQXUmbrella.MixProject.applications(release_type)
end
defp rebar_applications(profile) do
{:ok, props} =
File.cwd!()
|> Path.join("rebar.config.rendered")
|> :file.consult()
props[:profiles][profile][:relx]
|> Enum.find(&(elem(&1, 0) == :release))
|> elem(2)
|> Enum.map(fn
app when is_atom(app) ->
{app, :permanent}
{app, mode} ->
{app, mode}
end)
|> Enum.reject(fn {app, _mode} ->
# Elixir already includes those implicitly
app in @default_applications
end)
end
defp profile_of(%{
release_type: release_type,
package_type: package_type,
edition_type: edition_type
}) do
case {release_type, package_type, edition_type} do
{:cloud, :bin, :community} ->
:emqx
{:cloud, :pkg, :community} ->
:"emqx-pkg"
{:cloud, :bin, :enterprise} ->
:"emqx-enterprise"
{:cloud, :pkg, :enterprise} ->
:"emqx-enterprise-pkg"
{:edge, :bin, :community} ->
:"emqx-edge"
{:edge, :pkg, :community} ->
:"emqx-edge-pkg"
end
end
defp read_inputs() do
release_type =
read_enum_env_var(
"EMQX_RELEASE_TYPE",
[:cloud, :edge],
:cloud
)
package_type =
read_enum_env_var(
"EMQX_PACKAGE_TYPE",
[:bin, :pkg],
:bin
)
edition_type =
read_enum_env_var(
"EMQX_EDITION_TYPE",
[:community, :enterprise],
:community
)
%{
release_type: release_type,
package_type: package_type,
edition_type: edition_type
}
end
defp read_enum_env_var(env_var, allowed_values, default_value) do
case System.fetch_env(env_var) do
:error ->
default_value
{:ok, raw_value} ->
value =
raw_value
|> String.downcase()
|> String.to_atom()
if value not in allowed_values do
Mix.raise("""
Invalid value #{raw_value} for variable #{env_var}.
Allowed values are: #{inspect(allowed_values)}
""")
end
value
end
end
defp diff_apps(mix_apps, rebar_apps) do
app_names = Keyword.keys(rebar_apps)
mix_apps = Keyword.filter(mix_apps, fn {app, _mode} -> app in app_names end)
acc = %{
mix_apps: mix_apps,
missing_apps: [],
different_positions: [],
different_modes: [],
last_app: nil
}
Enum.reduce(
rebar_apps,
acc,
fn
{rebar_app, rebar_mode}, acc = %{mix_apps: [], last_app: last_app} ->
missing_app = %{
app: rebar_app,
mode: rebar_mode,
after: last_app
}
acc
|> Map.update!(:missing_apps, &[missing_app | &1])
|> Map.put(:last_app, rebar_app)
{rebar_app, rebar_mode},
acc = %{mix_apps: [{mix_app, mix_mode} | rest], last_app: last_app} ->
case {rebar_app, rebar_mode} do
{^mix_app, ^mix_mode} ->
acc
|> Map.put(:mix_apps, rest)
|> Map.put(:last_app, rebar_app)
{^mix_app, _mode} ->
different_mode = %{
app: rebar_app,
rebar_mode: rebar_mode,
mix_mode: mix_mode
}
acc
|> Map.put(:mix_apps, rest)
|> Map.update!(:different_modes, &[different_mode | &1])
|> Map.put(:last_app, rebar_app)
{_app, _mode} ->
case Keyword.pop(rest, rebar_app) do
{nil, _} ->
missing_app = %{
app: rebar_app,
mode: rebar_mode,
after: last_app
}
acc
|> Map.update!(:missing_apps, &[missing_app | &1])
|> Map.put(:last_app, rebar_app)
{^rebar_mode, rest} ->
different_position = %{
app: rebar_app,
mode: rebar_mode,
after: last_app
}
acc
|> Map.update!(:different_positions, &[different_position | &1])
|> Map.put(:last_app, rebar_app)
|> Map.put(:mix_apps, [{mix_app, mix_mode} | rest])
{mode, rest} ->
different_mode = %{
app: rebar_app,
rebar_mode: rebar_mode,
mix_mode: mode
}
different_position = %{
app: rebar_app,
mode: rebar_mode,
after: last_app
}
acc
|> Map.put(:mix_apps, [{mix_app, mix_mode} | rest])
|> Map.update!(:different_modes, &[different_mode | &1])
|> Map.update!(:different_positions, &[different_position | &1])
|> Map.put(:last_app, rebar_app)
end
end
end
)
end
defp report_discrepancy(diffs, header, line_fn) do
unless Enum.empty?(diffs) do
IO.puts(IO.ANSI.red() <> header)
diffs
|> Enum.reverse()
|> Enum.each(line_fn)
IO.puts(IO.ANSI.reset())
end
end
end
CheckElixirApplications.main()

View File

@ -0,0 +1,108 @@
#!/usr/bin/env elixir
# ensure we have a fresh rebar.lock
case File.stat("rebar.lock") do
{:ok, _} ->
File.rm!("rebar.lock")
_ ->
:ok
end
{_, 0} =
File.cwd!()
|> Path.join("rebar3")
|> System.cmd(["tree"], into: IO.stream())
{:ok, props} = :file.consult("rebar.lock")
{[{_, rebar_deps}], [props]} = Enum.split_with(props, &is_tuple/1)
# dpendencies declared as package versions have a "secondary index"
pkg_idx =
props
|> Keyword.fetch!(:pkg_hash)
|> Map.new()
rebar_deps =
Map.new(rebar_deps, fn {name, ref, _} ->
ref =
case ref do
{:pkg, _, _} ->
pkg_idx
|> Map.fetch!(name)
|> String.downcase()
{:git, _, {:ref, ref}} ->
to_string(ref)
end
{name, ref}
end)
{mix_deps, []} = Code.eval_file("mix.lock")
mix_deps =
Map.new(mix_deps, fn {name, ref} ->
ref =
case ref do
{:git, _, ref, _} ->
ref
{:hex, _, _, ref, _, _, _, _} ->
ref
end
{to_string(name), ref}
end)
diffs =
Enum.reduce(rebar_deps, %{}, fn {name, rebar_ref}, acc ->
mix_ref = mix_deps[name]
cond do
mix_ref && mix_ref != rebar_ref ->
Map.put(acc, name, {rebar_ref, mix_ref})
is_nil(mix_ref) ->
Map.put(acc, name, {rebar_ref, nil})
:otherwise ->
acc
end
end)
if diffs == %{} do
IO.puts(
IO.ANSI.green() <>
"* Mix and Rebar3 dependencies OK!" <>
IO.ANSI.reset()
)
System.halt(0)
else
IO.puts(
IO.ANSI.red() <>
"* Discrepancies between Elixir and Rebar3 dependencies found!" <>
IO.ANSI.reset()
)
Enum.each(diffs, fn {name, {rebar_ref, mix_ref}} ->
IO.puts(
IO.ANSI.red() <>
" * #{name}\n" <>
" * Rebar3 ref: #{rebar_ref}\n" <>
" * Mix ref: #{mix_ref}\n" <>
IO.ANSI.reset()
)
end)
IO.puts(
IO.ANSI.red() <>
"Update `mix.exs` to match Rebar3's references (use `overwrite: true` if necessary) and try again" <>
IO.ANSI.reset()
)
System.halt(1)
end

View File

@ -3,7 +3,10 @@
set -euo pipefail
target_files=()
while IFS='' read -r line; do target_files+=("$line"); done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .)
while IFS='' read -r line;
do
target_files+=("$line");
done < <(git grep -r -l "^#!/bin/" .)
return_code=0
for i in "${target_files[@]}"; do
echo checking "$i" ...