From 516c52bdc783a7f044846adb0c920d7bc262425c Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 30 Apr 2023 10:48:20 +0200
Subject: [PATCH 1/4] build: add a 'app' build target which only compiles the
 code

but no release
---
 Makefile |  5 +++++
 build    | 12 +++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index a39f44c07..e60e8e492 100644
--- a/Makefile
+++ b/Makefile
@@ -140,6 +140,11 @@ COMMON_DEPS := $(REBAR)
 $(REL_PROFILES:%=%): $(COMMON_DEPS)
 	@$(BUILD) $(@) rel
 
+.PHONY: compile $(PROFILES:%=compile-%)
+compile: $(PROFILES:%=compile-%)
+$(PROFILES:%=compile-%):
+	@$(BUILD) $(@:compile-%=%) apps
+
 ## Not calling rebar3 clean because
 ## 1. rebar3 clean relies on rebar3, meaning it reads config, fetches dependencies etc.
 ## 2. it's slow
diff --git a/build b/build
index eb75317cc..2924f8a6f 100755
--- a/build
+++ b/build
@@ -124,16 +124,19 @@ assert_no_compile_time_only_deps() {
     :
 }
 
-make_rel() {
-    local release_or_tar="${1}"
+just_compile() {
     ./scripts/pre-compile.sh "$PROFILE"
     # make_elixir_rel always create rebar.lock
     # delete it to make git clone + checkout work because we use shallow close for rebar deps
     rm -f rebar.lock
     # compile all beams
     ./rebar3 as "$PROFILE" compile
-    # generate docs (require beam compiled), generated to etc and priv dirs
     make_docs
+}
+
+make_rel() {
+    local release_or_tar="${1}"
+    just_compile
     # now assemble the release tar
     ./rebar3 as "$PROFILE" "$release_or_tar"
     assert_no_compile_time_only_deps
@@ -375,6 +378,9 @@ export_elixir_release_vars() {
 log "building artifact=$ARTIFACT for profile=$PROFILE"
 
 case "$ARTIFACT" in
+    apps)
+        just_compile
+        ;;
     doc|docs)
         make_docs
         ;;

From dbcb75f35bae1d4a00122b3244f2123cdb1051fb Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 1 May 2023 21:34:33 +0200
Subject: [PATCH 2/4] build: add ./dev

---
 dev | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 250 insertions(+)
 create mode 100755 dev

diff --git a/dev b/dev
new file mode 100755
index 000000000..69706bf6d
--- /dev/null
+++ b/dev
@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+usage() {
+cat <<EOF
+
+Run EMQX without building a release (which takes longer time).
+
+USAGE: $0 [OPTION]
+
+OPTIONS:
+  -h Print this help usage info.
+  -p 'emqx' or 'emqx-enterprise', defaults to 'PROFILE' env.
+  -c Force recompile, otherwise starts with the already built libs
+     in '_build/\$PROFILE/lib/'.
+
+ENVIRONMENT VARIABLES:
+  PROFILE: Overriden by -p option, defaults to 'emqx'.
+  EMQX_NODE_NAME: The node name of the EMQX node. Default to emqx@127.0.0.1'.
+
+EOF
+}
+
+if [ -n "${DEBUG:-}" ]; then
+    set -x
+fi
+
+export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
+EMQX_NODE_NAME="${EMQX_NODE_NAME:-emqx@127.0.0.1}"
+PROFILE="${PROFILE:-emqx}"
+FORCE_COMPILE=0
+while getopts ":p:ch" opt; do
+    case "${opt}" in
+        h)
+            usage
+            exit 0
+            ;;
+        p)
+            PROFILE="${OPTARG}"
+            ;;
+        c)
+            FORCE_COMPILE=1
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG" >&2
+            exit 1
+            ;;
+        :)
+            echo "Option -$OPTARG requires an argument." >&2
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+case "${PROFILE}" in
+    ce|emqx)
+        PROFILE='emqx'
+        ;;
+    ee|emqx-enterprise)
+        PROFILE='emqx-enterprise'
+        ;;
+    *)
+        echo "Unknown profile $PROFILE"
+        exit 1
+        ;;
+esac
+export PROFILE
+
+case "${PROFILE}" in
+    emqx)
+        SCHEMA_MOD='emqx_conf_schema'
+        ;;
+    emqx-enterprise)
+        SCHEMA_MOD='emqx_ee_conf_schema'
+        ;;
+esac
+
+PROJ_ROOT="$(git rev-parse --show-toplevel)"
+cd "$PROJ_ROOT"
+BASE_DIR="_build/dev-run/$PROFILE"
+export EMQX_ETC_DIR="$BASE_DIR/etc"
+export EMQX_DATA_DIR="$BASE_DIR/data"
+export EMQX_LOG_DIR="$BASE_DIR/log"
+CONFIGS_DIR="$EMQX_DATA_DIR/configs"
+COOKIE='emqxsecretcookie'
+mkdir -p "$EMQX_ETC_DIR" "$EMQX_DATA_DIR/patches" "$EMQX_LOG_DIR" "$CONFIGS_DIR"
+
+## build compile the profile is it's not compiled yet
+prepare_erl_libs() {
+    local profile="$1"
+    local libs_dir="_build/${profile}/lib"
+    local erl_libs=''
+    if [ $FORCE_COMPILE -eq 1 ] || [ ! -d "$libs_dir" ]; then
+        make "compile-${PROFILE}"
+    else
+        echo "Running from code in $libs_dir"
+    fi
+    for app in "${libs_dir}"/*; do
+        erl_libs="${erl_libs}:${app}"
+    done
+    export ERL_LIBS="$erl_libs"
+}
+
+## poorman's mustache templating
+mustache() {
+    local name="$1"
+    local value="$2"
+    local file="$3"
+    sed -i "s|{{\s*${name}\s*}}|${value}|g" "$file"
+}
+
+## render the merged boot conf file.
+## the merge action is done before the profile is compiled
+render_hocon_conf() {
+    input="apps/emqx_conf/etc/emqx.conf.all"
+    output="$EMQX_ETC_DIR/emqx.conf"
+    cp "$input" "$output"
+    mustache emqx_default_erlang_cookie "$COOKIE" "$output"
+    mustache platform_data_dir "${EMQX_DATA_DIR}" "$output"
+    mustache platform_log_dir "${EMQX_LOG_DIR}" "$output"
+    mustache platform_etc_dir "${EMQX_ETC_DIR}" "$output"
+}
+
+call_hocon() {
+    local in=("$@")
+    local args=''
+    for arg in "${in[@]}"; do
+        if [ -z "$args" ]; then
+            args="\"$arg\""
+        else
+            args="$args, \"$arg\""
+        fi
+    done
+    erl -noshell -eval "{ok, _} = application:ensure_all_started(hocon), ok = hocon_cli:main([$args]), init:stop()."
+}
+
+# Function to generate app.config and vm.args
+# sets two environment variables CONF_FILE and ARGS_FILE
+generate_app_conf() {
+    ## timestamp for each generation
+    local NOW_TIME
+    NOW_TIME="$(date +'%Y.%m.%d.%H.%M.%S')"
+
+    ## this command populates two files: app.<time>.config and vm.<time>.args
+    ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
+    ## but does not include the cluster-override.conf and local-override.conf
+    ## meaning, certain overrides will not be mapped to app.<time>.config file
+    call_hocon -v -t "$NOW_TIME" -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf -d "$EMQX_DATA_DIR"/configs generate
+
+    ## filenames are per-hocon convention
+    CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
+    ARGS_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args"
+}
+
+# apps/emqx/etc/vm.args.cloud
+append_args_file() {
+    ## ensure a new line at the end
+    echo '' >> "$ARGS_FILE"
+    cat <<EOF >> "$ARGS_FILE"
+-name $EMQX_NODE_NAME
+-mnesia dir '"$EMQX_DATA_DIR/mnesia/$EMQX_NODE_NAME"'
+-stdlib restricted_shell emqx_restricted_shell
++spp true
++A 4
++IOt 4
++SDio 8
+-shutdown_time 30000
+-pa '"$EMQX_DATA_DIR/patches"'
+-mnesia dump_log_write_threshold 5000
+-mnesia dump_log_time_threshold 60000
+-os_mon start_disksup false
+EOF
+}
+
+# copy cert files and acl.conf to etc
+copy_other_conf_files() {
+    cp -r apps/emqx/etc/certs "$EMQX_ETC_DIR"/
+    cp apps/emqx_authz/etc/acl.conf "$EMQX_ETC_DIR"/
+}
+
+is_current_profile_app() {
+    local app="$1"
+    case "$app" in
+        lib-ee*)
+            if [ "$PROFILE" = 'emqx-enterprise' ]; then
+                return 0
+            else
+                return 1
+            fi
+            ;;
+        *)
+            if [ "$PROFILE" = 'emqx' ]; then
+                if [ -f "$app"/BSL.txt ]; then
+                    return 1
+                else
+                    return 0
+                fi
+            else
+                return 0
+            fi
+            ;;
+    esac
+}
+
+## apps to load
+apps_to_load() {
+    local apps csl
+    apps="$(./scripts/find-apps.sh | xargs)"
+    csl=""
+    for app in $apps; do
+        if ! is_current_profile_app "$app"; then
+            continue
+        fi
+        name="$(basename "$app")"
+        if [ -z "$csl" ]; then
+            csl="$name"
+        else
+            csl="$csl,$name"
+        fi
+    done
+    echo "$csl"
+}
+
+## Make erl command aware where to load all the beams
+## this should be done before every erl command
+prepare_erl_libs "$PROFILE"
+render_hocon_conf
+generate_app_conf
+append_args_file
+copy_other_conf_files
+APPS="$(apps_to_load)"
+
+
+BOOT_SEQUENCE="
+    Apps=[${APPS}],
+    ok=lists:foreach(fun application:load/1, Apps),
+    io:format(user, \"~nLoaded ~p apps~n\", [length(Apps)]),
+    application:ensure_all_started(emqx_machine).
+"
+
+erl -name "$EMQX_NODE_NAME" \
+    -start_epmd false \
+    -epmd_module ekka_epmd \
+    -proto_dist ekka \
+    -args_file "$ARGS_FILE" \
+    -config "$CONF_FILE" \
+    -s emqx_restricted_shell set_prompt_func \
+    -eval "$BOOT_SEQUENCE"

From e6b947c02bb90c2119d8858fa457755c765609de Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 1 May 2023 22:51:03 +0200
Subject: [PATCH 3/4] build: change 'make run' to use ./dev -p $(PROFILE)

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index e60e8e492..d1a478279 100644
--- a/Makefile
+++ b/Makefile
@@ -228,11 +228,11 @@ endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
 .PHONY: run
-run: $(PROFILE) quickrun
+run: compile-$(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
-	./_build/$(PROFILE)/rel/emqx/bin/emqx console
+	./dev -p $(PROFILE)
 
 ## Take the currently set PROFILE
 docker:

From 7a549d71efa9efa95dbda4b729f258bf911885eb Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 2 May 2023 10:01:17 +0200
Subject: [PATCH 4/4] chore: make it easier for IDE to connect

1. By default, start the node with regular EPMD, not ekka_epmd
2. Use user's Erlang cookie
3. Add a -r option to attach remsh
---
 bin/emqx |   6 +--
 dev      | 139 ++++++++++++++++++++++++++++++++++++++-----------------
 2 files changed, 99 insertions(+), 46 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index fc0124b96..6d4c5cd4e 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -396,7 +396,7 @@ relx_get_pid() {
 remsh() {
     # Generate a unique id used to allow multiple remsh to the same node
     # transparently
-    id="remsh$(relx_gen_id)-${NAME}"
+    id="remsh$(gen_node_id)-${NAME}"
 
     # shellcheck disable=SC2086
     # Setup remote shell command to control node
@@ -424,7 +424,7 @@ remsh() {
 }
 
 # Generate a random id
-relx_gen_id() {
+gen_node_id() {
     od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}'
 }
 
@@ -1273,7 +1273,7 @@ case "${COMMAND}" in
         then
           "$REL_DIR/elixir" \
               --hidden \
-              --name "rand-$(relx_gen_id)-$NAME" \
+              --name "rand-$(gen_node_id)-$NAME" \
               --cookie "$COOKIE" \
               --boot "$REL_DIR/start_clean" \
               --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
diff --git a/dev b/dev
index 69706bf6d..39bdfbc41 100755
--- a/dev
+++ b/dev
@@ -2,22 +2,33 @@
 
 set -euo pipefail
 
+PROJ_ROOT="$(git rev-parse --show-toplevel)"
+cd "$PROJ_ROOT"
+
 usage() {
 cat <<EOF
 
 Run EMQX without building a release (which takes longer time).
+Node state is stored in '_build/dev-run/$PROFILE'.
+The node is started in interactive mode without a boot file.
 
 USAGE: $0 [OPTION]
 
 OPTIONS:
-  -h Print this help usage info.
-  -p 'emqx' or 'emqx-enterprise', defaults to 'PROFILE' env.
-  -c Force recompile, otherwise starts with the already built libs
-     in '_build/\$PROFILE/lib/'.
+  -h|--help:         Print this help usage info.
+  -p|--profile:      emqx | emqx-enterprise, defaults to 'PROFILE' env.
+  -c|--compile:      Force recompile, otherwise starts with the already built libs
+                     in '_build/\$PROFILE/lib/'.
+  -e|--ekka-epmd:    Force to use ekka_epmd.
+  -n|--name:         Node name, defaults to \$EMQX_NODE_NAME env.
+  -r|--remsh [NAME]: Attach to running node's remote console.
 
 ENVIRONMENT VARIABLES:
-  PROFILE: Overriden by -p option, defaults to 'emqx'.
-  EMQX_NODE_NAME: The node name of the EMQX node. Default to emqx@127.0.0.1'.
+
+  PROFILE:           Overriden by '-p|--profile' option, defaults to 'emqx'.
+  EMQX_NODE_NAME:    Overriden by '-n|--name' or '-r|--remsh' option.
+                     The node name of the EMQX node. Default to emqx@127.0.0.1'.
+  EMQX_NODE_COOKIE:  Erlang cookie, defaults to ~/.erlang.cookie
 
 EOF
 }
@@ -30,29 +41,43 @@ export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
 EMQX_NODE_NAME="${EMQX_NODE_NAME:-emqx@127.0.0.1}"
 PROFILE="${PROFILE:-emqx}"
 FORCE_COMPILE=0
-while getopts ":p:ch" opt; do
-    case "${opt}" in
-        h)
+# Do not start using ekka epmd by default, so your IDE can connect to it
+EKKA_EPMD=0
+REMSH=0
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
             usage
             exit 0
             ;;
-        p)
-            PROFILE="${OPTARG}"
+        -n|--name)
+            EMQX_NODE_NAME="$2"
+            shift 1
             ;;
-        c)
+        -r|--remsh)
+            REMSH=1
+            if [[ $2 == *@* ]]; then
+                EMQX_NODE_NAME="$2"
+                shift 1
+            fi
+            ;;
+        -p|--profile)
+            PROFILE="${2}"
+            shift 1;
+            ;;
+        -c|--compile)
             FORCE_COMPILE=1
             ;;
-        \?)
-            echo "Invalid option: -$OPTARG" >&2
-            exit 1
+        -e|--ekka-epmd)
+            EKKA_EPMD=1
             ;;
-        :)
-            echo "Option -$OPTARG requires an argument." >&2
+        *)
+            echo "Unknown argument $1" >&2
             exit 1
             ;;
     esac
+    shift 1;
 done
-shift $((OPTIND-1))
 
 case "${PROFILE}" in
     ce|emqx)
@@ -77,15 +102,20 @@ case "${PROFILE}" in
         ;;
 esac
 
-PROJ_ROOT="$(git rev-parse --show-toplevel)"
-cd "$PROJ_ROOT"
 BASE_DIR="_build/dev-run/$PROFILE"
 export EMQX_ETC_DIR="$BASE_DIR/etc"
 export EMQX_DATA_DIR="$BASE_DIR/data"
 export EMQX_LOG_DIR="$BASE_DIR/log"
 CONFIGS_DIR="$EMQX_DATA_DIR/configs"
-COOKIE='emqxsecretcookie'
+# Use your cookie so your IDE can connect to it.
+COOKIE="${EMQX_NODE__COOKIE:-${EMQX_NODE_COOKIE:-$(cat ~/.erlang.cookie || echo 'emqxsecretcookie')}}"
 mkdir -p "$EMQX_ETC_DIR" "$EMQX_DATA_DIR/patches" "$EMQX_LOG_DIR" "$CONFIGS_DIR"
+if [ $EKKA_EPMD -eq 1 ]; then
+    EPMD_ARGS='-start_epmd false -epmd_module ekka_epmd'
+else
+    EPMD_ARGS=''
+fi
+
 
 ## build compile the profile is it's not compiled yet
 prepare_erl_libs() {
@@ -223,28 +253,51 @@ apps_to_load() {
     echo "$csl"
 }
 
-## Make erl command aware where to load all the beams
-## this should be done before every erl command
-prepare_erl_libs "$PROFILE"
-render_hocon_conf
-generate_app_conf
-append_args_file
-copy_other_conf_files
-APPS="$(apps_to_load)"
+boot() {
+    ## Make erl command aware where to load all the beams
+    ## this should be done before every erl command
+    prepare_erl_libs "$PROFILE"
+    render_hocon_conf
+    generate_app_conf
+    append_args_file
+    copy_other_conf_files
+    APPS="$(apps_to_load)"
 
 
-BOOT_SEQUENCE="
-    Apps=[${APPS}],
-    ok=lists:foreach(fun application:load/1, Apps),
-    io:format(user, \"~nLoaded ~p apps~n\", [length(Apps)]),
-    application:ensure_all_started(emqx_machine).
-"
+    BOOT_SEQUENCE="
+        Apps=[${APPS}],
+        ok=lists:foreach(fun application:load/1, Apps),
+        io:format(user, \"~nLoaded ~p apps~n\", [length(Apps)]),
+        application:ensure_all_started(emqx_machine).
+    "
 
-erl -name "$EMQX_NODE_NAME" \
-    -start_epmd false \
-    -epmd_module ekka_epmd \
-    -proto_dist ekka \
-    -args_file "$ARGS_FILE" \
-    -config "$CONF_FILE" \
-    -s emqx_restricted_shell set_prompt_func \
-    -eval "$BOOT_SEQUENCE"
+    # shellcheck disable=SC2086
+    erl -name "$EMQX_NODE_NAME" \
+        $EPMD_ARGS \
+        -proto_dist ekka \
+        -args_file "$ARGS_FILE" \
+        -config "$CONF_FILE" \
+        -s emqx_restricted_shell set_prompt_func \
+        -eval "$BOOT_SEQUENCE"
+}
+
+# Generate a random id
+gen_node_id() {
+    od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}'
+}
+
+remsh() {
+    id="remsh$(gen_node_id)-${EMQX_NODE_NAME}"
+    # shellcheck disable=SC2086
+    erl -name "$id" \
+        -setcookie "$COOKIE" \
+        -hidden \
+        -remsh "$EMQX_NODE_NAME" \
+        $EPMD_ARGS
+}
+
+if [ $REMSH -eq 0 ]; then
+    boot
+else
+    remsh
+fi