Merge pull request #8712 from zmstone/0812-refactor-makefile-to-download-dashboard-per-profile

0812 refactor makefile to download dashboard per profile

.github/workflows/run_test_cases.yaml

@@ -145,6 +145,10 @@ jobs:
           fail-fast: false
           matrix:
             app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
+            profile:
+              - emqx
+              - emqx-enterprise
+
         runs-on: aws-amd64
         container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04"
         defaults:
@@ -163,13 +167,35 @@ jobs:
           # produces <app-name>.coverdata
         - name: run common test
           working-directory: source
+          env:
+            PROFILE: ${{ matrix.profile }}
+            WHICH_APP: ${{ matrix.app_name }}
           run: |
-            make ${{ matrix.app_name }}-ct
+            if [ "$PROFILE" = 'emqx-enterprise' ]; then
-        - uses: actions/upload-artifact@v1
+              COMPILE_FLAGS="$(grep -R "EMQX_RELEASE_EDITION" "$WHICH_APP" | wc -l || true)"
+              if [ "$COMPILE_FLAGS" -gt 0 ]; then
+                # need to clean first because the default profile was
+                make clean
+                make "${WHICH_APP}-ct"
+              else
+                echo "skip_common_test_run_for_app ${WHICH_APP}-ct"
+              fi
+            else
+              case "$WHICH_APP" in
+                lib-ee/*)
+                  echo "skip_opensource_edition_test_for_lib-ee"
+                  ;;
+                *)
+                  make "${WHICH_APP}-ct"
+                  ;;
+              esac
+            fi
+        - uses: actions/upload-artifact@v3
           with:
             name: coverdata
             path: source/_build/test/cover
-        - uses: actions/upload-artifact@v1
+            if-no-files-found: warn # do not fail if no coverdata found
+        - uses: actions/upload-artifact@v3
           if: failure()
           with:
             name: logs_${{ matrix.otp_release }}

Makefile

@@ -1,4 +1,3 @@
-$(shell $(CURDIR)/scripts/git-hooks-init.sh)
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
@@ -8,6 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.0.6
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)
@@ -30,6 +30,13 @@ export REBAR_GIT_CLONE_OPTIONS += --depth=1
 .PHONY: default
 default: $(REBAR) $(PROFILE)
 
+.PHONY: prepare
+prepare: FORCE
+	@$(SCRIPTS)/git-hooks-init.sh # this is no longer needed since 5.0 but we keep it anyway
+	@$(SCRIPTS)/prepare-build-deps.sh
+
+FORCE:
+
 .PHONY: all
 all: $(REBAR) $(PROFILES)
 
@@ -53,11 +60,7 @@ ensure-mix-rebar: $(REBAR)
 mix-deps-get: $(ELIXIR_COMMON_DEPS)
 	@mix deps.get
 
-$(REBAR): ensure-rebar3
+$(REBAR): prepare ensure-rebar3
-
-.PHONY: get-dashboard
-get-dashboard:
-	@$(SCRIPTS)/get-dashboard.sh
 
 .PHONY: eunit
 eunit: $(REBAR) conf-segs
@@ -75,13 +78,14 @@ ct: $(REBAR) conf-segs
 static_checks:
 	@$(REBAR) as check do dialyzer, xref, ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE)
 
-APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
+APPS=$(shell $(SCRIPTS)/find-apps.sh)
 
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct: $(REBAR) conf-segs
+$1-ct: $(REBAR)
-	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+	@$(SCRIPTS)/pre-compile.sh $(PROFILE)
+	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 
@@ -89,7 +93,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 .PHONY: $(APPS:%=%-prop)
 define gen-app-prop-target
 $1-prop:
-	$(REBAR) proper -d test/props -v -m $(shell $(CURDIR)/scripts/find-props.sh $1)
+	$(REBAR) proper -d test/props -v -m $(shell $(SCRIPTS)/find-props.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app))))
 
@@ -111,7 +115,8 @@ cover: $(REBAR)
 coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
-COMMON_DEPS := $(REBAR) prepare-build-deps get-dashboard conf-segs
+COMMON_DEPS := $(REBAR)
+
 ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
 
 .PHONY: $(REL_PROFILES)
@@ -147,6 +152,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
 ## which may not have the right credentials
 .PHONY: $(PROFILES:%=deps-%)
 $(PROFILES:%=deps-%): $(COMMON_DEPS)
+	@$(SCRIPTS)/pre-compile.sh $(@:deps-%=%)
 	@$(REBAR) as $(@:deps-%=%) get-deps
 	@rm -f rebar.lock
 
@@ -167,7 +173,7 @@ $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 .PHONY: $(REL_PROFILES:%=%-relup-downloads)
 define download-relup-packages
 $1-relup-downloads:
-	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-build/download-base-packages.sh $1; fi
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(SCRIPTS)/relup-build/download-base-packages.sh $1; fi
 endef
 ALL_ZIPS = $(REL_PROFILES)
 $(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
@@ -216,11 +222,8 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
 
 .PHONY:
 conf-segs:
-	@scripts/merge-config.escript
+	@$(SCRIPTS)/merge-config.escript
-	@scripts/merge-i18n.escript
+	@$(SCRIPTS)/merge-i18n.escript
-
-prepare-build-deps:
-	@scripts/prepare-build-deps.sh
 
 ## elixir target is to create release packages using Elixir's Mix
 .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)
@@ -247,6 +250,6 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))
 
 .PHONY: fmt
 fmt: $(REBAR)
-	@./scripts/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}'
+	@$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}'
-	@./scripts/erlfmt -w 'rebar.config.erl'
+	@$(SCRIPTS)/erlfmt -w 'rebar.config.erl'
 	@mix format

build

@@ -14,9 +14,18 @@ if [ "$DEBUG" -eq 1 ]; then
     set -x
 fi
 
-PROFILE="$1"
+PROFILE_ARG="$1"
 ARTIFACT="$2"
 
+if [[ "${PROFILE:-${PROFILE_ARG}}" != "$PROFILE_ARG" ]]; then
+    echo "PROFILE env var is set to '$PROFILE', but '$0' arg1 is '$1'"
+    exit 1
+fi
+
+# make sure PROFILE is exported, it is needed by rebar.config.erl
+PROFILE=$PROFILE_ARG
+export PROFILE
+
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
@@ -106,6 +115,7 @@ assert_no_compile_time_only_deps() {
 }
 
 make_rel() {
+    ./scripts/pre-compile.sh "$PROFILE"
     # compile all beams
     ./rebar3 as "$PROFILE" compile
     # generate docs (require beam compiled), generated to etc and priv dirs
@@ -116,6 +126,7 @@ make_rel() {
 }
 
 make_elixir_rel() {
+  ./scripts/pre-compile.sh "$PROFILE"
   export_release_vars "$PROFILE"
   mix release --overwrite
   assert_no_compile_time_only_deps

deploy/docker/Dockerfile

@@ -7,14 +7,15 @@ COPY . /emqx
 ARG EMQX_NAME=emqx
 ENV EMQX_RELUP=false
 
-RUN export PROFILE="$EMQX_NAME" \
+RUN export PROFILE=${EMQX_NAME%%-elixir} \
-    && export EMQX_NAME=${EMQX_NAME%%-elixir} \
+    && export EMQX_NAME1=$EMQX_NAME \
+    && export EMQX_NAME=$PROFILE \
     && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \
     && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
     && export EMQX_REL_FORM='docker' \
     && cd /emqx \
     && rm -rf $EMQX_LIB_PATH \
-    && make $PROFILE \
+    && make $EMQX_NAME1 \
     && mkdir -p /emqx-rel \
     && mv $EMQX_REL_PATH /emqx-rel
 

deploy/docker/Dockerfile.alpine

@@ -28,14 +28,15 @@ COPY . /emqx
 ARG EMQX_NAME=emqx
 ENV EMQX_RELUP=false
 
-RUN export PROFILE="$EMQX_NAME" \
+RUN export PROFILE=${EMQX_NAME%%-elixir} \
-    && export EMQX_NAME=${EMQX_NAME%%-elixir} \
+    && export EMQX_NAME1=$EMQX_NAME \
+    && export EMQX_NAME=$PROFILE \
     && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \
     && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
     && export EMQX_REL_FORM='docker' \
     && cd /emqx \
     && rm -rf $EMQX_LIB_PATH \
-    && make $PROFILE \
+    && make $EMQX_NAME1 \
     && mkdir -p /emqx-rel \
     && mv $EMQX_REL_PATH /emqx-rel
 

rebar.config.erl

@@ -4,6 +4,7 @@
 
 do(Dir, CONFIG) ->
     ok = assert_otp(),
+    ok = warn_profile_env(),
     case iolist_to_binary(Dir) of
         <<".">> ->
             C1 = deps(CONFIG),
@@ -117,6 +118,9 @@ is_raspbian() ->
 is_win32() ->
     win32 =:= element(1, os:type()).
 
+project_app_dirs() ->
+    project_app_dirs(get_edition_from_profille_env()).
+
 project_app_dirs(Edition) ->
     ["apps/*"] ++
         case is_enterprise(Edition) of
@@ -149,6 +153,9 @@ test_deps() ->
         {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}
     ].
 
+common_compile_opts(Vsn) ->
+    common_compile_opts(get_edition_from_profille_env(), Vsn).
+
 common_compile_opts(Edition, Vsn) ->
     % always include debug_info
     [
@@ -159,6 +166,36 @@ common_compile_opts(Edition, Vsn) ->
         [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++
         [{d, 'BUILD_WITHOUT_QUIC'} || not is_quicer_supported()].
 
+warn_profile_env() ->
+    case os:getenv("PROFILE") of
+        false ->
+            io:format(
+                standard_error,
+                "WARN: environment variable PROFILE is not set, using 'emqx-enterprise'~n",
+                []
+            );
+        _ ->
+            ok
+    end.
+
+%% this function is only used for test/check profiles
+get_edition_from_profille_env() ->
+    case os:getenv("PROFILE") of
+        "emqx" ->
+            ce;
+        "emqx-" ++ _ ->
+            ce;
+        "emqx-enterprise" ->
+            ee;
+        "emqx-enterprise-" ++ _ ->
+            ee;
+        false ->
+            ee;
+        V ->
+            io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]),
+            exit(bad_PROFILE)
+    end.
+
 prod_compile_opts(Edition, Vsn) ->
     [
         compressed,
@@ -212,14 +249,14 @@ profiles_dev() ->
     Vsn = get_vsn('emqx-enterprise'),
     [
         {check, [
-            {erl_opts, common_compile_opts(ee, Vsn)},
+            {erl_opts, common_compile_opts(Vsn)},
-            {project_app_dirs, project_app_dirs(ee)}
+            {project_app_dirs, project_app_dirs()}
         ]},
         {test, [
             {deps, test_deps()},
-            {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()},
+            {erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()},
             {extra_src_dirs, [{"test", [{recursive, true}]}]},
-            {project_app_dirs, project_app_dirs(ee)}
+            {project_app_dirs, project_app_dirs()}
         ]}
     ].
 

scripts/get-dashboard.sh

@@ -5,8 +5,20 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 
+VERSION="${1}"
+case "$VERSION" in
+    v*)
         RELEASE_ASSET_FILE="emqx-dashboard.zip"
-VERSION="${EMQX_DASHBOARD_VERSION}"
+        ;;
+    e*)
+        RELEASE_ASSET_FILE="emqx-enterprise-dashboard.zip"
+        ;;
+    *)
+        echo "Unknown version $VERSION"
+        exit 1
+        ;;
+esac
+
 DASHBOARD_PATH='apps/emqx_dashboard/priv'
 DASHBOARD_REPO='emqx-dashboard-web-new'
 DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}"

scripts/pre-compile.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise)
+# it might be with suffix such as -pkg etc.
+PROFILE_STR="${1}"
+
+case "$PROFILE_STR" in
+    *enterprise*)
+        dashboard_version="$EMQX_EE_DASHBOARD_VERSION"
+        ;;
+    *)
+        dashboard_version="$EMQX_DASHBOARD_VERSION"
+        ;;
+esac
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+./scripts/get-dashboard.sh "$dashboard_version"
+./scripts/merge-config.escript
+./scripts/merge-i18n.escript