diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e08e3906b..3b65fe885 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -145,6 +145,10 @@ jobs: fail-fast: false matrix: app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} + profile: + - emqx + - emqx-enterprise + runs-on: aws-amd64 container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" defaults: @@ -163,13 +167,35 @@ jobs: # produces .coverdata - name: run common test working-directory: source + env: + PROFILE: ${{ matrix.profile }} + WHICH_APP: ${{ matrix.app_name }} run: | - make ${{ matrix.app_name }}-ct - - uses: actions/upload-artifact@v1 + if [ "$PROFILE" = 'emqx-enterprise' ]; then + COMPILE_FLAGS="$(grep -R "EMQX_RELEASE_EDITION" "$WHICH_APP" | wc -l || true)" + if [ "$COMPILE_FLAGS" -gt 0 ]; then + # need to clean first because the default profile was + make clean + make "${WHICH_APP}-ct" + else + echo "skip_common_test_run_for_app ${WHICH_APP}-ct" + fi + else + case "$WHICH_APP" in + lib-ee/*) + echo "skip_opensource_edition_test_for_lib-ee" + ;; + *) + make "${WHICH_APP}-ct" + ;; + esac + fi + - uses: actions/upload-artifact@v3 with: name: coverdata path: source/_build/test/cover - - uses: actions/upload-artifact@v1 + if-no-files-found: warn # do not fail if no coverdata found + - uses: actions/upload-artifact@v3 if: failure() with: name: logs_${{ matrix.otp_release }} diff --git a/Makefile b/Makefile index 49c2b1ed7..2c3077acc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -$(shell $(CURDIR)/scripts/git-hooks-init.sh) REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -8,6 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.0.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) @@ -30,6 +30,13 @@ export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default default: $(REBAR) $(PROFILE) +.PHONY: prepare +prepare: FORCE + @$(SCRIPTS)/git-hooks-init.sh # this is no longer needed since 5.0 but we keep it anyway + @$(SCRIPTS)/prepare-build-deps.sh + +FORCE: + .PHONY: all all: $(REBAR) $(PROFILES) @@ -53,11 +60,7 @@ ensure-mix-rebar: $(REBAR) mix-deps-get: $(ELIXIR_COMMON_DEPS) @mix deps.get -$(REBAR): ensure-rebar3 - -.PHONY: get-dashboard -get-dashboard: - @$(SCRIPTS)/get-dashboard.sh +$(REBAR): prepare ensure-rebar3 .PHONY: eunit eunit: $(REBAR) conf-segs @@ -75,13 +78,14 @@ ct: $(REBAR) conf-segs static_checks: @$(REBAR) as check do dialyzer, xref, ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE) -APPS=$(shell $(CURDIR)/scripts/find-apps.sh) +APPS=$(shell $(SCRIPTS)/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: $(REBAR) conf-segs - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) +$1-ct: $(REBAR) + @$(SCRIPTS)/pre-compile.sh $(PROFILE) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -89,7 +93,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) .PHONY: $(APPS:%=%-prop) define gen-app-prop-target $1-prop: - $(REBAR) proper -d test/props -v -m $(shell $(CURDIR)/scripts/find-props.sh $1) + $(REBAR) proper -d test/props -v -m $(shell $(SCRIPTS)/find-props.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) @@ -111,7 +115,8 @@ cover: $(REBAR) coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send -COMMON_DEPS := $(REBAR) prepare-build-deps get-dashboard conf-segs +COMMON_DEPS := $(REBAR) + ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar .PHONY: $(REL_PROFILES) @@ -147,6 +152,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%) ## which may not have the right credentials .PHONY: $(PROFILES:%=deps-%) $(PROFILES:%=deps-%): $(COMMON_DEPS) + @$(SCRIPTS)/pre-compile.sh $(@:deps-%=%) @$(REBAR) as $(@:deps-%=%) get-deps @rm -f rebar.lock @@ -167,7 +173,7 @@ $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS) .PHONY: $(REL_PROFILES:%=%-relup-downloads) define download-relup-packages $1-relup-downloads: - @if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-build/download-base-packages.sh $1; fi + @if [ "$${EMQX_RELUP}" = "true" ]; then $(SCRIPTS)/relup-build/download-base-packages.sh $1; fi endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt)))) @@ -216,11 +222,8 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) .PHONY: conf-segs: - @scripts/merge-config.escript - @scripts/merge-i18n.escript - -prepare-build-deps: - @scripts/prepare-build-deps.sh + @$(SCRIPTS)/merge-config.escript + @$(SCRIPTS)/merge-i18n.escript ## elixir target is to create release packages using Elixir's Mix .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) @@ -247,6 +250,6 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) .PHONY: fmt fmt: $(REBAR) - @./scripts/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' - @./scripts/erlfmt -w 'rebar.config.erl' + @$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' + @$(SCRIPTS)/erlfmt -w 'rebar.config.erl' @mix format diff --git a/build b/build index 07c16b69e..99728d03e 100755 --- a/build +++ b/build @@ -14,9 +14,18 @@ if [ "$DEBUG" -eq 1 ]; then set -x fi -PROFILE="$1" +PROFILE_ARG="$1" ARTIFACT="$2" +if [[ "${PROFILE:-${PROFILE_ARG}}" != "$PROFILE_ARG" ]]; then + echo "PROFILE env var is set to '$PROFILE', but '$0' arg1 is '$1'" + exit 1 +fi + +# make sure PROFILE is exported, it is needed by rebar.config.erl +PROFILE=$PROFILE_ARG +export PROFILE + # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" @@ -106,6 +115,7 @@ assert_no_compile_time_only_deps() { } make_rel() { + ./scripts/pre-compile.sh "$PROFILE" # compile all beams ./rebar3 as "$PROFILE" compile # generate docs (require beam compiled), generated to etc and priv dirs @@ -116,6 +126,7 @@ make_rel() { } make_elixir_rel() { + ./scripts/pre-compile.sh "$PROFILE" export_release_vars "$PROFILE" mix release --overwrite assert_no_compile_time_only_deps diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 5cc247977..6c5baa391 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -7,14 +7,15 @@ COPY . /emqx ARG EMQX_NAME=emqx ENV EMQX_RELUP=false -RUN export PROFILE="$EMQX_NAME" \ - && export EMQX_NAME=${EMQX_NAME%%-elixir} \ +RUN export PROFILE=${EMQX_NAME%%-elixir} \ + && export EMQX_NAME1=$EMQX_NAME \ + && export EMQX_NAME=$PROFILE \ && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && export EMQX_REL_FORM='docker' \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ - && make $PROFILE \ + && make $EMQX_NAME1 \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index a8aee2f50..6368a0b51 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -28,14 +28,15 @@ COPY . /emqx ARG EMQX_NAME=emqx ENV EMQX_RELUP=false -RUN export PROFILE="$EMQX_NAME" \ - && export EMQX_NAME=${EMQX_NAME%%-elixir} \ +RUN export PROFILE=${EMQX_NAME%%-elixir} \ + && export EMQX_NAME1=$EMQX_NAME \ + && export EMQX_NAME=$PROFILE \ && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && export EMQX_REL_FORM='docker' \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ - && make $PROFILE \ + && make $EMQX_NAME1 \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel diff --git a/rebar.config.erl b/rebar.config.erl index 976c0d3b8..a4265cae3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -4,6 +4,7 @@ do(Dir, CONFIG) -> ok = assert_otp(), + ok = warn_profile_env(), case iolist_to_binary(Dir) of <<".">> -> C1 = deps(CONFIG), @@ -117,6 +118,9 @@ is_raspbian() -> is_win32() -> win32 =:= element(1, os:type()). +project_app_dirs() -> + project_app_dirs(get_edition_from_profille_env()). + project_app_dirs(Edition) -> ["apps/*"] ++ case is_enterprise(Edition) of @@ -149,6 +153,9 @@ test_deps() -> {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}} ]. +common_compile_opts(Vsn) -> + common_compile_opts(get_edition_from_profille_env(), Vsn). + common_compile_opts(Edition, Vsn) -> % always include debug_info [ @@ -159,6 +166,36 @@ common_compile_opts(Edition, Vsn) -> [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ [{d, 'BUILD_WITHOUT_QUIC'} || not is_quicer_supported()]. +warn_profile_env() -> + case os:getenv("PROFILE") of + false -> + io:format( + standard_error, + "WARN: environment variable PROFILE is not set, using 'emqx-enterprise'~n", + [] + ); + _ -> + ok + end. + +%% this function is only used for test/check profiles +get_edition_from_profille_env() -> + case os:getenv("PROFILE") of + "emqx" -> + ce; + "emqx-" ++ _ -> + ce; + "emqx-enterprise" -> + ee; + "emqx-enterprise-" ++ _ -> + ee; + false -> + ee; + V -> + io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]), + exit(bad_PROFILE) + end. + prod_compile_opts(Edition, Vsn) -> [ compressed, @@ -212,14 +249,14 @@ profiles_dev() -> Vsn = get_vsn('emqx-enterprise'), [ {check, [ - {erl_opts, common_compile_opts(ee, Vsn)}, - {project_app_dirs, project_app_dirs(ee)} + {erl_opts, common_compile_opts(Vsn)}, + {project_app_dirs, project_app_dirs()} ]}, {test, [ {deps, test_deps()}, - {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()}, + {erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()}, {extra_src_dirs, [{"test", [{recursive, true}]}]}, - {project_app_dirs, project_app_dirs(ee)} + {project_app_dirs, project_app_dirs()} ]} ]. diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 481a932ee..0069f3cc2 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -5,8 +5,20 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -RELEASE_ASSET_FILE="emqx-dashboard.zip" -VERSION="${EMQX_DASHBOARD_VERSION}" +VERSION="${1}" +case "$VERSION" in + v*) + RELEASE_ASSET_FILE="emqx-dashboard.zip" + ;; + e*) + RELEASE_ASSET_FILE="emqx-enterprise-dashboard.zip" + ;; + *) + echo "Unknown version $VERSION" + exit 1 + ;; +esac + DASHBOARD_PATH='apps/emqx_dashboard/priv' DASHBOARD_REPO='emqx-dashboard-web-new' DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}" diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh new file mode 100755 index 000000000..0fe99c6b2 --- /dev/null +++ b/scripts/pre-compile.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise) +# it might be with suffix such as -pkg etc. +PROFILE_STR="${1}" + +case "$PROFILE_STR" in + *enterprise*) + dashboard_version="$EMQX_EE_DASHBOARD_VERSION" + ;; + *) + dashboard_version="$EMQX_DASHBOARD_VERSION" + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +./scripts/get-dashboard.sh "$dashboard_version" +./scripts/merge-config.escript +./scripts/merge-i18n.escript