diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 73622495d..75a973990 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -202,7 +202,7 @@ EOF } relup_test(){ - TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)" + TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")" if [ -d "${RELUP_PACKAGE_PATH}" ];then cd "${RELUP_PACKAGE_PATH}" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 465cd1a1b..7e9968a5e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -22,7 +22,8 @@ jobs: container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" outputs: - old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }} + ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }} + ee_old_vsns: ${{ steps.find_old_versons.outputs.ee_old_vsns }} steps: - uses: actions/checkout@v2 @@ -35,10 +36,14 @@ jobs: shell: bash working-directory: source run: | - vsn="$(./pkg-vsn.sh)" - pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" - echo "::set-output name=old_vsns::$old_vsns" + ce_vsn="$(./pkg-vsn.sh community)" + ee_vsn="$(./pkg-vsn.sh enterprise)" + ce_base_vsn_prefix="$(echo $ce_vsn | grep -oE '^[0-9]+\.[0-9]+')" + ee_base_vsn_prefix="$(echo $ee_vsn | grep -oE '^[0-9]+\.[0-9]+')" + ce_old_vsns="$(git tag -l | grep -E "v${ce_base_vsn_prefix}\.[0-9]+$" | grep -v "v${ee_vsn}" | xargs)" + ee_old_vsns="$(git tag -l | grep -E "e${ee_base_vsn_prefix}\.[0-9]+$" | grep -v "e${ee_vsn}" | xargs)" + echo "::set-output name=ce_old_vsns::${ce_old_vsns}" + echo "::set-output name=ee_old_vsns::${ee_old_vsns}" - name: get_all_deps run: | make -C source deps-all @@ -301,16 +306,20 @@ jobs: PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} - OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} + CE_OLD_VSNS: ${{ needs.prepare.outputs.ce_old_vsns }} + EE_OLD_VSNS: ${{ needs.prepare.outputs.ee_old_vsns }} working-directory: source run: | set -e -x -u if [ $PROFILE = 'emqx' ]; then s3dir='emqx-ce' + OLD_VSNS="$CE_OLD_VSNS" elif [ $PROFILE = 'emqx-enterprise' ]; then s3dir='emqx-ee' + OLD_VSNS="$EE_OLD_VSNS" elif [ $PROFILE = 'emqx-edge' ]; then s3dir='emqx-edge' + OLD_VSNS="$CE_OLD_VSNS" else echo "unknown profile $PROFILE" exit 1 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 1694c177a..fc4a2d8f5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -47,7 +47,7 @@ jobs: run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh)-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV + echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV - name: Get deps git refs for cache id: deps-refs run: | diff --git a/.github/workflows/run_broker_tests.yaml b/.github/workflows/run_broker_tests.yaml index d87f5092c..dc526a1a8 100644 --- a/.github/workflows/run_broker_tests.yaml +++ b/.github/workflows/run_broker_tests.yaml @@ -27,9 +27,12 @@ jobs: id: build_docker if: endsWith(github.repository, 'emqx') run: | - make emqx-docker - echo "::set-output name=version::$(./pkg-vsn.sh)" - docker save -o emqx.tar emqx/emqx:$(./pkg-vsn.sh) + ## TODO: make profile a matrix dimension + PROFILE='emqx' + make "${PROFILE}-docker" + VSN="$(./pkg-vsn.sh $PROFILE)" + echo "::set-output name=version::${VSN}" + docker save -o emqx.tar emqx/emqx:${VSN} - uses: actions/upload-artifact@v2 with: name: emqx.tar diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 9b04ffd7b..882f4dd4b 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -90,7 +90,7 @@ jobs: working-directory: source run: | set -x - IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh) + IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }}) ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - name: make paho tests run: | @@ -156,7 +156,7 @@ jobs: run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - run: minikube start - name: run emqx on chart timeout-minutes: 5 diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 630130156..0af50f599 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -76,7 +76,7 @@ jobs: fi echo "BROKER=$broker" >> $GITHUB_ENV - vsn="$(./pkg-vsn.sh)" + vsn="$(./pkg-vsn.sh $PROFILE)" echo "VSN=$vsn" >> $GITHUB_ENV pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" diff --git a/Makefile b/Makefile index d82bcdb7c..07a9ff419 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ SCRIPTS = $(CURDIR)/scripts export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) -export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.18.0 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ce70cc172..54db4ea73 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -17,7 +17,7 @@ %% NOTE: this is the release version which is not always the same %% as the emqx app version defined in emqx.app.src %% App (plugin) versions are bumped independently. -%% e.g. EMQX_RELEASE being 4.3.1 does no always imply emqx app +%% e.g. EMQX_RELEASE_CE being 4.3.1 does no always imply emqx app %% should be 4.3.1, as it might be the case that only one of the %% plugins had a bug to fix. So for a hot beam upgrade, only the app %% with beam files changed needs an upgrade. @@ -28,4 +28,8 @@ %% (Major.Minor.Patch), and extra info can be added after a final %% hyphen. --define(EMQX_RELEASE, "5.0.0-beta.3"). +%% Community edition +-define(EMQX_RELEASE_CE, "5.0.0-beta.3"). + +%% Enterprise edition +-define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl index 31466dc8f..2ff3fca81 100644 --- a/apps/emqx/src/emqx_release.erl +++ b/apps/emqx/src/emqx_release.erl @@ -70,10 +70,10 @@ edition(Desc) -> %% @doc Return the release version. version() -> case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of - false -> %% For TEST build or depedency build. - ?EMQX_RELEASE; + false -> %% For TEST build or depedency build. + build_vsn(); {_, Vsn} -> %% For emqx release build - VsnStr = ?EMQX_RELEASE, + VsnStr = build_vsn(), case string:str(Vsn, VsnStr) of 1 -> ok; _ -> @@ -84,3 +84,9 @@ version() -> end, Vsn end. + +-ifdef(EMQX_ENTERPRISE). +build_vsn() -> ?EMQX_RELEASE_EE. +-else. +build_vsn() -> ?EMQX_RELEASE_CE. +-endif. diff --git a/build b/build index b6bd62cbc..6790522ca 100755 --- a/build +++ b/build @@ -12,7 +12,7 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" export PKG_VSN SYSTEM="$(./scripts/get-distro.sh)" diff --git a/mix.exs b/mix.exs index 5dc97264f..88a243e58 100644 --- a/mix.exs +++ b/mix.exs @@ -520,9 +520,10 @@ defmodule EMQXUmbrella.MixProject do end defp pkg_vsn() do + %{edition_type: edition_type} = read_inputs() basedir = Path.dirname(__ENV__.file) script = Path.join(basedir, "pkg-vsn.sh") - {str_vsn, 0} = System.cmd(script, []) + {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)]) String.trim(str_vsn) end diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 0f16c7809..2ba9fc606 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,12 +6,23 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" +case "${1:-}" in + *enterprise*) + RELEASE_EDITION="EMQX_RELEASE_EE" + GIT_TAG_PREFIX="e" + ;; + *) + RELEASE_EDITION="EMQX_RELEASE_CE" + GIT_TAG_PREFIX="v" + ;; +esac + ## emqx_release.hrl is the single source of truth for release version -RELEASE="$(grep -E "define.+EMQX_RELEASE" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" +RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" git_exact_vsn() { local tag - tag="$(git describe --tags --match "[e|v]*" --exact 2>/dev/null)" + tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)" echo "${tag//^[v|e]/}" } diff --git a/rebar.config.erl b/rebar.config.erl index e0f5c38a5..2630e004e 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -25,8 +25,7 @@ deps(Config) -> lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}). overrides() -> - [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]} - , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]} + [ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]} ]} ] ++ snabbkaffe_overrides(). @@ -99,75 +98,89 @@ test_deps() -> , {proper, "1.4.0"} ]. -common_compile_opts() -> +common_compile_opts(Vsn) -> [ debug_info % alwyas include debug_info - , {compile_info, [{emqx_vsn, get_vsn()}]} + , {compile_info, [{emqx_vsn, Vsn}]} ] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. -prod_compile_opts() -> +prod_compile_opts(Vsn) -> [ compressed , deterministic , warnings_as_errors - | common_compile_opts() + | common_compile_opts(Vsn) ]. prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. profiles() -> - Vsn = get_vsn(), + profiles_ce() ++ profiles_ee() ++ profiles_dev(). + +profiles_ce() -> + Vsn = get_vsn(emqx), [ {'emqx', - [ {erl_opts, prod_compile_opts()} + [ {erl_opts, prod_compile_opts(Vsn)} , {relx, relx(Vsn, cloud, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} , {post_hooks, [{compile, "bash build emqx doc"}]} ]} , {'emqx-pkg', - [ {erl_opts, prod_compile_opts()} + [ {erl_opts, prod_compile_opts(Vsn)} , {relx, relx(Vsn, cloud, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} , {post_hooks, [{compile, "bash build emqx-pkg doc"}]} ]} - , {'emqx-enterprise', - [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, bin, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} - ]} - , {'emqx-enterprise-pkg', - [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, pkg, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} - ]} , {'emqx-edge', - [ {erl_opts, prod_compile_opts()} + [ {erl_opts, prod_compile_opts(Vsn)} , {relx, relx(Vsn, edge, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} , {post_hooks, [{compile, "bash build emqx-edge doc"}]} ]} , {'emqx-edge-pkg', - [ {erl_opts, prod_compile_opts()} + [ {erl_opts, prod_compile_opts(Vsn)} , {relx, relx(Vsn, edge, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]} ]} - , {check, - [ {erl_opts, common_compile_opts()} - , {project_app_dirs, project_app_dirs(ce)} + ]. + +profiles_ee() -> + Vsn = get_vsn('emqx-enterprise'), + EE = {d, 'EMQX_ENTERPRISE'}, + [ {'emqx-enterprise', + [ {erl_opts, [EE | prod_compile_opts(Vsn)]} + , {relx, relx(Vsn, cloud, bin, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} + ]} + , {'emqx-enterprise-pkg', + [ {erl_opts, [EE | prod_compile_opts(Vsn)]} + , {relx, relx(Vsn, cloud, pkg, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} + ]} + ]. + +%% EE has more files than CE, always test/check with EE options. +profiles_dev() -> + Vsn = get_vsn('emqx-enterprise'), + EE = {d, 'EMQX_ENTERPRISE'}, + [ {check, + [ {erl_opts, [EE | common_compile_opts(Vsn)]} + , {project_app_dirs, project_app_dirs(ee)} ]} , {test, [ {deps, test_deps()} - , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) } + , {erl_opts, [EE | common_compile_opts(Vsn) ++ erl_opts_i()]} , {extra_src_dirs, [{"test", [{recursive, true}]}]} - , {project_app_dirs, project_app_dirs(ce)} + , {project_app_dirs, project_app_dirs(ee)} ]} ]. @@ -373,11 +386,11 @@ emqx_etc_overlay_common() -> , {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"} ]. -get_vsn() -> +get_vsn(Profile) -> %% to make it compatible to Linux and Windows, %% we must use bash to execute the bash file %% because "./" will not be recognized as an internal or external command - PkgVsn = os:cmd("bash pkg-vsn.sh"), + PkgVsn = os:cmd("bash pkg-vsn.sh " ++ atom_to_list(Profile)), re:replace(PkgVsn, "\n", "", [{return ,list}]). maybe_dump(Config) -> @@ -399,14 +412,10 @@ provide_bcrypt_dep() -> provide_bcrypt_release(ReleaseType) -> provide_bcrypt_dep() andalso ReleaseType =:= cloud. -erl_opts_i(Edition) -> +erl_opts_i() -> [{i, "apps"}] ++ [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ - case is_enterprise(Edition) of - true -> - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))]; - false -> [] - end. + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))]. dialyzer(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 1ed0fd56d..2ec943149 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -70,7 +70,7 @@ fi cd "${SRC_DIR:-.}" -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}" OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}"