Merge pull request #6854 from zmstone/build-parameterise-package-vsn

build: parameterise package vsn with build profile.

.ci/build_packages/tests.sh

@@ -202,7 +202,7 @@ EOF
 }
 
 relup_test(){
-    TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)"
+    TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh "${EMQX_NAME}")"
     if [ -d "${RELUP_PACKAGE_PATH}" ];then
         cd "${RELUP_PACKAGE_PATH}"
 

.github/workflows/build_packages.yaml

@@ -22,7 +22,8 @@ jobs:
     container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
 
     outputs:
-      old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }}
+      ce_old_vsns: ${{ steps.find_old_versons.outputs.ce_old_vsns }}
+      ee_old_vsns: ${{ steps.find_old_versons.outputs.ee_old_vsns }}
 
     steps:
       - uses: actions/checkout@v2
@@ -35,10 +36,14 @@ jobs:
         shell: bash
         working-directory: source
         run: |
-          vsn="$(./pkg-vsn.sh)"
+          ce_vsn="$(./pkg-vsn.sh community)"
-          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
+          ee_vsn="$(./pkg-vsn.sh enterprise)"
-          old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
+          ce_base_vsn_prefix="$(echo $ce_vsn | grep -oE '^[0-9]+\.[0-9]+')"
-          echo "::set-output name=old_vsns::$old_vsns"
+          ee_base_vsn_prefix="$(echo $ee_vsn | grep -oE '^[0-9]+\.[0-9]+')"
+          ce_old_vsns="$(git tag -l | grep -E "v${ce_base_vsn_prefix}\.[0-9]+$" | grep -v "v${ee_vsn}" | xargs)"
+          ee_old_vsns="$(git tag -l | grep -E "e${ee_base_vsn_prefix}\.[0-9]+$" | grep -v "e${ee_vsn}" | xargs)"
+          echo "::set-output name=ce_old_vsns::${ce_old_vsns}"
+          echo "::set-output name=ee_old_vsns::${ee_old_vsns}"
       - name: get_all_deps
         run: |
           make -C source deps-all
@@ -301,16 +306,20 @@ jobs:
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
+        CE_OLD_VSNS: ${{ needs.prepare.outputs.ce_old_vsns }}
+        EE_OLD_VSNS: ${{ needs.prepare.outputs.ee_old_vsns }}
       working-directory: source
       run: |
         set -e -x -u
         if [ $PROFILE = 'emqx' ]; then
             s3dir='emqx-ce'
+            OLD_VSNS="$CE_OLD_VSNS"
         elif [ $PROFILE = 'emqx-enterprise' ]; then
             s3dir='emqx-ee'
+            OLD_VSNS="$EE_OLD_VSNS"
         elif [ $PROFILE = 'emqx-edge' ]; then
             s3dir='emqx-edge'
+            OLD_VSNS="$CE_OLD_VSNS"
         else
             echo "unknown profile $PROFILE"
             exit 1

.github/workflows/build_slim_packages.yaml

@@ -47,7 +47,7 @@ jobs:
       run: |
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
         echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
-        echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh)-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
+        echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
     - name: Get deps git refs for cache
       id: deps-refs
       run: |

.github/workflows/run_broker_tests.yaml

@@ -27,9 +27,12 @@ jobs:
       id: build_docker
       if: endsWith(github.repository, 'emqx')
       run: |
-        make emqx-docker
+        ## TODO: make profile a matrix dimension
-        echo "::set-output name=version::$(./pkg-vsn.sh)"
+        PROFILE='emqx'
-        docker save -o emqx.tar emqx/emqx:$(./pkg-vsn.sh)
+        make "${PROFILE}-docker"
+        VSN="$(./pkg-vsn.sh $PROFILE)"
+        echo "::set-output name=version::${VSN}"
+        docker save -o emqx.tar emqx/emqx:${VSN}
     - uses: actions/upload-artifact@v2
       with:
         name: emqx.tar

.github/workflows/run_fvt_tests.yaml

@@ -90,7 +90,7 @@ jobs:
       working-directory: source
       run: |
         set -x
-        IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh)
+        IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
         ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
     - name: make paho tests
       run: |
@@ -156,7 +156,7 @@ jobs:
       run: |
         make ${{ matrix.profile }}-docker
         echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
-        echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+        echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
     - run: minikube start
     - name: run emqx on chart
       timeout-minutes: 5

.github/workflows/run_relup_tests.yaml

@@ -76,7 +76,7 @@ jobs:
         fi
         echo "BROKER=$broker" >> $GITHUB_ENV
 
-        vsn="$(./pkg-vsn.sh)"
+        vsn="$(./pkg-vsn.sh $PROFILE)"
         echo "VSN=$vsn" >> $GITHUB_ENV
 
         pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"

Makefile

@@ -6,7 +6,6 @@ SCRIPTS = $(CURDIR)/scripts
 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
-export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v0.18.0
 export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing

apps/emqx/include/emqx_release.hrl

@@ -17,7 +17,7 @@
 %% NOTE: this is the release version which is not always the same
 %% as the emqx app version defined in emqx.app.src
 %% App (plugin) versions are bumped independently.
-%% e.g. EMQX_RELEASE being 4.3.1 does no always imply emqx app
+%% e.g. EMQX_RELEASE_CE being 4.3.1 does no always imply emqx app
 %% should be 4.3.1, as it might be the case that only one of the
 %% plugins had a bug to fix. So for a hot beam upgrade, only the app
 %% with beam files changed needs an upgrade.
@@ -28,4 +28,8 @@
 %% (Major.Minor.Patch), and extra info can be added after a final
 %% hyphen.
 
--define(EMQX_RELEASE, "5.0.0-beta.3").
+%% Community edition
+-define(EMQX_RELEASE_CE, "5.0.0-beta.3").
+
+%% Enterprise edition
+-define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

apps/emqx/src/emqx_release.erl

@@ -71,9 +71,9 @@ edition(Desc) ->
 version() ->
     case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of
         false -> %% For TEST build or depedency build.
-            ?EMQX_RELEASE;
+            build_vsn();
         {_, Vsn} -> %% For emqx release build
-            VsnStr = ?EMQX_RELEASE,
+            VsnStr = build_vsn(),
             case string:str(Vsn, VsnStr) of
                 1 -> ok;
                 _ ->
@@ -84,3 +84,9 @@ version() ->
             end,
             Vsn
     end.
+
+-ifdef(EMQX_ENTERPRISE).
+build_vsn() -> ?EMQX_RELEASE_EE.
+-else.
+build_vsn() -> ?EMQX_RELEASE_CE.
+-endif.

build

@@ -12,7 +12,7 @@ ARTIFACT="$2"
 # ensure dir
 cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
 
-PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
+PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
 export PKG_VSN
 
 SYSTEM="$(./scripts/get-distro.sh)"

mix.exs

@@ -520,9 +520,10 @@ defmodule EMQXUmbrella.MixProject do
   end
 
   defp pkg_vsn() do
+    %{edition_type: edition_type} = read_inputs()
     basedir = Path.dirname(__ENV__.file)
     script = Path.join(basedir, "pkg-vsn.sh")
-    {str_vsn, 0} = System.cmd(script, [])
+    {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)])
 
     String.trim(str_vsn)
   end

pkg-vsn.sh

@@ -6,12 +6,23 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "$0")"
 
+case "${1:-}" in
+    *enterprise*)
+        RELEASE_EDITION="EMQX_RELEASE_EE"
+        GIT_TAG_PREFIX="e"
+        ;;
+    *)
+        RELEASE_EDITION="EMQX_RELEASE_CE"
+        GIT_TAG_PREFIX="v"
+        ;;
+esac
+
 ## emqx_release.hrl is the single source of truth for release version
-RELEASE="$(grep -E "define.+EMQX_RELEASE" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)"
+RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)"
 
 git_exact_vsn() {
     local tag
-    tag="$(git describe --tags --match "[e|v]*" --exact 2>/dev/null)"
+    tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)"
     echo "${tag//^[v|e]/}"
 }
 

rebar.config.erl

@@ -25,8 +25,7 @@ deps(Config) ->
     lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}).
 
 overrides() ->
-    [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]}
+    [ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]}
-            , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]}
             ]}
     ] ++ snabbkaffe_overrides().
 
@@ -99,75 +98,89 @@ test_deps() ->
     , {proper, "1.4.0"}
     ].
 
-common_compile_opts() ->
+common_compile_opts(Vsn) ->
     [ debug_info % alwyas include debug_info
-    , {compile_info, [{emqx_vsn, get_vsn()}]}
+    , {compile_info, [{emqx_vsn, Vsn}]}
     ] ++
     [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ].
 
-prod_compile_opts() ->
+prod_compile_opts(Vsn) ->
     [ compressed
     , deterministic
     , warnings_as_errors
-    | common_compile_opts()
+    | common_compile_opts(Vsn)
     ].
 
 prod_overrides() ->
     [{add, [ {erl_opts, [deterministic]}]}].
 
 profiles() ->
-    Vsn = get_vsn(),
+    profiles_ce() ++ profiles_ee() ++ profiles_dev().
+
+profiles_ce() ->
+    Vsn = get_vsn(emqx),
     [ {'emqx',
-       [ {erl_opts, prod_compile_opts()}
+       [ {erl_opts, prod_compile_opts(Vsn)}
        , {relx, relx(Vsn, cloud, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
        , {post_hooks, [{compile, "bash build emqx doc"}]}
        ]}
     , {'emqx-pkg',
-       [ {erl_opts, prod_compile_opts()}
+       [ {erl_opts, prod_compile_opts(Vsn)}
        , {relx, relx(Vsn, cloud, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
        , {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
        ]}
-    , {'emqx-enterprise',
-       [ {erl_opts, prod_compile_opts()}
-       , {relx, relx(Vsn, cloud, bin, ee)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
-       ]}
-    , {'emqx-enterprise-pkg',
-       [ {erl_opts, prod_compile_opts()}
-       , {relx, relx(Vsn, cloud, pkg, ee)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
-       ]}
     , {'emqx-edge',
-       [ {erl_opts, prod_compile_opts()}
+       [ {erl_opts, prod_compile_opts(Vsn)}
        , {relx, relx(Vsn, edge, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
        , {post_hooks, [{compile, "bash build emqx-edge doc"}]}
        ]}
     , {'emqx-edge-pkg',
-       [ {erl_opts, prod_compile_opts()}
+       [ {erl_opts, prod_compile_opts(Vsn)}
        , {relx, relx(Vsn, edge, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
        , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
        ]}
-    , {check,
+    ].
-       [ {erl_opts, common_compile_opts()}
+
-       , {project_app_dirs, project_app_dirs(ce)}
+profiles_ee() ->
+    Vsn = get_vsn('emqx-enterprise'),
+    EE = {d, 'EMQX_ENTERPRISE'},
+    [ {'emqx-enterprise',
+       [ {erl_opts, [EE | prod_compile_opts(Vsn)]}
+       , {relx, relx(Vsn, cloud, bin, ee)}
+       , {overrides, prod_overrides()}
+       , {project_app_dirs, project_app_dirs(ee)}
+       , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
+       ]}
+    , {'emqx-enterprise-pkg',
+       [ {erl_opts, [EE | prod_compile_opts(Vsn)]}
+       , {relx, relx(Vsn, cloud, pkg, ee)}
+       , {overrides, prod_overrides()}
+       , {project_app_dirs, project_app_dirs(ee)}
+       , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
+       ]}
+    ].
+
+%% EE has more files than CE, always test/check with EE options.
+profiles_dev() ->
+    Vsn = get_vsn('emqx-enterprise'),
+    EE = {d, 'EMQX_ENTERPRISE'},
+    [ {check,
+       [ {erl_opts, [EE | common_compile_opts(Vsn)]}
+       , {project_app_dirs, project_app_dirs(ee)}
        ]}
     , {test,
        [ {deps, test_deps()}
-       , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) }
+       , {erl_opts, [EE | common_compile_opts(Vsn) ++ erl_opts_i()]}
        , {extra_src_dirs, [{"test", [{recursive, true}]}]}
-       , {project_app_dirs, project_app_dirs(ce)}
+       , {project_app_dirs, project_app_dirs(ee)}
        ]}
     ].
 
@@ -373,11 +386,11 @@ emqx_etc_overlay_common() ->
     , {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}
     ].
 
-get_vsn() ->
+get_vsn(Profile) ->
     %% to make it compatible to Linux and Windows,
     %% we must use bash to execute the bash file
     %% because "./" will not be recognized as an internal or external command
-    PkgVsn = os:cmd("bash pkg-vsn.sh"),
+    PkgVsn = os:cmd("bash pkg-vsn.sh " ++ atom_to_list(Profile)),
     re:replace(PkgVsn, "\n", "", [{return ,list}]).
 
 maybe_dump(Config) ->
@@ -399,14 +412,10 @@ provide_bcrypt_dep() ->
 provide_bcrypt_release(ReleaseType) ->
     provide_bcrypt_dep() andalso ReleaseType =:= cloud.
 
-erl_opts_i(Edition) ->
+erl_opts_i() ->
     [{i, "apps"}] ++
     [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++
-    case is_enterprise(Edition) of
+    [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))].
-        true ->
-            [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))];
-        false -> []
-    end.
 
 dialyzer(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),

scripts/buildx.sh

@@ -70,7 +70,7 @@ fi
 
 cd "${SRC_DIR:-.}"
 
-PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
+PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
 OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2)
 PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}"