From 6edd862dd5b38760e013cc3915d39c11eb172e00 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 20 Nov 2021 00:39:58 +0100 Subject: [PATCH] refactor: no more EMQX_ENTERPRISE compile flag The compile flag was introduced in EQM X 4.3 series where CE and EE code was diverged large enough which made non-practicle to determin edition at runtime. such approach made testing quite challenging as we'll have to build with different compile flags inorder to run per-edition test cases In this commit, we try to retrieve edition info from EMQX's description text, (put to PT for fast access) at runtime so we can test ALL editions from a super-set edition (EE). --- apps/emqx/include/emqx_release.hrl | 14 +-- apps/emqx/src/emqx.erl | 27 ------ apps/emqx/src/emqx_app.erl | 29 +------ apps/emqx/src/emqx_misc.erl | 16 +--- apps/emqx/src/emqx_release.erl | 86 +++++++++++++++++++ apps/emqx/test/emqx_authentication_SUITE.erl | 2 +- .../emqx_dashboard/src/emqx_dashboard_api.erl | 15 +--- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 30 +++---- lib-ee/.gitkeep | 0 pkg-vsn.sh | 8 +- rebar.config.erl | 79 ++++++----------- scripts/check-deps-integrity.escript | 7 +- scripts/find-apps.sh | 4 +- scripts/git-hook-pre-push.sh | 2 + 14 files changed, 142 insertions(+), 177 deletions(-) create mode 100644 apps/emqx/src/emqx_release.erl create mode 100644 lib-ee/.gitkeep diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 5b4bc7d00..1424eb8a5 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -14,9 +14,6 @@ %% limitations under the License. %%-------------------------------------------------------------------- --ifndef(EMQX_RELEASE_HRL). --define(EMQX_RELEASE_HRL, true). - %% NOTE: this is the release version which is not always the same %% as the emqx app version defined in emqx.app.src %% App (plugin) versions are bumped independently. @@ -27,13 +24,4 @@ %% NOTE: This version number should be manually bumped for each release --ifndef(EMQX_ENTERPRISE). - --define(EMQX_RELEASE, {opensource, "5.0-beta.1"}). - --else. - - --endif. - --endif. +-define(EMQX_RELEASE, "5.0-beta.2"). diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 1df7f0149..52125a74c 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -51,10 +51,6 @@ , run_fold_hook/3 ]). -%% Troubleshooting --export([ set_debug_secret/1 - ]). - %% Configs APIs -export([ get_config/1 , get_config/2 @@ -71,29 +67,6 @@ -define(APP, ?MODULE). -%% @hidden Path to the file which has debug_info encryption secret in it. -%% Evaluate this function if there is a need to access encrypted debug_info. -%% NOTE: Do not change the API to accept the secret text because it may -%% get logged everywhere. -set_debug_secret(PathToSecretFile) -> - SecretText = - case file:read_file(PathToSecretFile) of - {ok, Secret} -> - try string:trim(binary_to_list(Secret)) - catch _ : _ -> error({badfile, PathToSecretFile}) - end; - {error, Reason} -> - ?ULOG("Failed to read debug_info encryption key file ~ts: ~p~n", - [PathToSecretFile, Reason]), - error(Reason) - end, - F = fun(init) -> ok; - (clear) -> ok; - ({debug_info, _Mode, _Module, _Filename}) -> SecretText - end, - _ = beam_lib:clear_crypto_key_fun(), - ok = beam_lib:crypto_key_fun(F). - %%-------------------------------------------------------------------- %% Bootstrap, is_running... %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index d1090803d..4e130ef36 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -30,7 +30,6 @@ ]). -include("emqx.hrl"). --include("emqx_release.hrl"). -include("logger.hrl"). -define(APP, emqx). @@ -40,6 +39,7 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> + ok = emqx_release:put_edition(), ok = maybe_load_config(), ok = emqx_persistent_session:init_db_backend(), ok = maybe_start_quicer(), @@ -107,30 +107,7 @@ is_quicer_app_present() -> is_quic_listener_configured() -> emqx_listeners:has_enabled_listener_conf_by_type(quic). -get_description() -> - {ok, Descr0} = application:get_key(?APP, description), - case os:getenv("EMQX_DESCRIPTION") of - false -> Descr0; - "" -> Descr0; - Str -> string:strip(Str, both, $\n) - end. +get_description() -> emqx_release:description(). get_release() -> - case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of - false -> %% For TEST build or depedency build. - release_in_macro(); - {_, Vsn} -> %% For emqx release build - VsnStr = release_in_macro(), - case string:str(Vsn, VsnStr) of - 1 -> ok; - _ -> - erlang:error(#{ reason => version_mismatch - , source => VsnStr - , built_for => Vsn - }) - end, - Vsn - end. - -release_in_macro() -> - element(2, ?EMQX_RELEASE). + emqx_release:version(). diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 446039778..3dc706464 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -65,21 +65,13 @@ maybe_parse_ip(Host) -> end. %% @doc Add `ipv6_probe' socket option if it's supported. +%% gen_tcp:ipv6_probe() -> true. is added to EMQ's OTP forks ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) + case erlang:function_exported(gen_tcp, ipv6_probe, 0) of + true -> [{ipv6_probe, true} | Opts]; + false -> Opts end. -ipv6_probe(false, Opts) -> Opts; -ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. - %% @doc Merge options -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). merge_opts(Defaults, Options) -> diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl new file mode 100644 index 000000000..1e362d0f0 --- /dev/null +++ b/apps/emqx/src/emqx_release.erl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_release). + +-export([ edition/0 + , put_edition/0 + , put_edition/1 + , description/0 + , version/0 + ]). + +-include("emqx_release.hrl"). + +%% @doc Return EMQ X description. +description() -> + case os:getenv("EMQX_DESCRIPTION") of + false -> "EMQ X Community Edition"; + "" -> "EMQ X Community Edition"; + Str -> string:strip(Str, both, $\n) + end. + +%% @doc Return EMQ X edition info. +%% Read info from persistent_term at runtime. +%% Or meck this function to run tests for another eidtion. +-spec edition() -> ce | ee | edge. +edition() -> + try persistent_term:get(emqx_edition) + catch error : badarg -> get_edition() end. + +%% @private initiate EMQ X edition info in persistent_term. +put_edition() -> + ok = put_edition(get_edition()). + +%% @hidden This function is mostly for testing. +%% Switch to another eidtion at runtime to run edition-specific tests. +-spec put_edition(ce | ee | edge) -> ok. +put_edition(Which) -> + persistent_term:put(emqx_edition, Which), + ok. + +-spec get_edition() -> ce | ee | edge. +get_edition() -> + edition(description()). + +edition(Desc) -> + case re:run(Desc, "enterprise", [caseless]) of + {match, _} -> + ee; + _ -> + case re:run(Desc, "edge", [caseless]) of + {match, _} -> edge; + _ -> ce + end + end. + +%% @doc Return the release version. +version() -> + case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of + false -> %% For TEST build or depedency build. + ?EMQX_RELEASE; + {_, Vsn} -> %% For emqx release build + VsnStr = ?EMQX_RELEASE, + case string:str(Vsn, VsnStr) of + 1 -> ok; + _ -> + erlang:error(#{ reason => version_mismatch + , source => VsnStr + , built_for => Vsn + }) + end, + Vsn + end. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index e04b8566b..21a59e0c9 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -270,7 +270,7 @@ t_restart(Config) when is_list(Config) -> ?assertEqual({ok, [test_chain]}, ?AUTHN:list_chain_names()); -t_restart({'end', Config}) -> +t_restart({'end', _Config}) -> ?AUTHN:delete_chain(test_chain), ok. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index cb3545da3..c40c4bd22 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -16,16 +16,6 @@ -module(emqx_dashboard_api). --ifndef(EMQX_ENTERPRISE). - --define(RELEASE, community). - --else. - --define(VERSION, enterprise). - --endif. - -behaviour(minirest_api). -include("emqx_dashboard.hrl"). @@ -200,7 +190,10 @@ login(post, #{body := Params}) -> case emqx_dashboard_admin:sign_token(Username, Password) of {ok, Token} -> Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())), - {200, #{token => Token, version => Version, license => #{edition => ?RELEASE}}}; + {200, #{token => Token, + version => Version, + license => #{edition => emqx_release:edition()} + }}; {error, _} -> {401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}} end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index c94242a99..06b710492 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -884,29 +884,27 @@ time_unit(<<"nanosecond">>) -> nanosecond. %% Here the emqx_rule_funcs module acts as a proxy, forwarding %% the function handling to the worker module. %% @end --ifdef(EMQX_ENTERPRISE). -'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> - emqx_schema_parser:decode(SchemaId, Data, MoreArgs); -'$handle_undefined_function'(schema_decode, Args) -> - error({args_count_error, {schema_decode, Args}}); +% '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> +% emqx_schema_parser:decode(SchemaId, Data, MoreArgs); +% '$handle_undefined_function'(schema_decode, Args) -> +% error({args_count_error, {schema_decode, Args}}); -'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> - emqx_schema_parser:encode(SchemaId, Term, MoreArgs); -'$handle_undefined_function'(schema_encode, Args) -> - error({args_count_error, {schema_encode, Args}}); +% '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> +% emqx_schema_parser:encode(SchemaId, Term, MoreArgs); +% '$handle_undefined_function'(schema_encode, Args) -> +% error({args_count_error, {schema_encode, Args}}); + +% '$handle_undefined_function'(sprintf, [Format|Args]) -> +% erlang:apply(fun sprintf_s/2, [Format, Args]); + +% '$handle_undefined_function'(Fun, Args) -> +% error({sql_function_not_supported, function_literal(Fun, Args)}). '$handle_undefined_function'(sprintf, [Format|Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). --else. -'$handle_undefined_function'(sprintf, [Format|Args]) -> - erlang:apply(fun sprintf_s/2, [Format, Args]); - -'$handle_undefined_function'(Fun, Args) -> - error({sql_function_not_supported, function_literal(Fun, Args)}). --endif. % EMQX_ENTERPRISE map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. diff --git a/lib-ee/.gitkeep b/lib-ee/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 2b8ac6fd1..ad24bb974 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,14 +6,8 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" -if [ -f EMQX_ENTERPRISE ]; then - EDITION='enterprise' -else - EDITION='opensource' -fi - ## emqx_release.hrl is the single source of truth for release version -RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" +RELEASE="$(grep -E "define.+EMQX_RELEASE" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" git_exact_vsn() { local tag diff --git a/rebar.config.erl b/rebar.config.erl index 1b82e281f..78e4f3e41 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -130,18 +130,17 @@ test_deps() -> , {proper, "1.4.0"} ]. -common_compile_opts(Edition) -> +common_compile_opts() -> [ debug_info % alwyas include debug_info , {compile_info, [{emqx_vsn, get_vsn()}]} ] ++ - [{d, 'EMQX_ENTERPRISE'} || is_enterprise(Edition)] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. -prod_compile_opts(Edition) -> +prod_compile_opts() -> [ compressed , deterministic , warnings_as_errors - | common_compile_opts(Edition) + | common_compile_opts() ]. prod_overrides() -> @@ -149,34 +148,41 @@ prod_overrides() -> profiles() -> Vsn = get_vsn(), - ce_profiles(Vsn) ++ ee_profiles(Vsn). - -ce_profiles(Vsn) -> - [ {'emqx', [ {erl_opts, prod_compile_opts(ce)} + [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-ee', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, bin, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, pkg, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {check, [ {erl_opts, common_compile_opts(ce)} + , {check, [ {erl_opts, common_compile_opts()} , {project_app_dirs, project_app_dirs(ce)} ]} , {test, [ {deps, test_deps()} - , {erl_opts, common_compile_opts(ce) ++ erl_opts_i(ce) } + , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) } , {extra_src_dirs, [{"test", [{recursive, true}]}]} , {project_app_dirs, project_app_dirs(ce)} ]} @@ -198,13 +204,11 @@ relx(Vsn, RelType, PkgType, Edition) -> | overlay_vars(RelType, PkgType, Edition)]} ]. -emqx_description(cloud, ee) -> "EMQ X Enterprise"; -emqx_description(cloud, ce) -> "EMQ X Broker"; -emqx_description(edge, ce) -> "EMQ X Edge". +emqx_description(cloud, ee) -> "EMQ X Enterprise Edition"; +emqx_description(cloud, ce) -> "EMQ X Community Edition"; +emqx_description(edge, ce) -> "EMQ X Edge Edition". -overlay_vars(_RelType, PkgType, ee) -> - ee_overlay_vars(PkgType); -overlay_vars(RelType, PkgType, ce) -> +overlay_vars(RelType, PkgType, _Edition) -> overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType). %% vars per release type, cloud or edge @@ -289,7 +293,7 @@ relx_apps(ReleaseType, Edition) -> , emqx_limiter ] ++ [quicer || is_quicer_supported()] - ++ [emqx_license || is_enterprise(Edition)] + %++ [emqx_license || is_enterprise(Edition)] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ relx_apps_per_rel(ReleaseType) %% NOTE: applications below are only loaded after node start/restart @@ -472,38 +476,3 @@ list_dir(Dir) -> false -> [] end. - -%% ==== Enterprise supports below ================================================================== - -ee_profiles(Vsn) -> - [ {'emqx-ee', [ {erl_opts, prod_compile_opts(ee)} - , {relx, relx(Vsn, cloud, bin, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts(ee)} - , {relx, relx(Vsn, cloud, pkg, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'check-ee', [ {erl_opts, common_compile_opts(ee)} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'test-ee', [ {deps, test_deps()} - , {erl_opts, common_compile_opts(ee) ++ erl_opts_i(ee) } - , {extra_src_dirs, [{"test", [{recursive, true}]}]} - , {project_app_dirs, project_app_dirs(ee)} - ]} - ]. - -ee_overlay_vars(PkgType) -> - Common = [], - Common ++ ee_overlay_vars_pkg(PkgType). - -%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) -ee_overlay_vars_pkg(bin) -> - [ - ]; -ee_overlay_vars_pkg(pkg) -> - [ - ]. diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript index 3cc8fdc53..43223701b 100755 --- a/scripts/check-deps-integrity.escript +++ b/scripts/check-deps-integrity.escript @@ -7,12 +7,7 @@ main([]) -> Files = ["rebar.config"] ++ apps_rebar_config("apps") ++ - case filelib:is_file("EMQX_ENTERPRISE") of - true -> - true = filelib:is_dir("lib-ee"), - apps_rebar_config("lib-ee"); - false -> [] - end, + apps_rebar_config("lib-ee"), Deps = collect_deps(Files, #{}), case count_bad_deps(Deps) of 0 -> diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 47b71c7ca..003ffbc5e 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -11,9 +11,7 @@ find_app() { } find_app 'apps' -if [ -f 'EMQX_ENTERPRISE' ]; then - find_app 'lib-ee' -fi +find_app 'lib-ee' ## find directories in lib-extra find_app 'lib-extra' diff --git a/scripts/git-hook-pre-push.sh b/scripts/git-hook-pre-push.sh index 5b3f2edf2..60df47333 100755 --- a/scripts/git-hook-pre-push.sh +++ b/scripts/git-hook-pre-push.sh @@ -4,6 +4,8 @@ set -euo pipefail url="$2" +# we keep this to secure OLD private repo +# even after we have disclosed new code under EMQ BSL 1.0 if [ -f 'EMQX_ENTERPRISE' ]; then if [[ "$url" != *emqx-enterprise* ]]; then echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo"