From a069b351fd79e5eb891d2729e47b78278130973a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 20:55:16 +0200 Subject: [PATCH] test: add test script to verify config example files --- apps/emqx/src/emqx_hocon.erl | 29 +++++++- apps/emqx/src/emqx_schema.erl | 14 +++- .../emqx_conf/test/emqx_conf_schema_tests.erl | 71 ++++++++++++++++++- scripts/test/check-example-configs.sh | 54 ++++++++++++++ 4 files changed, 164 insertions(+), 4 deletions(-) create mode 100755 scripts/test/check-example-configs.sh diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index e028ebb14..62573f201 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -24,7 +24,8 @@ compact_errors/2, format_error/1, format_error/2, - make_schema/1 + make_schema/1, + load_and_check/2 ]). %% @doc Format hocon config field path to dot-separated string in iolist format. @@ -46,7 +47,8 @@ check(SchemaModule, Conf) -> check(SchemaModule, Conf, Opts) when is_map(Conf) -> try - {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts)} + RootNames = maps:keys(Conf), + {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts, RootNames)} catch throw:Errors:Stacktrace -> compact_errors(Errors, Stacktrace) @@ -135,3 +137,26 @@ compact_errors(SchemaModule, Error, Stacktrace) -> exception => Error, stacktrace => Stacktrace }}. + +%% @doc This is only used in static check scripts in the CI. +-spec load_and_check(module(), filename:filename_all()) -> {ok, term()} | {error, any()}. +load_and_check(SchemaModule, File) -> + try + do_load_and_check(SchemaModule, File) + catch + throw:Reason -> + {error, Reason} + end. + +do_load_and_check(SchemaModule, File) -> + Conf = + case hocon:load(File, #{format => map}) of + {ok, Conf0} -> + Conf0; + {error, {parse_error, Reason}} -> + throw(Reason); + {error, Reason} -> + throw(Reason) + end, + Opts = #{atom_key => false, required => false}, + check(SchemaModule, Conf, Opts). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 9de6ef34a..6c0c511fb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3271,7 +3271,19 @@ tombstone() -> tombstone_map(Name, Type) -> %% marked_for_deletion must be the last member of the union %% because we need to first union member to populate the default values - map(Name, ?UNION([Type, ?TOMBSTONE_TYPE])). + map( + Name, + hoconsc:union( + fun + (all_union_members) -> + [Type, ?TOMBSTONE_TYPE]; + ({value, V}) when is_map(V) -> + [Type]; + ({value, _}) -> + [?TOMBSTONE_TYPE] + end + ) + ). %% inverse of mark_del_map get_tombstone_map_value_type(Schema) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 76efd8ec4..855b8ff12 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_conf_schema_tests). @@ -488,3 +500,60 @@ check(Config) -> atom_key => false, required => false, format => map }), emqx_utils_maps:unsafe_atom_key_map(Conf). + +with_file(Path, Content, F) -> + ok = file:write_file(Path, Content), + try + F() + after + file:delete(Path) + end. + +load_and_check_test_() -> + [ + {"non-existing file", fun() -> + File = "/tmp/nonexistingfilename.hocon", + ?assertEqual( + {error, {enoent, File}}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end}, + {"bad syntax", fun() -> + %% use abs path to match error return + File = "/tmp/emqx-conf-bad-syntax-test.hocon", + with_file( + File, + "{", + fun() -> + ?assertMatch( + {error, #{file := File}}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end + ) + end}, + {"type-check failure", fun() -> + File = "emqx-conf-type-check-failure.hocon", + %% typecheck fail because cookie is required field + with_file( + File, + "node {}", + fun() -> + ?assertMatch( + {error, #{ + kind := validation_error, + path := "node.cookie", + reason := required_field + }}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end + ) + end}, + {"ok load", fun() -> + File = "emqx-conf-test-tmp-file-load-ok.hocon", + with_file(File, "plugins: {}", fun() -> + ?assertMatch({ok, _}, emqx_hocon:load_and_check(emqx_conf_schema, File)) + end) + end} + ]. diff --git a/scripts/test/check-example-configs.sh b/scripts/test/check-example-configs.sh new file mode 100755 index 000000000..bbf00cf8d --- /dev/null +++ b/scripts/test/check-example-configs.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -euo pipefail +PROJ_DIR="$(git rev-parse --show-toplevel)" + +PROFILE="${PROFILE:-emqx}" +DIR_NAME='examples' +SCHEMA_MOD='emqx_conf_schema' +if [ "${PROFILE}" = 'emqx-enterprise' ]; then + DIR_NAME='ee-examples' + SCHEMA_MOD='emqx_enterprise_schema' + PA="" +fi + +IFS=$'\n' read -r -d '' -a FILES < <(find "${PROJ_DIR}/rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') + +prepare_erl_libs() { + local libs_dir="$1" + local erl_libs="${ERL_LIBS:-}" + local sep=':' + for app in "${libs_dir}"/*; do + if [ -d "${app}/ebin" ]; then + if [ -n "$erl_libs" ]; then + erl_libs="${erl_libs}${sep}${app}" + else + erl_libs="${app}" + fi + fi + done + export ERL_LIBS="$erl_libs" +} + +# This is needed when checking schema +export EMQX_ETC_DIR="${PROJ_DIR}/apps/emqx/etc" + +prepare_erl_libs "_build/$PROFILE/lib" + +check_file() { + local file="$1" + erl -noshell -eval \ + "File=\"$file\", + case emqx_hocon:load_and_check($SCHEMA_MOD, File) of + {ok, _} -> + io:format(\"check_example_config_ok: ~s~n\", [File]), + halt(0); + {error, Reason} -> + io:format(\"failed_to_check_example_config: ~s~n~p~n\", [File, Reason]), + halt(1) + end." +} + +for file in ${FILES[@]}; do + check_file "$file" +done