From 3d20e566d884beffdec8c27337dc84d460dd3ff6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 10:47:16 +0100 Subject: [PATCH 01/21] docs: add namespace to authn and authz schema --- apps/emqx_auth_http/src/emqx_authn_http_schema.erl | 5 ++++- apps/emqx_auth_http/src/emqx_authz_http_schema.erl | 5 ++++- apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl | 5 ++++- apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl | 5 ++++- apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl | 5 ++++- apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl | 5 ++++- apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl | 5 ++++- apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl | 5 ++++- apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl | 3 +++ apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl | 3 +++ .../src/emqx_authn_postgresql_schema.erl | 3 +++ .../src/emqx_authz_postgresql_schema.erl | 3 +++ apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl | 3 +++ apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl | 3 +++ 14 files changed, 50 insertions(+), 8 deletions(-) diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index 1eaac6378..9464b0aaf 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -27,7 +27,8 @@ validations/0, desc/1, refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)). @@ -38,6 +39,8 @@ }) ). +namespace() -> "authn". + refs() -> [?R_REF(http_get), ?R_REF(http_post)]. diff --git a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl index 18ec23757..5f9af846b 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl @@ -26,7 +26,8 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). -export([ @@ -38,6 +39,8 @@ -import(emqx_schema, [mk_duration/2]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. source_refs() -> diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl index 9a21766e3..580673d85 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl @@ -25,9 +25,12 @@ fields/1, desc/1, refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authn". + refs() -> [?R_REF(ldap_bind)]. diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl index 491b0debf..e6a060f42 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl @@ -26,9 +26,12 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(ldap) -> diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl index bb5ccfe1a..373d95fc8 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl @@ -25,9 +25,12 @@ fields/1, desc/1, refs/1, - select_union_member/2 + select_union_member/2, + namespace/0 ]). +namespace() -> "authn". + refs(api_write) -> [?R_REF(builtin_db_api)]; refs(_) -> diff --git a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl index cab544bf7..4d467397e 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl @@ -26,9 +26,12 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(builtin_db) -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl index 8f76bedc2..53c6a6a10 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl @@ -25,9 +25,12 @@ fields/1, desc/1, refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authn". + refs() -> [ ?R_REF(mongo_single), diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl index aff399e68..ee20b962e 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl @@ -24,9 +24,12 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. source_refs() -> diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl index 0189ecc61..17609acb8 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + refs() -> [?R_REF(mysql)]. diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl index a9ce422e6..43f6ca6fa 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,6 +30,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(mysql) -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl index 6b3b600ee..ef7d00df3 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + select_union_member( #{ <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl index 2be7e9387..296b00126 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,6 +30,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(postgresql) -> diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl b/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl index 7b5794c48..f3e124ca1 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + refs() -> [ ?R_REF(redis_single), diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl b/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl index 755192bfc..5cd084795 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,6 +30,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(redis_single) -> From af3dfc91b120641a60c8b84f2c3450b7c29c99ed Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 16:31:57 +0100 Subject: [PATCH 02/21] refactor(schema): change log_lvel to enum type --- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ba9560157..5c16bc56b 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -28,14 +28,12 @@ -include("emqx_conf.hrl"). --type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). -type cipher() :: map(). -behaviour(hocon_schema). -reflect_type([ - log_level/0, file/0, cipher/0 ]). @@ -1289,7 +1287,7 @@ log_handler_common_confs(Handler, Default) -> [ {"level", sc( - log_level(), + hoconsc:enum([debug, info, notice, warning, error, critical, alert, emergency, all]), #{ default => maps:get(level, Default, warning), desc => ?DESC(LevelDesc), From 7a0a0bee98a7af4c42fdbf0de46d7282da0c7b5d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 16:33:07 +0100 Subject: [PATCH 03/21] refactor(schema): change file() type to string() --- apps/emqx_conf/src/emqx_conf_schema.erl | 12 +++++------- apps/emqx_enterprise/src/emqx_enterprise_schema.erl | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 5c16bc56b..2fad19061 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -28,13 +28,11 @@ -include("emqx_conf.hrl"). --type file() :: string(). -type cipher() :: map(). -behaviour(hocon_schema). -reflect_type([ - file/0, cipher/0 ]). @@ -546,7 +544,7 @@ fields("node") -> )}, {"crash_dump_file", sc( - file(), + string(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), @@ -837,7 +835,7 @@ fields("rpc") -> )}, {"certfile", sc( - file(), + string(), #{ mapping => "gen_rpc.certfile", converter => fun ensure_unicode_path/2, @@ -846,7 +844,7 @@ fields("rpc") -> )}, {"keyfile", sc( - file(), + string(), #{ mapping => "gen_rpc.keyfile", converter => fun ensure_unicode_path/2, @@ -855,7 +853,7 @@ fields("rpc") -> )}, {"cacertfile", sc( - file(), + string(), #{ mapping => "gen_rpc.cacertfile", converter => fun ensure_unicode_path/2, @@ -1002,7 +1000,7 @@ fields("log_file_handler") -> [ {"path", sc( - file(), + string(), #{ desc => ?DESC("log_file_handler_file"), default => <<"${EMQX_LOG_DIR}/emqx.log">>, diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 16474b424..658666fc7 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -53,7 +53,7 @@ fields("log_audit_handler") -> {"path", hoconsc:mk( - emqx_conf_schema:file(), + string(), #{ desc => ?DESC(emqx_conf_schema, "audit_file_handler_path"), default => <<"${EMQX_LOG_DIR}/audit.log">>, From 40c2ca134d355690e7a1ce11f9c83945599bd991 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 16:33:47 +0100 Subject: [PATCH 04/21] refactor(schema): delete unused type cipher() --- apps/emqx_conf/src/emqx_conf_schema.erl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 2fad19061..6f1a13d4e 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -28,14 +28,8 @@ -include("emqx_conf.hrl"). --type cipher() :: map(). - -behaviour(hocon_schema). --reflect_type([ - cipher/0 -]). - -export([ namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0 ]). From df96798c53116da20b476f1e293aa0731707775d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 18:21:15 +0100 Subject: [PATCH 05/21] refactor(http_bridge): use more readable type for pool_type --- .../src/emqx_bridge_http_connector.erl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index b2f876d21..743ab97fe 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -46,14 +46,6 @@ -export([validate_method/1, join_paths/2]). --type connect_timeout() :: emqx_schema:duration() | infinity. --type pool_type() :: random | hash. - --reflect_type([ - connect_timeout/0, - pool_type/0 -]). - -define(DEFAULT_PIPELINE_SIZE, 100). -define(DEFAULT_REQUEST_TIMEOUT_MS, 30_000). @@ -89,7 +81,7 @@ fields(config) -> )}, {pool_type, sc( - pool_type(), + hoconsc:enum([random, hash]), #{ default => random, desc => ?DESC("pool_type") From b3df8604fca0feafe0e6ffd7c8fadc70731ce92d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 18:45:41 +0100 Subject: [PATCH 06/21] refactor(emqx_conf_schema): use hocon enum type for log levels --- apps/emqx_conf/src/emqx_conf_schema.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 6f1a13d4e..79877184d 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -33,6 +33,9 @@ -export([ namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0 ]). + +-export([log_level/0]). + -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([upgrade_raw_conf/1]). @@ -1279,7 +1282,7 @@ log_handler_common_confs(Handler, Default) -> [ {"level", sc( - hoconsc:enum([debug, info, notice, warning, error, critical, alert, emergency, all]), + log_level(), #{ default => maps:get(level, Default, warning), desc => ?DESC(LevelDesc), @@ -1528,3 +1531,6 @@ ensure_unicode_path(Path, _) when is_list(Path) -> Path; ensure_unicode_path(Path, _) -> throw({"not_string", Path}). + +log_level() -> + hoconsc:enum([debug, info, notice, warning, error, critical, alert, emergency, all]). From a2700771c96c1bc562ff706211b372d1055c5212 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 18:46:16 +0100 Subject: [PATCH 07/21] refactor(emqx_s3_schema): make type readable --- apps/emqx_s3/src/emqx_s3_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index db37c6e2d..5478f6416 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -37,7 +37,7 @@ fields(s3) -> )}, {secret_access_key, mk( - secret_access_key(), + typerefl:alias("string", secret_access_key()), #{ desc => ?DESC("secret_access_key"), required => false, From ae24b45d1b7322294808b9fed9583c5a1f81f5f3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 18:47:39 +0100 Subject: [PATCH 08/21] fix(schema): add type namespce to emqx_dashboard_sso_ldap --- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl index 499e24c5b..583f1d683 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl @@ -12,6 +12,7 @@ -behaviour(emqx_dashboard_sso). -export([ + namespace/0, fields/1, desc/1 ]). @@ -30,6 +31,9 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> + "sso". + hocon_ref() -> hoconsc:ref(?MODULE, ldap). From 46fffa401e90b992e7a85ffffce045c705356ee3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 19:42:44 +0100 Subject: [PATCH 09/21] refactor(emqx_prometheus_schema): use typerefl alias --- apps/emqx_prometheus/src/emqx_prometheus_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index f34675c0b..91d176142 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -57,7 +57,7 @@ fields("prometheus") -> )}, {headers, ?HOCON( - list({string(), string()}), + typerefl:alias("map", list({string(), string()})), #{ default => #{}, required => false, From 3e2ce57cb1ae53f7b93560466faeafdeea274f87 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 19:44:58 +0100 Subject: [PATCH 10/21] refactor(emqx_bridge_greptimedb): use typerefl alias --- .../emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl | 2 +- apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index d63103e2e..f5ae714d7 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -126,7 +126,7 @@ desc(_) -> undefined. write_syntax(type) -> - emqx_bridge_influxdb:write_syntax(); + emqx_bridge_influxdb:write_syntax_type(); write_syntax(required) -> true; write_syntax(validator) -> diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index 47eeecb4e..acb295752 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -11,7 +11,8 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - conn_bridge_examples/1 + conn_bridge_examples/1, + write_syntax_type/0 ]). -export([ @@ -29,6 +30,9 @@ %% ------------------------------------------------------------------------------------------------- %% api +write_syntax_type() -> + typerefl:alias("string", write_syntax()). + conn_bridge_examples(Method) -> [ #{ @@ -154,7 +158,7 @@ desc(_) -> undefined. write_syntax(type) -> - ?MODULE:write_syntax(); + write_syntax_type(); write_syntax(required) -> true; write_syntax(validator) -> From b24b66081ab7f3fd15369f2f304b18db6655d613 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 19:45:39 +0100 Subject: [PATCH 11/21] refactor(authn/authz_http_schema): use typerefl alias --- apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl | 4 ++-- apps/emqx_auth_http/src/emqx_authn_http_schema.erl | 2 +- apps/emqx_auth_http/src/emqx_authz_http_schema.erl | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl index 6d6ea420f..23532b4af 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl @@ -54,7 +54,7 @@ t_check_schema(_Config) -> ?assertThrow( #{ path := "authentication.1.password_hash_algorithm.name", - matched_type := "builtin_db/authn-hash:simple", + matched_type := "authn:builtin_db/authn-hash:simple", reason := unable_to_convert_to_enum_symbol }, Check(ConfigNotOk) @@ -73,7 +73,7 @@ t_check_schema(_Config) -> #{ path := "authentication.1.password_hash_algorithm", reason := "algorithm_name_missing", - matched_type := "builtin_db" + matched_type := "authn:builtin_db" }, Check(ConfigMissingAlgoName) ). diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index 9464b0aaf..19704d152 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -100,7 +100,7 @@ common_fields() -> {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, {url, fun url/1}, {body, - hoconsc:mk(map([{fuzzy, term(), binary()}]), #{ + hoconsc:mk(typerefl:alias("map", map([{fuzzy, term(), binary()}])), #{ required => false, desc => ?DESC(body) })}, {request_timeout, fun request_timeout/1} diff --git a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl index 5f9af846b..21f70de64 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl @@ -99,7 +99,7 @@ http_common_fields() -> mk_duration("Request timeout", #{ required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })}, - {body, ?HOCON(map(), #{required => false, desc => ?DESC(body)})} + {body, ?HOCON(hoconsc:map(name, binary()), #{required => false, desc => ?DESC(body)})} ] ++ lists:keydelete( pool_type, @@ -108,7 +108,7 @@ http_common_fields() -> ). headers(type) -> - list({binary(), binary()}); + typerefl:alias("map", list({binary(), binary()})); headers(desc) -> ?DESC(?FUNCTION_NAME); headers(converter) -> @@ -121,7 +121,7 @@ headers(_) -> undefined. headers_no_content_type(type) -> - list({binary(), binary()}); + typerefl:alias("map", list({binary(), binary()})); headers_no_content_type(desc) -> ?DESC(?FUNCTION_NAME); headers_no_content_type(converter) -> From 6b3aaf5b242a8f6f6f183172c0dfb5ea03582d14 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 19:46:08 +0100 Subject: [PATCH 12/21] refactor(emqx_bridge_gcp_pubsub): use typerefl alias --- apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 685fd3397..a42047b43 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -101,7 +101,7 @@ fields(connector_config) -> )}, {service_account_json, sc( - service_account_json(), + typerefl:alias("map", ?MODULE:service_account_json()), #{ required => true, validator => fun ?MODULE:service_account_json_validator/1, From 7b59d4685456f74c86835e3974514180392b99dd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 22:16:22 +0100 Subject: [PATCH 13/21] fix(schema): add namesapce to sso schema --- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl | 4 +++- apps/emqx_ldap/src/emqx_ldap.erl | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index 92f9ba519..aa032a3cc 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -8,7 +8,7 @@ -include_lib("typerefl/include/types.hrl"). %% Hocon --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). -export([ common_backend_schema/1, @@ -21,6 +21,8 @@ %%------------------------------------------------------------------------------ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "sso". + fields(sso) -> lists:map( fun({Type, Module}) -> diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index a77a8ecf0..bb16efbda 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -37,7 +37,7 @@ %% ecpool connect & reconnect -export([connect/1]). --export([roots/0, fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1]). -export([do_get_status/1]). @@ -57,6 +57,9 @@ %%===================================================================== %% Hocon schema + +namespace() -> "ldap". + roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. From d603de10e6ca26b010b587f46d1b3a9c17c45bae Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 23:13:26 +0100 Subject: [PATCH 14/21] refactor(schema): add namespace/0 to all schema modules --- .../src/emqx_bridge_rabbitmq_connector.erl | 4 +++- apps/emqx_ldap/src/emqx_ldap.erl | 2 +- apps/emqx_license/src/emqx_license_schema.erl | 4 +++- apps/emqx_rule_engine/src/emqx_rule_api_schema.erl | 4 +++- apps/emqx_telemetry/src/emqx_telemetry_schema.erl | 4 ++++ 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index ff439b676..2af1c16c8 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -20,7 +20,7 @@ -behaviour(ecpool_worker). %% hocon_schema callbacks --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). %% HTTP API callbacks -export([values/1]). @@ -43,6 +43,8 @@ %% Internal callbacks -export([publish_messages/3]). +namespace() -> "rabbitmq". + roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index bb16efbda..1c0c7124f 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -59,7 +59,7 @@ %% Hocon schema namespace() -> "ldap". - + roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. diff --git a/apps/emqx_license/src/emqx_license_schema.erl b/apps/emqx_license/src/emqx_license_schema.erl index 8f2d7f20d..f2b91811e 100644 --- a/apps/emqx_license/src/emqx_license_schema.erl +++ b/apps/emqx_license/src/emqx_license_schema.erl @@ -13,12 +13,14 @@ -behaviour(hocon_schema). --export([roots/0, fields/1, validations/0, desc/1, tags/0]). +-export([namespace/0, roots/0, fields/1, validations/0, desc/1, tags/0]). -export([ default_license/0 ]). +namespace() -> "license". + roots() -> [ {license, diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 0424bfb60..e9adbbdf6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -24,7 +24,7 @@ -export([check_params/2]). --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). -type tag() :: rule_creation | rule_test | rule_engine. @@ -46,6 +46,8 @@ check_params(Params, Tag) -> %%====================================================================================== %% Hocon Schema Definitions +namespace() -> "rule_engine". + roots() -> [ {"rule_engine", sc(ref("rule_engine"), #{desc => ?DESC("root_rule_engine")})}, diff --git a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl index 1e1f547c5..586b70f72 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl @@ -22,11 +22,15 @@ -behaviour(hocon_schema). -export([ + namespace/0, roots/0, fields/1, desc/1 ]). +%% 'emqxtel' to distinguish open-telemetry +namespace() -> "emqxtel". + roots() -> ["telemetry"]. fields("telemetry") -> From 8a4fba431e39a20307f5bf959fbad1506ab1b01c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 19:43:56 +0100 Subject: [PATCH 15/21] refactor(emqx_limiter_schema): use typerefl alias --- .../emqx_limiter/src/emqx_limiter_schema.erl | 24 +++++++++++++------ .../src/emqx_retainer_schema.erl | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 2dd4aa241..09ab6099c 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -33,7 +33,8 @@ desc/1, types/0, short_paths/0, - short_paths_fields/0 + short_paths_fields/0, + rate_type/0 ]). -define(KILOBYTE, 1024). @@ -129,9 +130,9 @@ fields(limiter) -> ]; fields(node_opts) -> [ - {rate, ?HOCON(rate(), #{desc => deprecated_desc(rate), default => <<"infinity">>})}, + {rate, ?HOCON(rate_type(), #{desc => deprecated_desc(rate), default => <<"infinity">>})}, {burst, - ?HOCON(burst_rate(), #{ + ?HOCON(burst_rate_type(), #{ desc => deprecated_desc(burst), default => <<"0">> })} @@ -142,7 +143,7 @@ fields(bucket_opts) -> fields_of_bucket(<<"infinity">>); fields(client_opts) -> [ - {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => deprecated_desc(rate)})}, + {rate, ?HOCON(rate_type(), #{default => <<"infinity">>, desc => deprecated_desc(rate)})}, {initial, ?HOCON(initial(), #{ default => <<"0">>, @@ -164,7 +165,7 @@ fields(client_opts) -> } )}, {burst, - ?HOCON(burst(), #{ + ?HOCON(burst_type(), #{ desc => deprecated_desc(burst), default => <<"0">>, importance => ?IMPORTANCE_HIDDEN, @@ -211,7 +212,7 @@ short_paths_fields() -> short_paths_fields(Importance) -> [ {Name, - ?HOCON(rate(), #{ + ?HOCON(rate_type(), #{ desc => ?DESC(Name), required => false, importance => Importance, @@ -415,7 +416,7 @@ composite_bucket_fields(Types, ClientRef) -> fields_of_bucket(Default) -> [ - {rate, ?HOCON(rate(), #{desc => deprecated_desc(rate), default => Default})}, + {rate, ?HOCON(rate_type(), #{desc => deprecated_desc(rate), default => Default})}, {burst, ?HOCON(burst(), #{ desc => deprecated_desc(burst), @@ -461,3 +462,12 @@ alias_of_type(_) -> deprecated_desc(_Field) -> <<"Deprecated since v5.0.25">>. + +rate_type() -> + typerefl:alias("string", rate()). + +burst_type() -> + typerefl:alias("string", burst()). + +burst_rate_type() -> + typerefl:alias("string", burst_rate()). diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 7b1a9675e..983b27601 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -77,7 +77,7 @@ fields("retainer") -> )}, {delivery_rate, ?HOCON( - emqx_limiter_schema:rate(), + emqx_limiter_schema:rate_type(), #{ required => false, desc => ?DESC(delivery_rate), From 86110824eb2ade0a5b5a9a3fa3a89af6b3841941 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 10 Nov 2023 08:52:52 +0100 Subject: [PATCH 16/21] feat: upgrade hocon to 0.40.0 which supports union type display name --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl | 4 ++-- apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl | 1 - apps/emqx_conf/src/emqx_conf.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_listeners.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- mix.exs | 2 +- rebar.config | 2 +- 11 files changed, 11 insertions(+), 12 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9f67caf5d..71f581267 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.19"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 3848e77b4..4ed8a3107 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3316,7 +3316,7 @@ get_tombstone_map_value_type(Schema) -> %% hoconsc:map_value_type(Schema) ?MAP(_Name, Union) = hocon_schema:field_schema(Schema, type), %% TODO: violation of abstraction, fix hoconsc:union_members/1 - ?UNION(Members) = Union, + ?UNION(Members, _) = Union, Tombstone = tombstone(), [Type, Tombstone] = hoconsc:union_members(Members), Type. diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl index ac2c2503d..426c7a9f6 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl @@ -136,7 +136,7 @@ authz_fields() -> [ {sources, ?HOCON( - ?ARRAY(?UNION(UnionMemberSelector)), + ?ARRAY(hoconsc:union(UnionMemberSelector)), #{ default => [default_authz()], desc => ?DESC(sources), @@ -153,7 +153,7 @@ api_authz_fields() -> [{sources, ?HOCON(?ARRAY(api_source_type()), #{desc => ?DESC(sources)})}]. api_source_type() -> - ?UNION(api_authz_refs()). + hoconsc:union(api_authz_refs()). api_authz_refs() -> lists:concat([api_source_refs(Mod) || Mod <- source_schema_mods()]). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl index 747a1d15a..61a15b139 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl @@ -16,7 +16,6 @@ -module(emqx_authn_chains_SUITE). --behaviour(hocon_schema). -behaviour(emqx_authn_provider). -compile(export_all). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index c4bd0efc9..1fb3bab28 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -292,7 +292,7 @@ hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> }, SubRefs }; -hocon_schema_to_spec(?UNION(Types), LocalModule) -> +hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) -> {OneOf, Refs} = lists:foldl( fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 79877184d..3a2b5d972 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -978,7 +978,7 @@ fields("log") -> })}, {"file", sc( - ?UNION([ + hoconsc:union([ ?R_REF("log_file_handler"), ?MAP(handler_name, ?R_REF("log_file_handler")) ]), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 75e93fdd1..4a8072804 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -776,7 +776,7 @@ hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> }, SubRefs }; -hocon_schema_to_spec(?UNION(Types), LocalModule) -> +hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) -> {OneOf, Refs} = lists:foldl( fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 8295047b9..1718a14cf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -313,7 +313,7 @@ create_listener_schema(Opts) -> ], Example = maps:remove(id, tcp_schema_example()), emqx_dashboard_swagger:schema_with_example( - ?UNION(Schemas), + hoconsc:union(Schemas), Example#{name => <<"demo">>} ). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index d0019a1c5..c6d3c7ff8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -262,7 +262,7 @@ actions() -> end. qos() -> - ?UNION([emqx_schema:qos(), binary()]). + hoconsc:union([emqx_schema:qos(), binary()]). rule_engine_settings() -> [ diff --git a/mix.exs b/mix.exs index 18e72a6a8..463642c93 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.19", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.40.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index c7101abc3..fed48e3aa 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.19"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From f1de0aa1761636bffe43a50a27fc46ed1671336f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 10 Nov 2023 09:33:15 +0100 Subject: [PATCH 17/21] fix(schema): add namespace to authn schemas --- .../src/emqx_authn/emqx_authn_schema.erl | 6 +++- .../emqx_authn/emqx_authn_schema_tests.erl | 30 ++++++++++++------- .../src/emqx_authn_http_schema.erl | 8 ++--- .../src/emqx_authn_jwt_schema.erl | 9 ++++-- .../src/emqx_authn_ldap_bind_schema.erl | 6 ++-- .../src/emqx_authn_ldap_schema.erl | 9 ++++-- .../src/emqx_authn_scram_mnesia_schema.erl | 3 ++ .../src/emqx_authn_mongodb_schema.erl | 10 +++---- .../src/emqx_authz_mongodb_schema.erl | 6 ++-- .../src/emqx_authn_mysql_schema.erl | 6 ++-- .../test/emqx_authn_postgresql_SUITE.erl | 2 +- .../test/emqx_authn_redis_SUITE.erl | 2 +- .../src/emqx_gcp_device_authn_schema.erl | 9 ++++-- apps/emqx_mongodb/src/emqx_mongodb.erl | 7 ++++- 14 files changed, 71 insertions(+), 42 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index 9b9935a1f..371c6f2be 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -38,7 +38,8 @@ authenticator_type_without/1, authenticator_type_without/2, mechanism/1, - backend/1 + backend/1, + namespace/0 ]). -export([ @@ -60,6 +61,7 @@ api_write %% config: schema for config validation | config. +-callback namespace() -> string(). -callback refs() -> [schema_ref()]. -callback refs(shema_kind()) -> [schema_ref()]. -callback select_union_member(emqx_config:raw_config()) -> [schema_ref()] | undefined | no_return(). @@ -74,6 +76,8 @@ refs/1 ]). +namespace() -> "authn". + roots() -> []. injected_fields(AuthnSchemaMods) -> diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl index b0451e110..b4835cdaa 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl @@ -22,6 +22,7 @@ -define(ERR(Reason), {error, Reason}). union_member_selector_mongo_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -31,25 +32,26 @@ union_member_selector_mongo_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_single"}), + ?ERR(#{matched_type := "authn:mongo_single"}), check("{mechanism = password_based, backend = mongodb, mongo_type = single}") ) end}, {"replica-set", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_rs"}), + ?ERR(#{matched_type := "authn:mongo_rs"}), check("{mechanism = password_based, backend = mongodb, mongo_type = rs}") ) end}, {"sharded", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_sharded"}), + ?ERR(#{matched_type := "authn:mongo_sharded"}), check("{mechanism = password_based, backend = mongodb, mongo_type = sharded}") ) end} ]. union_member_selector_jwt_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -59,25 +61,26 @@ union_member_selector_jwt_test_() -> end}, {"jwks", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_jwks"}), + ?ERR(#{matched_type := "authn:jwt_jwks"}), check("{mechanism = jwt, use_jwks = true}") ) end}, {"publick-key", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_public_key"}), + ?ERR(#{matched_type := "authn:jwt_public_key"}), check("{mechanism = jwt, use_jwks = false, public_key = 1}") ) end}, {"hmac-based", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_hmac"}), + ?ERR(#{matched_type := "authn:jwt_hmac"}), check("{mechanism = jwt, use_jwks = false}") ) end} ]. union_member_selector_redis_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -87,25 +90,26 @@ union_member_selector_redis_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_single"}), + ?ERR(#{matched_type := "authn:redis_single"}), check("{mechanism = password_based, backend = redis, redis_type = single}") ) end}, {"cluster", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_cluster"}), + ?ERR(#{matched_type := "authn:redis_cluster"}), check("{mechanism = password_based, backend = redis, redis_type = cluster}") ) end}, {"sentinel", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_sentinel"}), + ?ERR(#{matched_type := "authn:redis_sentinel"}), check("{mechanism = password_based, backend = redis, redis_type = sentinel}") ) end} ]. union_member_selector_http_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -115,13 +119,13 @@ union_member_selector_http_test_() -> end}, {"get", fun() -> ?assertMatch( - ?ERR(#{matched_type := "http_get"}), + ?ERR(#{matched_type := "authn:http_get"}), check("{mechanism = password_based, backend = http, method = get}") ) end}, {"post", fun() -> ?assertMatch( - ?ERR(#{matched_type := "http_post"}), + ?ERR(#{matched_type := "authn:http_post"}), check("{mechanism = password_based, backend = http, method = post}") ) end} @@ -132,3 +136,7 @@ check(HoconConf) -> #{roots => emqx_authn_schema:global_auth_fields()}, ["authentication= ", HoconConf] ). + +ensure_schema_load() -> + _ = emqx_conf_schema:roots(), + ok. diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index 19704d152..7b7af727d 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -16,10 +16,6 @@ -module(emqx_authn_http_schema). --include("emqx_auth_http.hrl"). --include_lib("emqx_auth/include/emqx_authn.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ @@ -31,6 +27,10 @@ namespace/0 ]). +-include("emqx_auth_http.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)). -define(THROW_VALIDATION_ERROR(ERROR, MESSAGE), throw(#{ diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl index fc7de7cd8..9118d3d1b 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_jwt_schema). --include("emqx_auth_jwt.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_jwt.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [ ?R_REF(jwt_hmac), diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl index 580673d85..e5e83daa1 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl @@ -16,9 +16,6 @@ -module(emqx_authn_ldap_bind_schema). --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ @@ -29,6 +26,9 @@ namespace/0 ]). +-include("emqx_auth_ldap.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + namespace() -> "authn". refs() -> diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl index c26ca94e8..fe9917fa1 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_ldap_schema). --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_ldap.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(ldap)]. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl index fa22693b3..ef4ec6e05 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + refs() -> [?R_REF(scram)]. diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl index 53c6a6a10..b72a1e83a 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl @@ -16,19 +16,19 @@ -module(emqx_authn_mongodb_schema). --include("emqx_auth_mongodb.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, - select_union_member/1, - namespace/0 + select_union_member/1 ]). +-include("emqx_auth_mongodb.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + namespace() -> "authn". refs() -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl index ee20b962e..bdde704f9 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl @@ -16,9 +16,6 @@ -module(emqx_authz_mongodb_schema). --include("emqx_auth_mongodb.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -export([ type/0, fields/1, @@ -28,6 +25,9 @@ namespace/0 ]). +-include("emqx_auth_mongodb.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + namespace() -> "authz". type() -> ?AUTHZ_TYPE. diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl index 17609acb8..6472794fe 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl @@ -16,9 +16,6 @@ -module(emqx_authn_mysql_schema). --include("emqx_auth_mysql.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ @@ -29,6 +26,9 @@ select_union_member/1 ]). +-include("emqx_auth_mysql.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + namespace() -> "authn". refs() -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index ea44c0a45..af1f1db2d 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -104,7 +104,7 @@ t_update_with_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "postgresql", + matched_type := "authn:postgresql", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index 081c4e641..e7673b790 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -170,7 +170,7 @@ test_create_invalid_config(InvalidAuthConfig, Path) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "redis_single", + matched_type := "authn:redis_single", path := Path }}, emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, InvalidAuthConfig}) diff --git a/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl b/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl index a01c6d0e4..975e17ff0 100644 --- a/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl +++ b/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl @@ -16,18 +16,21 @@ -module(emqx_gcp_device_authn_schema). --include("emqx_gcp_device.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_gcp_device.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(gcp_device)]. select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN}) -> diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index 77161911a..a5795a554 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -22,6 +22,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(emqx_resource). +-behaviour(hocon_schema). %% callbacks of behaviour emqx_resource -export([ @@ -29,7 +30,8 @@ on_start/2, on_stop/2, on_query/3, - on_get_status/2 + on_get_status/2, + namespace/0 ]). %% ecpool callback @@ -50,6 +52,9 @@ }). %%===================================================================== + +namespace() -> "mongo". + roots() -> [ {config, #{ From 855b3c5b294e104195e7e32c471932c9904c3d3d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 10 Nov 2023 09:58:02 +0100 Subject: [PATCH 18/21] test: ensure atom exists --- apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl index 7169ea3d2..29299dcc9 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl @@ -11,7 +11,7 @@ %%=========================================================================== pulsar_producer_validations_test() -> - Name = my_producer, + Name = list_to_atom("my_producer"), Conf0 = pulsar_producer_hocon(), Conf1 = Conf0 ++ From 1d77d07774e34d45f89404406299ab6393b1ab3f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 10 Nov 2023 10:30:22 +0100 Subject: [PATCH 19/21] build: log red text for error message --- scripts/apps-version-check.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index b32b39fd1..b76e8d345 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -4,6 +4,12 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +log_red() { + local RED='\033[0;31m' # Red + local NC='\033[0m' # No Color + echo -e "${RED}${1}${NC}" +} + # match any official release tag 'e*' and 'v*' latest_release="$(env PREV_TAG_MATCH_PATTERN='*' ./scripts/find-prev-rel-tag.sh)" echo "Version check compare base: $latest_release" @@ -47,7 +53,7 @@ for app in ${APPS}; do -- "$app_path/priv" \ -- "$app_path/c_src" | wc -l ) " if [ "$changed_lines" -gt 0 ]; then - echo "ERROR: $src_file needs a vsn bump" + log_red "ERROR: $src_file needs a vsn bump" bad_app_count=$(( bad_app_count + 1)) fi else From e653c6b4e08ac17f04e4b2e53de9f09b5459e31c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 23:41:06 +0100 Subject: [PATCH 20/21] chore: bump app versions --- apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src | 2 +- apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src | 2 +- apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src | 2 +- apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src | 2 +- apps/emqx_mongodb/src/emqx_mongodb.app.src | 2 +- apps/emqx_telemetry/src/emqx_telemetry.app.src | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index 4c5a15b79..a8a938a0b 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_greptimedb, [ {description, "EMQX GreptimeDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 27fe1659c..c6236d97c 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index c7d931c93..7e32b5a89 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index e00a3cbfa..71788947b 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index eb846a7ab..2212ac7d4 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_telemetry/src/emqx_telemetry.app.src b/apps/emqx_telemetry/src/emqx_telemetry.app.src index d9483298f..32c2baa91 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry.app.src +++ b/apps/emqx_telemetry/src/emqx_telemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_telemetry, [ {description, "Report telemetry data for EMQX Opensource edition"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, [emqx_telemetry_sup, emqx_telemetry]}, {mod, {emqx_telemetry_app, []}}, {applications, [ From 101990b8fcf61781caa384e5c24da5f73bfb98b3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 10 Nov 2023 11:43:04 +0100 Subject: [PATCH 21/21] test: fix schema tests after types change --- .../src/emqx_dashboard_swagger.erl | 28 ++-- .../test/emqx_swagger_response_SUITE.erl | 125 +++++++++--------- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 4a8072804..4a316e248 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -345,15 +345,7 @@ parse_spec_ref(Module, Path, Options) -> erlang:apply(Module, schema, [Path]) catch Error:Reason:Stacktrace -> - %% This error is intended to fail the build - %% hence print to standard_error - io:format( - standard_error, - "Failed to generate swagger for path ~p in module ~p~n" - "error:~p~nreason:~p~n~p~n", - [Module, Path, Error, Reason, Stacktrace] - ), - error({failed_to_generate_swagger_spec, Module, Path}) + failed_to_generate_swagger_spec(Module, Path, Error, Reason, Stacktrace) end, OperationId = maps:get('operationId', Schema), {Specs, Refs} = maps:fold( @@ -369,6 +361,24 @@ parse_spec_ref(Module, Path, Options) -> RouteOpts = generate_route_opts(Schema, Options), {OperationId, Specs, Refs, RouteOpts}. +-ifdef(TEST). +-spec failed_to_generate_swagger_spec(_, _, _, _, _) -> no_return(). +failed_to_generate_swagger_spec(Module, Path, _Error, _Reason, _Stacktrace) -> + error({failed_to_generate_swagger_spec, Module, Path}). +-else. +-spec failed_to_generate_swagger_spec(_, _, _, _, _) -> no_return(). +failed_to_generate_swagger_spec(Module, Path, Error, Reason, Stacktrace) -> + %% This error is intended to fail the build + %% hence print to standard_error + io:format( + standard_error, + "Failed to generate swagger for path ~p in module ~p~n" + "error:~p~nreason:~p~n~p~n", + [Module, Path, Error, Reason, Stacktrace] + ), + error({failed_to_generate_swagger_spec, Module, Path}). + +-endif. generate_route_opts(Schema, Options) -> #{filter => compose_filters(filter(Options), custom_filter(Schema))}. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index d84f17c44..376ace5c2 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -317,68 +317,72 @@ t_sub_fields(_Config) -> validate(Path, Object, ExpectRefs), ok. -t_complicated_type(_Config) -> +t_complex_type(_Config) -> Path = "/ref/complex_type", - Object = #{ - <<"content">> => #{ - <<"application/json">> => - #{ - <<"schema">> => #{ - <<"properties">> => - [ - {<<"no_neg_integer">>, #{minimum => 0, type => integer}}, - {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}}, - {<<"server">>, #{example => <<"127.0.0.1:80">>, type => string}}, - {<<"connect_timeout">>, #{ - example => infinity, - <<"oneOf">> => [ - #{example => infinity, type => string}, - #{type => integer} - ] - }}, - {<<"pool_type">>, #{enum => [random, hash], type => string}}, - {<<"timeout">>, #{ - example => infinity, - <<"oneOf">> => [ - #{example => infinity, type => string}, #{type => integer} - ] - }}, - {<<"bytesize">>, #{example => <<"32MB">>, type => string}}, - {<<"wordsize">>, #{example => <<"1024KB">>, type => string}}, - {<<"maps">>, #{example => #{}, type => object}}, - {<<"comma_separated_list">>, #{ - example => <<"item1,item2">>, type => string - }}, - {<<"comma_separated_atoms">>, #{ - example => <<"item1,item2">>, type => string - }}, - {<<"log_level">>, #{ - enum => [ - debug, - info, - notice, - warning, - error, - critical, - alert, - emergency, - all - ], - type => string - }}, - {<<"fix_integer">>, #{ - default => 100, enum => [100], type => integer - }} - ], - <<"type">> => object - } - } - } - }, {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), - ?assertEqual(Object, maps:get(<<"200">>, Response)), + ResponseBody = maps:get(<<"200">>, Response), + Content = maps:get(<<"content">>, ResponseBody), + JsonContent = maps:get(<<"application/json">>, Content), + Schema = maps:get(<<"schema">>, JsonContent), + ?assertMatch(#{<<"type">> := object}, Schema), + Properties = maps:get(<<"properties">>, Schema), + ?assertMatch( + [ + {<<"no_neg_integer">>, #{minimum := 0, type := integer}}, + {<<"url">>, #{ + example := <<"http://127.0.0.1">>, type := string + }}, + {<<"server">>, #{ + example := <<"127.0.0.1:80">>, type := string + }}, + {<<"connect_timeout">>, #{ + example := _, type := string + }}, + {<<"pool_type">>, #{ + enum := [random, hash], type := string + }}, + {<<"timeout">>, #{ + example := infinity, + <<"oneOf">> := [ + #{example := infinity, type := string}, + #{type := integer} + ] + }}, + {<<"bytesize">>, #{ + example := <<"32MB">>, type := string + }}, + {<<"wordsize">>, #{ + example := <<"1024KB">>, type := string + }}, + {<<"maps">>, #{example := #{}, type := object}}, + {<<"comma_separated_list">>, #{ + example := <<"item1,item2">>, type := string + }}, + {<<"comma_separated_atoms">>, #{ + example := <<"item1,item2">>, type := string + }}, + {<<"log_level">>, #{ + enum := [ + debug, + info, + notice, + warning, + error, + critical, + alert, + emergency, + all + ], + type := string + }}, + {<<"fix_integer">>, #{ + default := 100, enum := [100], type := integer + }} + ], + Properties + ), ?assertEqual([], Refs), ok. @@ -647,9 +651,8 @@ schema("/ref/complex_type") -> {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, {url, hoconsc:mk(url(), #{})}, {server, hoconsc:mk(emqx_schema:ip_port(), #{})}, - {connect_timeout, - hoconsc:mk(emqx_bridge_http_connector:connect_timeout(), #{})}, - {pool_type, hoconsc:mk(emqx_bridge_http_connector:pool_type(), #{})}, + {connect_timeout, hoconsc:mk(emqx_schema:timeout_duration(), #{})}, + {pool_type, hoconsc:mk(hoconsc:enum([random, hash]), #{})}, {timeout, hoconsc:mk(timeout(), #{})}, {bytesize, hoconsc:mk(emqx_schema:bytesize(), #{})}, {wordsize, hoconsc:mk(emqx_schema:wordsize(), #{})},