From 1b2c0526468792b8984630404977f420a6d935d6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 16 Nov 2023 20:36:55 +0100 Subject: [PATCH 1/9] docs: add type namespaces --- .../src/emqx_authz/sources/emqx_authz_file_schema.erl | 3 +++ apps/emqx_conf/src/emqx_conf_schema.erl | 3 +-- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl | 3 +++ apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl | 2 +- apps/emqx_gateway_coap/src/emqx_coap_schema.erl | 4 +++- apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl | 4 +++- apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src | 2 +- apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src | 2 +- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl | 4 +++- apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src | 2 +- apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl | 4 +++- apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl | 4 +++- apps/emqx_retainer/src/emqx_retainer_schema.erl | 2 +- apps/emqx_schema_registry/src/emqx_schema_registry.app.src | 2 +- apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl | 3 +++ 15 files changed, 31 insertions(+), 13 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authz/sources/emqx_authz_file_schema.erl b/apps/emqx_auth/src/emqx_authz/sources/emqx_authz_file_schema.erl index cea697d66..ae06147ff 100644 --- a/apps/emqx_auth/src/emqx_authz/sources/emqx_authz_file_schema.erl +++ b/apps/emqx_auth/src/emqx_authz/sources/emqx_authz_file_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -30,6 +31,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(file) -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 3a2b5d972..a872a6a56 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -79,8 +79,7 @@ upgrade_raw_conf(RawConf) -> emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2(RawConf). -%% root config should not have a namespace -namespace() -> undefined. +namespace() -> emqx. tags() -> [<<"EMQX">>]. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl index 907d2dcde..42a0e8f74 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl @@ -12,6 +12,7 @@ -behaviour(emqx_dashboard_sso). -export([ + namespace/0, hocon_ref/0, login_ref/0, fields/1, @@ -43,6 +44,8 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "dashboard". + hocon_ref() -> hoconsc:ref(?MODULE, saml). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index aa032a3cc..a73f13ca8 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -21,7 +21,7 @@ %%------------------------------------------------------------------------------ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "sso". +namespace() -> dashboard. fields(sso) -> lists:map( diff --git a/apps/emqx_gateway_coap/src/emqx_coap_schema.erl b/apps/emqx_gateway_coap/src/emqx_coap_schema.erl index b7ce88451..c4879f553 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_schema.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_schema.erl @@ -26,7 +26,9 @@ -reflect_type([duration/0]). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). + +namespace() -> "gateway". fields(coap) -> [ diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl index 10583e41a..7eeceb3cb 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl @@ -28,7 +28,9 @@ ]). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). + +namespace() -> "gateway". fields(exproto) -> [ diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 09622763b..ffd8fd3d1 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index e5afd7871..371f74625 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl index b674c3260..41df3b970 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl @@ -28,7 +28,9 @@ -reflect_type([duration/0, duration_s/0]). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). + +namespace() -> gateway. fields(lwm2m) -> [ diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index c2f6d642b..a7de83b74 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl index 08fb854b4..e028a698b 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl @@ -21,7 +21,9 @@ -include_lib("typerefl/include/types.hrl"). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). + +namespace() -> "gateway". fields(mqttsn) -> [ diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl index b1c6a92e2..d4dcd2897 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl @@ -20,7 +20,9 @@ -include_lib("typerefl/include/types.hrl"). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). + +namespace() -> "gateway". fields(stomp) -> [ diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 983b27601..1c5d8e55f 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -30,7 +30,7 @@ -define(INVALID_SPEC(_REASON_), throw({_REASON_, #{default => ?DEFAULT_INDICES}})). -namespace() -> "retainer". +namespace() -> retainer. roots() -> [ diff --git a/apps/emqx_schema_registry/src/emqx_schema_registry.app.src b/apps/emqx_schema_registry/src/emqx_schema_registry.app.src index e64d104f7..f4089fdc1 100644 --- a/apps/emqx_schema_registry/src/emqx_schema_registry.app.src +++ b/apps/emqx_schema_registry/src/emqx_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, [emqx_schema_registry_sup]}, {mod, {emqx_schema_registry_app, []}}, {included_applications, [ diff --git a/apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl b/apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl index d131aa48f..564496629 100644 --- a/apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl +++ b/apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl @@ -10,6 +10,7 @@ %% `hocon_schema' API -export([ + namespace/0, roots/0, fields/1, desc/1, @@ -26,6 +27,8 @@ %% `hocon_schema' APIs %%------------------------------------------------------------------------------ +namespace() -> ?CONF_KEY_ROOT. + roots() -> [{?CONF_KEY_ROOT, mk(ref(?CONF_KEY_ROOT), #{required => false})}]. From db33bc616ae1b942fd73b23c788c98a5d2af6105 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 23:31:29 +0100 Subject: [PATCH 2/9] feat(schema): Add v2 scheam JSON dump --- .../emqx_authn_password_hashing.erl | 2 +- .../src/emqx_bridge_http_connector.erl | 2 +- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector_schema.erl | 2 +- apps/emqx_conf/src/emqx_conf.erl | 271 +++++++++++++++++- apps/emqx_psk/src/emqx_psk.app.src | 2 +- apps/emqx_psk/src/emqx_psk_schema.erl | 2 +- 7 files changed, 266 insertions(+), 17 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl index 756f39d06..16af4fd23 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl @@ -67,7 +67,7 @@ -define(SALT_ROUNDS_MIN, 5). -define(SALT_ROUNDS_MAX, 10). -namespace() -> "authn-hash". +namespace() -> "authn_hash". roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple]. fields(bcrypt_rw) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 743ab97fe..5a5e790e5 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -54,7 +54,7 @@ %%===================================================================== %% Hocon schema -namespace() -> "connector-http". +namespace() -> "connector_http". roots() -> fields(config). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e39c4df69..cbef0dda8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 1dc3ca5f8..f671bec71 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -36,7 +36,7 @@ -define(MQTT_HOST_OPTS, #{default_port => 1883}). -namespace() -> "connector-mqtt". +namespace() -> "connector_mqtt". roots() -> fields("config"). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 7ff06b0ef..0925141de 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -28,7 +28,7 @@ -export([remove/2, remove/3]). -export([tombstone/2]). -export([reset/2, reset/3]). --export([dump_schema/2]). +-export([dump_schema/2, reformat_schema_dump/1]). -export([schema_module/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time @@ -180,9 +180,263 @@ gen_schema_json(Dir, SchemaModule, Lang) -> include_importance_up_from => IncludeImportance, desc_resolver => make_desc_resolver(Lang) }, - JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), - ok = file:write_file(SchemaJsonFile, IoData). + StructsJsonArray = hocon_schema_json:gen(SchemaModule, Opts), + IoData = emqx_utils_json:encode(StructsJsonArray, [pretty, force_utf8]), + ok = file:write_file(SchemaJsonFile, IoData), + ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang). + +gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> + NestedStruct = reformat_schema_dump(StructsJsonArray), + %% write to files + NestedJsonFile = filename:join([Dir, "schmea-v2-" ++ Lang ++ ".json"]), + io:format(user, "===< Generating: ~s~n", [NestedJsonFile]), + ok = file:write_file( + NestedJsonFile, emqx_utils_json:encode(NestedStruct, [pretty, force_utf8]) + ), + ok. + +%% @doc This function is exported for scripts/schema-dump-reformat.escript +reformat_schema_dump(StructsJsonArray0) -> + %% prepare + StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0), + #{fields := RootFields} = hd(StructsJsonArray), + RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields), + RootNames = lists:map(fun to_bin/1, RootNames0), + %% reformat + [Root | FlatStructs0] = lists:map( + fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray + ), + FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0], + gen_nested_doc(FlatStructs). + +deduplicate_by_full_name(Structs) -> + deduplicate_by_full_name(Structs, #{}, []). + +deduplicate_by_full_name([], _Seen, Acc) -> + lists:reverse(Acc); +deduplicate_by_full_name([#{full_name := FullName} = H | T], Seen, Acc) -> + case maps:get(FullName, Seen, false) of + false -> + deduplicate_by_full_name(T, Seen#{FullName => H}, [H | Acc]); + H -> + %% Name clash, but identical, ignore + deduplicate_by_full_name(T, Seen, Acc); + _Different -> + %% ADD NAMESPACE! + throw({duplicate_full_name, FullName}) + end. + +%% Ggenerate nested docs from root struct. +%% Due to the fact that the same struct can be referenced by multiple fields, +%% we need to generate a unique nested doc for each reference. +%% The unique path to each type and is of the below format: +%% - A a path starts either with 'T-' or 'V-'. T stands for type, V stands for value. +%% - A path is a list of strings delimited by '-'. +%% - The letter S is used to separate struct name from field name. +%% - Field names are however NOT denoted by a leading 'F-'. +%% For example: +%% - T-root: the root struct; +%% - T-foo-S-footype: the struct named "footype" in the foo field of root struct; +%% - V-foo-S-footype-bar: the field named "bar" in the struct named "footype" in the foo field of root struct +gen_nested_doc(Structs) -> + KeyByFullName = lists:foldl( + fun(#{hash := FullName} = Struct, Acc) -> + maps:put(FullName, Struct, Acc) + end, + #{}, + Structs + ), + FindFn = fun(Hash) -> maps:get(Hash, KeyByFullName) end, + gen_nested_doc(hd(Structs), FindFn, []). + +gen_nested_doc(#{fields := Fields} = Struct, FindFn, Path) -> + TypeAnchor = make_type_anchor(Path), + ValueAnchor = fun(FieldName) -> make_value_anchor(Path, FieldName) end, + NewFields = lists:map( + fun(#{text := Name} = Field) -> + NewField = expand_field(Field, FindFn, Path), + NewField#{hash => ValueAnchor(Name)} + end, + Fields + ), + Struct#{ + fields => NewFields, + hash => TypeAnchor + }. + +%% Make anchor for type. +%% Start with "T-" to distinguish from value anchor. +make_type_anchor([]) -> + <<"T-root">>; +make_type_anchor(Path) -> + to_bin(["T-", lists:join("-", lists:reverse(Path))]). + +%% Value anchor is used to link to the field's struct. +%% Start with "V-" to distinguish from type anchor. +make_value_anchor(Path, FieldName) -> + to_bin(["V-", join_path_hash(Path, FieldName)]). + +%% Make a globally unique "hash" (the http anchor) for each struct field. +join_path_hash([], Name) -> + Name; +join_path_hash(Path, Name) -> + to_bin(lists:join("-", lists:reverse([Name | Path]))). + +%% Expand field's struct reference to nested doc. +expand_field(#{text := Name, refs := References} = Field, FindFn, Path) -> + %% Add struct type name in path to make it unique. + NewReferences = lists:map( + fun(#{text := StructName} = Ref) -> + expand_ref(Ref, FindFn, [StructName, "S", Name | Path]) + end, + References + ), + Field#{refs => NewReferences}; +expand_field(Field, _FindFn, _Path) -> + %% No reference, no need to expand. + Field. + +expand_ref(#{hash := FullName}, FindFn, Path) -> + Struct = FindFn(FullName), + gen_nested_doc(Struct, FindFn, Path). + +%% generate flat docs for each struct. +%% using references to link to other structs. +gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> + ShortName = short_name(FullName), + case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of + true -> + io:format(standard_error, "WARN: no_namespace_for: ~s~n", [FullName]); + false -> + ok + end, + #{ + text => short_name(FullName), + hash => format_hash(FullName), + doc => maps:get(desc, S, <<"">>), + fields => format_fields(Fields) + }. + +format_fields([]) -> + []; +format_fields([Field | Fields]) -> + [format_field(Field) | format_fields(Fields)]. + +format_field(#{name := Name, aliases := Aliases, type := Type} = F) -> + L = [ + {text, Name}, + {type, format_type(Type)}, + {refs, format_refs(Type)}, + {aliases, + case Aliases of + [] -> undefined; + _ -> Aliases + end}, + {default, maps:get(hocon, maps:get(default, F, #{}), undefined)}, + {doc, maps:get(desc, F, undefined)} + ], + maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]). + +format_refs(Type) -> + References = find_refs(Type), + case lists:map(fun format_ref/1, References) of + [] -> undefined; + L -> L + end. + +format_ref(FullName) -> + #{text => short_name(FullName), hash => format_hash(FullName)}. + +find_refs(Type) -> + lists:reverse(find_refs(Type, [])). + +%% go deep into union, array, and map to find references +find_refs(#{kind := union, members := Members}, Acc) -> + lists:foldl(fun find_refs/2, Acc, Members); +find_refs(#{kind := array, elements := Elements}, Acc) -> + find_refs(Elements, Acc); +find_refs(#{kind := map, values := Values}, Acc) -> + find_refs(Values, Acc); +find_refs(#{kind := struct, name := FullName}, Acc) -> + [FullName | Acc]; +find_refs(_, Acc) -> + Acc. + +format_type(#{kind := primitive, name := Name}) -> + format_primitive_type(Name); +format_type(#{kind := singleton, name := Name}) -> + to_bin(["String(\"", to_bin(Name), "\")"]); +format_type(#{kind := enum, symbols := Symbols}) -> + CommaSep = lists:join(",", lists:map(fun(S) -> to_bin(S) end, Symbols)), + to_bin(["Enum(", CommaSep, ")"]); +format_type(#{kind := array, elements := ElementsType}) -> + to_bin(["Array(", format_type(ElementsType), ")"]); +format_type(#{kind := union, members := MemberTypes} = U) -> + DN = maps:get(display_name, U, undefined), + case DN of + undefined -> + to_bin(["OneOf(", format_union_members(MemberTypes), ")"]); + Name -> + format_primitive_type(Name) + end; +format_type(#{kind := struct, name := FullName}) -> + to_bin(["Struct(", short_name(FullName), ")"]); +format_type(#{kind := map, name := Name, values := ValuesType}) -> + to_bin(["Map($", Name, "->", format_type(ValuesType), ")"]). + +format_union_members(Members) -> + format_union_members(Members, []). + +format_union_members([], Acc) -> + lists:join(",", lists:reverse(Acc)); +format_union_members([Member | Members], Acc) -> + NewAcc = [format_type(Member) | Acc], + format_union_members(Members, NewAcc). + +format_primitive_type(TypeStr) -> + Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr), + to_bin(maps:get(type, Spec)). + +%% All types should have a namespace to avlid name clashing. +is_missing_namespace(ShortName, FullName, RootNames) -> + case lists:member(ShortName, RootNames) of + true -> + false; + false -> + ShortName =:= FullName + end. + +%% Returns short name from full name, fullname delemited by colon(:). +short_name(FullName) -> + case string:split(FullName, ":") of + [_, Name] -> to_bin(Name); + _ -> to_bin(FullName) + end. + +%% Returns the hash-anchor from full name, fullname delemited by colon(:). +format_hash(FullName) -> + case string:split(FullName, ":") of + [Namespace, Name] -> + ok = warn_bad_namespace(Namespace), + iolist_to_binary([Namespace, "__", Name]); + _ -> + iolist_to_binary(FullName) + end. + +%% namespace should only have letters, numbers, and underscores. +warn_bad_namespace(Namespace) -> + case re:run(Namespace, "^[a-zA-Z0-9_]+$", [{capture, none}]) of + nomatch -> + case erlang:get({bad_namespace, Namespace}) of + true -> + ok; + _ -> + erlang:put({bad_namespace, Namespace}, true), + io:format(standard_error, "WARN: bad_namespace: ~s~n", [Namespace]) + end; + _ -> + ok + end. %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. hotconf_schema_json() -> @@ -306,12 +560,7 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> typename_to_spec(TypeStr, Module) -> emqx_conf_schema_types:readable_dashboard(Module, TypeStr). -to_bin(List) when is_list(List) -> - case io_lib:printable_list(List) of - true -> unicode:characters_to_binary(List); - false -> List - end; +to_bin(List) when is_list(List) -> iolist_to_binary(List); to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -to_bin(X) -> - X. +to_bin(X) -> X. diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index be24112e4..abd862613 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index e6c922c1e..0a6e5d298 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -28,7 +28,7 @@ fields/1 ]). -namespace() -> "authn-psk". +namespace() -> "psk". roots() -> ["psk_authentication"]. From b643741920fcad8eb3a87ec9542950c8cc12d43c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Nov 2023 20:30:21 +0100 Subject: [PATCH 3/9] feat: add a escript to help re-format older version schema dumps --- scripts/schema-dump-reformat.escript | 132 +++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 scripts/schema-dump-reformat.escript diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript new file mode 100755 index 000000000..31cfdd7d9 --- /dev/null +++ b/scripts/schema-dump-reformat.escript @@ -0,0 +1,132 @@ +#!/usr/bin/env escript + +%% This script translates the hocon_schema_json's schema dump to a new format. +%% It is used to convert older version EMQX's schema dumps to the new format +%% after all files are upgraded to the new format, this script can be removed. + +-mode(compile). + +main([Input]) -> + ok = add_libs(), + _ = atoms(), + {ok, Data} = file:read_file(Input), + Json = jsx:decode(Data), + NewJson = reformat(Json), + io:format("~s~n", [jsx:encode(NewJson)]); +main(_) -> + io:format("Usage: schema-dump-reformat.escript ~n"), + halt(1). + +reformat(Json) -> + emqx_conf:reformat_schema_dump(fix(Json)). + +%% fix old type specs to make them compatible with new type specs +fix(#{ + <<"kind">> := <<"union">>, + <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}] +}) -> + %% s3_exporter.secret_access_key + #{ + kind => primitive, + name => <<"string()">> + }; +fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) -> + #{ + kind => enum, + symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all] + }; +fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) -> + #{kind => enum, symbols => [random, hash]}; +fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) -> + #{kind => enum, symbols => [random, hash]}; +fix(Map) when is_map(Map) -> + maps:from_list(fix(maps:to_list(Map))); +fix(List) when is_list(List) -> + lists:map(fun fix/1, List); +fix({<<"kind">>, Kind}) -> + {kind, binary_to_atom(Kind, utf8)}; +fix({<<"name">>, Type}) -> + {name, fix_type(Type)}; +fix({K, V}) -> + {binary_to_atom(K, utf8), fix(V)}; +fix(V) when is_number(V) -> + V; +fix(V) when is_atom(V) -> + V; +fix(V) when is_binary(V) -> + V. + +%% ensure below ebin dirs are added to code path: +%% _build/default/lib/*/ebin +%% _build/emqx/lib/*/ebin +%% _build/emqx-enterprise/lib/*/ebin +add_libs() -> + Profile = os:getenv("PROFILE"), + case Profile of + "emqx" -> + ok; + "emqx-enterprise" -> + ok; + _ -> + io:format("PROFILE is not set~n"), + halt(1) + end, + Dirs = + filelib:wildcard("_build/default/lib/*/ebin") ++ + filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"), + lists:foreach(fun add_lib/1, Dirs). + +add_lib(Dir) -> + code:add_patha(Dir), + Beams = filelib:wildcard(Dir ++ "/*.beam"), + _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end), + ok. + +load_beam(BeamFile) -> + ModuleName = filename:basename(BeamFile, ".beam"), + Module = list_to_atom(ModuleName), + %% load the beams to make sure the atoms are existing + code:ensure_loaded(Module), + ok. + +fix_type(<<"[{string(), string()}]">>) -> + <<"map()">>; +fix_type(<<"[{binary(), binary()}]">>) -> + <<"map()">>; +fix_type(<<"emqx_limiter_schema:rate()">>) -> + <<"string()">>; +fix_type(<<"emqx_limiter_schema:burst_rate()">>) -> + <<"string()">>; +fix_type(<<"emqx_limiter_schema:capacity()">>) -> + <<"string()">>; +fix_type(<<"emqx_limiter_schema:initial()">>) -> + <<"string()">>; +fix_type(<<"emqx_limiter_schema:failure_strategy()">>) -> + <<"string()">>; +fix_type(<<"emqx_conf_schema:file()">>) -> + <<"string()">>; +fix_type(<<"#{term() => binary()}">>) -> + <<"map()">>; +fix_type(<<"[term()]">>) -> + %% jwt claims + <<"map()">>; +fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) -> + <<"string()">>; +fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) -> + <<"string()">>; +fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) -> + <<"non_neg_integer()">>; +fix_type(<<"emqx_s3_schema:secret_access_key()">>) -> + <<"string()">>; +fix_type(Type) -> + Type. + +%% ensure atoms are loaded +%% these atoms are from older version of emqx +atoms() -> + [ + emqx_ee_connector_clickhouse, + emqx_ee_bridge_gcp_pubsub, + emqx_ee_bridge_influxdb, + emqx_connector_http + ]. From fc849f0c05325c8d289dc53aed128efad248e93e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Nov 2023 12:36:10 -0300 Subject: [PATCH 4/9] ci(test): add info to help diagnose flaky test --- .../test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index be6a306e0..5e1e885db 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -1494,10 +1494,11 @@ t_pull_worker_death(Config) -> ok. t_async_worker_death_mid_pull(Config) -> - ct:timetrap({seconds, 120}), + ct:timetrap({seconds, 122}), [#{pubsub_topic := PubSubTopic}] = ?config(topic_mapping, Config), Payload = emqx_guid:to_hexstr(emqx_guid:gen()), ?check_trace( + #{timetrap => 120_000}, begin start_and_subscribe_mqtt(Config), @@ -1513,18 +1514,22 @@ t_async_worker_death_mid_pull(Config) -> #{?snk_kind := gcp_pubsub_consumer_worker_reply_delegator} ), spawn_link(fun() -> + ct:pal("will kill async worker"), ?tp_span( kill_async_worker, #{}, begin %% produce a message while worker is being killed Messages = [#{<<"data">> => Payload}], + ct:pal("publishing message"), pubsub_publish(Config, PubSubTopic, Messages), + ct:pal("published message"), AsyncWorkerPids = get_async_worker_pids(Config), emqx_utils:pmap( fun(AsyncWorkerPid) -> Ref = monitor(process, AsyncWorkerPid), + ct:pal("killing pid ~p", [AsyncWorkerPid]), sys:terminate(AsyncWorkerPid, die), receive {'DOWN', Ref, process, AsyncWorkerPid, _} -> @@ -1538,7 +1543,8 @@ t_async_worker_death_mid_pull(Config) -> ok end - ) + ), + ct:pal("killed async worker") end), ?assertMatch( From d9f964a44f9bc1e3116584e84d894c83fc242ebc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Nov 2023 16:58:05 +0100 Subject: [PATCH 5/9] test: fix test cases after schema type namespace change --- apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl | 2 +- apps/emqx_retainer/test/emqx_retainer_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl index 23532b4af..f2688fff9 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl @@ -54,7 +54,7 @@ t_check_schema(_Config) -> ?assertThrow( #{ path := "authentication.1.password_hash_algorithm.name", - matched_type := "authn:builtin_db/authn-hash:simple", + matched_type := "authn:builtin_db/authn_hash:simple", reason := unable_to_convert_to_enum_symbol }, Check(ConfigNotOk) diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index d75e2ca07..595f37fff 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -555,7 +555,7 @@ t_page_read(_) -> ok = emqtt:disconnect(C1). t_only_for_coverage(_) -> - ?assertEqual("retainer", emqx_retainer_schema:namespace()), + ?assertEqual(retainer, emqx_retainer_schema:namespace()), ignored = gen_server:call(emqx_retainer, unexpected), ok = gen_server:cast(emqx_retainer, unexpected), unexpected = erlang:send(erlang:whereis(emqx_retainer), unexpected), From db83457d13a50c62f1b2465caa8c57ff4dff75a4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Nov 2023 16:02:23 -0300 Subject: [PATCH 6/9] test: fix flaky test The cause was that the call `sys:terminate/2` was timing out... `exit/2` doens't always work: ``` 2023-11-22 19:14:40.974 killed async workers Error: -22T19:14:40.974563+00:00 [error] crasher: initial call: gun:proc_lib_hack/5, pid: <0.15908.7>, registered_name: [], exit: {{{owner_gone,killed},[{gun,owner_gone,1,[{file,"gun.erl"},{line,970}]},{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,649}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]},[{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,654}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]}, ancestors: [gun_sup,<0.15387.7>], message_queue_len: 0, messages: [], links: [<0.15388.7>], dictionary: [], trap_exit: false, status: running, heap_size: 987, stack_size: 28, reductions: 1822; neighbours: Error: -22T19:14:40.998051+00:00 [error] Supervisor: {local,gun_sup}. Context: child_terminated. Reason: {{owner_gone,killed},[{gun,owner_gone,1,[{file,"gun.erl"},{line,970}]},{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,649}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]}. Offender: id=gun,pid=<0.15908.7>. 2023-11-22T19:15:41.088752+00:00 [critical] Run stage failed: error:{badmatch,{timeout,#{expected_remaining => 1,mailbox => {messages,[]},msgs_so_far => []}}}, Stacktrace: [{emqx_bridge_gcp_pubsub_consumer_SUITE,'-t_async_worker_death_mid_pull/1-fun-17-',3,[{file,"/emqx/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl"},{line,1576}]},{emqx_bridge_gcp_pubsub_consumer_SUITE,t_async_worker_death_mid_pull,1,[{file,"/emqx/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl"},{line,1505}]}], Trace dump: "/emqx/_build/test/logs/ct_run.test@127.0.0.1.2023-11-22_19.14.27/snabbkaffe/1700680540975786370.log", mfa: undefined Error: -22T19:15:46.095702+00:00 [error] crasher: initial call: gun:proc_lib_hack/5, pid: <0.15934.7>, registered_name: [], exit: {{{owner_gone,killed},[{gun,owner_gone,1,[{file,"gun.erl"},{line,970}]},{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,649}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]},[{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,654}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]}, ancestors: [gun_sup,<0.15387.7>], message_queue_len: 0, messages: [], links: [<0.15388.7>], dictionary: [], trap_exit: false, status: running, heap_size: 610, stack_size: 28, reductions: 1471; neighbours: Error: -22T19:15:46.095192+00:00 [error] Supervisor: {local,ehttpc_sup}. Context: shutdown_error. Reason: killed. Offender: id={ehttpc_pool_sup,<<98,114,105,100,103,101,58,103,99,112,95,112,117,98,115,117,98,95,99,111,110,115,117,109,101,114,58,116,95,97,115,121,110,99,95,119,111,114,107,101,114,95,100,101,97,116,104,95,109,105,100,95,112,117,108,108,45,53,55,54,52,54,48,55,53,50,51,48,51,52,50,50,55,53,49>>},pid=<0.15903.7>. Error: -22T19:15:46.095470+00:00 [error] Supervisor: {<0.15906.7>,ehttpc_worker_sup}. Context: shutdown_error. Reason: killed. Offender: id={worker,1},pid=<0.15924.7>. Error: -22T19:15:46.096762+00:00 [error] Supervisor: {local,gun_sup}. Context: child_terminated. Reason: {{owner_gone,killed},[{gun,owner_gone,1,[{file,"gun.erl"},{line,970}]},{gun,proc_lib_hack,5,[{file,"gun.erl"},{line,649}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]}. Offender: id=gun,pid=<0.15934.7>. Warning: 2T19:15:46.098278+00:00 [warning] msg: remove_local_resource_failed, mfa: emqx_resource:remove_local/1(362), error: {error,timeout}, resource_id: <<"bridge:gcp_pubsub_consumer:t_async_worker_death_mid_pull-576460752303422751">> Error: -22T19:15:46.149090+00:00 [error] Generic server <0.15904.7> terminating. Reason: killed. Last message: {'EXIT',<0.15903.7>,killed}. State: {state,<<"bridge:gcp_pubsub_consumer:t_async_worker_death_mid_pull-576460752303422751">>,1,random}. Error: -22T19:15:46.149525+00:00 [error] crasher: initial call: ehttpc_pool:init/1, pid: <0.15904.7>, registered_name: [], exit: {killed,[{gen_server,decode_msg,9,[{file,"gen_server.erl"},{line,909}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,240}]}]}, ancestors: [<0.15903.7>,ehttpc_sup,<0.15731.7>], message_queue_len: 0, messages: [], links: [], dictionary: [], trap_exit: true, status: running, heap_size: 376, stack_size: 28, reductions: 3428; neighbours: ``` --- .../emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 5e1e885db..b0e4e4ac8 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -34,16 +34,22 @@ init_per_suite(Config) -> emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), case emqx_common_test_helpers:is_tcp_server_available(GCPEmulatorHost, GCPEmulatorPort) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([ - emqx_resource, emqx_bridge, emqx_rule_engine - ]), - {ok, _} = application:ensure_all_started(emqx_connector), + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_bridge_gcp_pubsub, + emqx_bridge, + emqx_rule_engine + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), emqx_mgmt_api_test_util:init_suite(), HostPort = GCPEmulatorHost ++ ":" ++ GCPEmulatorPortStr, true = os:putenv("PUBSUB_EMULATOR_HOST", HostPort), Client = start_control_client(), [ + {apps, Apps}, {proxy_name, ProxyName}, {proxy_host, ProxyHost}, {proxy_port, ProxyPort}, @@ -62,12 +68,11 @@ init_per_suite(Config) -> end. end_per_suite(Config) -> + Apps = ?config(apps, Config), Client = ?config(client, Config), stop_control_client(Client), emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), - _ = application:stop(emqx_connector), + emqx_cth_suite:stop(Apps), os:unsetenv("PUBSUB_EMULATOR_HOST"), ok. @@ -1514,7 +1519,7 @@ t_async_worker_death_mid_pull(Config) -> #{?snk_kind := gcp_pubsub_consumer_worker_reply_delegator} ), spawn_link(fun() -> - ct:pal("will kill async worker"), + ct:pal("will kill async workers"), ?tp_span( kill_async_worker, #{}, @@ -1530,11 +1535,12 @@ t_async_worker_death_mid_pull(Config) -> fun(AsyncWorkerPid) -> Ref = monitor(process, AsyncWorkerPid), ct:pal("killing pid ~p", [AsyncWorkerPid]), - sys:terminate(AsyncWorkerPid, die), + sys:terminate(AsyncWorkerPid, die, 20_000), receive {'DOWN', Ref, process, AsyncWorkerPid, _} -> + ct:pal("killed pid ~p", [AsyncWorkerPid]), ok - after 500 -> ct:fail("async worker didn't die") + after 500 -> ct:fail("async worker ~p didn't die", [AsyncWorkerPid]) end, ok end, @@ -1544,7 +1550,7 @@ t_async_worker_death_mid_pull(Config) -> ok end ), - ct:pal("killed async worker") + ct:pal("killed async workers") end), ?assertMatch( From c89ec0b1f7a9897170bef95dfb998f6cadf4394b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 23 Nov 2023 10:25:36 -0300 Subject: [PATCH 7/9] fix(bridge_api): don't mangle configs, use correct type as argument Fixes https://emqx.atlassian.net/browse/EMQX-11412 - The wrong type was being used in a list lookup function, resulting in the automatic transformation being called erroneously and mangling the config. - There was a left-over workaround still around which could still mangle the config. --- apps/emqx_bridge/src/emqx_bridge.erl | 1 - apps/emqx_bridge/src/emqx_bridge_api.erl | 18 +---------- apps/emqx_bridge/src/emqx_bridge_v2.erl | 30 +++++++++---------- apps/emqx_bridge/test/emqx_bridge_testlib.erl | 16 ++++++++++ .../test/emqx_bridge_v2_testlib.erl | 1 + .../emqx_bridge_azure_event_hub_v2_SUITE.erl | 9 ++++++ .../emqx_bridge_v2_kafka_producer_SUITE.erl | 14 +++++++++ 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 64bec3a4e..569c1e75a 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -313,7 +313,6 @@ list() -> BridgeV2Bridges = emqx_bridge_v2:bridge_v1_list_and_transform(), BridgeV1Bridges ++ BridgeV2Bridges. -%%BridgeV2Bridges = emqx_bridge_v2:list(). lookup(Id) -> {Type, Name} = emqx_bridge_resource:parse_bridge_id(Id), diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index d263817bf..188f26ab5 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -900,7 +900,7 @@ format_resource( case emqx_bridge_v2:is_bridge_v2_type(Type) of true -> %% The defaults are already filled in - downgrade_raw_conf(Type, RawConf); + RawConf; false -> fill_defaults(Type, RawConf) end, @@ -1164,19 +1164,3 @@ upgrade_type(Type) -> downgrade_type(Type) -> emqx_bridge_lib:downgrade_type(Type). - -%% TODO: move it to callback -downgrade_raw_conf(kafka_producer, RawConf) -> - rename(<<"parameters">>, <<"kafka">>, RawConf); -downgrade_raw_conf(azure_event_hub_producer, RawConf) -> - rename(<<"parameters">>, <<"kafka">>, RawConf); -downgrade_raw_conf(_Type, RawConf) -> - RawConf. - -rename(OldKey, NewKey, Map) -> - case maps:find(OldKey, Map) of - {ok, Value} -> - maps:remove(OldKey, maps:put(NewKey, Value, Map)); - error -> - Map - end. diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 706849965..d9ca1acce 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1063,17 +1063,17 @@ bridge_v1_list_and_transform() -> Bridges = list_with_lookup_fun(fun bridge_v1_lookup_and_transform/2), [B || B <- Bridges, B =/= not_bridge_v1_compatible_error()]. -bridge_v1_lookup_and_transform(BridgeV1Type, Name) -> +bridge_v1_lookup_and_transform(ActionType, Name) -> + BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType), case ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of true -> - Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), - case lookup(Type, Name) of + case lookup(ActionType, Name) of {ok, #{raw_config := #{<<"connector">> := ConnectorName}} = BridgeV2} -> - ConnectorType = connector_type(Type), + ConnectorType = connector_type(ActionType), case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( - BridgeV1Type, Name, Type, BridgeV2, ConnectorType, Connector + BridgeV1Type, Name, ActionType, BridgeV2, ConnectorType, Connector ); Error -> Error @@ -1089,7 +1089,7 @@ not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. bridge_v1_lookup_and_transform_helper( - BridgeV1Type, BridgeName, BridgeV2Type, BridgeV2, ConnectorType, Connector + BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> ConnectorRawConfig1 = maps:get(raw_config, Connector), ConnectorRawConfig2 = fill_defaults( @@ -1098,10 +1098,10 @@ bridge_v1_lookup_and_transform_helper( <<"connectors">>, emqx_connector_schema ), - BridgeV2RawConfig1 = maps:get(raw_config, BridgeV2), - BridgeV2RawConfig2 = fill_defaults( - BridgeV2Type, - BridgeV2RawConfig1, + ActionRawConfig1 = maps:get(raw_config, Action), + ActionRawConfig2 = fill_defaults( + ActionType, + ActionRawConfig1, <<"actions">>, emqx_bridge_v2_schema ), @@ -1110,7 +1110,7 @@ bridge_v1_lookup_and_transform_helper( emqx_action_info:has_custom_connector_action_config_to_bridge_v1_config(BridgeV1Type) of false -> - BridgeV1Config1 = maps:remove(<<"connector">>, BridgeV2RawConfig2), + BridgeV1Config1 = maps:remove(<<"connector">>, ActionRawConfig2), %% Move parameters to the top level ParametersMap = maps:get(<<"parameters">>, BridgeV1Config1, #{}), BridgeV1Config2 = maps:remove(<<"parameters">>, BridgeV1Config1), @@ -1118,13 +1118,13 @@ bridge_v1_lookup_and_transform_helper( emqx_utils_maps:deep_merge(ConnectorRawConfig2, BridgeV1Config3); true -> emqx_action_info:connector_action_config_to_bridge_v1_config( - BridgeV1Type, ConnectorRawConfig2, BridgeV2RawConfig2 + BridgeV1Type, ConnectorRawConfig2, ActionRawConfig2 ) end, - BridgeV1Tmp = maps:put(raw_config, BridgeV1ConfigFinal, BridgeV2), + BridgeV1Tmp = maps:put(raw_config, BridgeV1ConfigFinal, Action), BridgeV1 = maps:remove(status, BridgeV1Tmp), - BridgeV2Status = maps:get(status, BridgeV2, undefined), - BridgeV2Error = maps:get(error, BridgeV2, undefined), + BridgeV2Status = maps:get(status, Action, undefined), + BridgeV2Error = maps:get(error, Action, undefined), ResourceData1 = maps:get(resource_data, BridgeV1, #{}), %% Replace id in resouce data BridgeV1Id = <<"bridge:", (bin(BridgeV1Type))/binary, ":", (bin(BridgeName))/binary>>, diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index df404d9b0..f486e5d64 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -120,6 +120,22 @@ create_bridge(Config, Overrides) -> ct:pal("creating bridge with config: ~p", [BridgeConfig]), emqx_bridge:create(BridgeType, BridgeName, BridgeConfig). +list_bridges_api() -> + Params = [], + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("listing bridges (via http)"), + Res = + case emqx_mgmt_api_test_util:request_api(get, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; + Error -> + Error + end, + ct:pal("list bridge result: ~p", [Res]), + Res. + create_bridge_api(Config) -> create_bridge_api(Config, _Overrides = #{}). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 6c48f5663..5cb9b043f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -139,6 +139,7 @@ create_bridge(Config, Overrides) -> ConnectorName = ?config(connector_name, Config), ConnectorType = ?config(connector_type, Config), ConnectorConfig = ?config(connector_config, Config), + ct:pal("creating connector with config: ~p", [ConnectorConfig]), {ok, _} = emqx_connector:create(ConnectorType, ConnectorName, ConnectorConfig), diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl index 4d441ea0b..9661004d0 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl @@ -368,3 +368,12 @@ t_parameters_key_api_spec(_Config) -> ?assert(is_map_key(<<"parameters">>, ActionProps), #{action_props => ActionProps}), ok. + +t_http_api_get(Config) -> + ?assertMatch({ok, _}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% v1 api; no mangling of configs; has `kafka' top level config key + ?assertMatch( + {ok, {{_, 200, _}, _, [#{<<"kafka">> := _}]}}, + emqx_bridge_testlib:list_bridges_api() + ), + ok. diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 8ce3b7f6b..2ad0504b4 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -369,3 +369,17 @@ t_parameters_key_api_spec(_Config) -> ?assert(is_map_key(<<"parameters">>, ActionProps), #{action_props => ActionProps}), ok. + +t_http_api_get(_Config) -> + ConnectorName = <<"test_connector">>, + ActionName = <<"test_action">>, + ActionConfig = bridge_v2_config(<<"test_connector">>), + ConnectorConfig = connector_config(), + ?assertMatch({ok, _}, create_connector(ConnectorName, ConnectorConfig)), + ?assertMatch({ok, _}, create_action(ActionName, ActionConfig)), + %% v1 api; no mangling of configs; has `kafka' top level config key + ?assertMatch( + {ok, {{_, 200, _}, _, [#{<<"kafka">> := _}]}}, + emqx_bridge_testlib:list_bridges_api() + ), + ok. From 8ba116d3782205494536ec529ef52a522bcfa67b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 23 Nov 2023 14:54:55 +0100 Subject: [PATCH 8/9] fix(emqx_auth): check authenticator exists in /authenticator/:id/users --- apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl | 15 +++++++++++---- .../test/emqx_authn/emqx_authn_api_SUITE.erl | 13 +++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl index f30f7f473..1b299fa64 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl @@ -1111,10 +1111,7 @@ list_users(ChainName, AuthenticatorID, QueryString) -> {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; {error, Reason} -> - {400, #{ - code => <<"INVALID_PARAMETER">>, - message => list_to_binary(io_lib:format("Reason ~p", [Reason])) - }}; + serialize_error({user_error, Reason}); Result -> {200, Result} end. @@ -1176,6 +1173,16 @@ serialize_error({user_error, not_found}) -> code => <<"NOT_FOUND">>, message => binfmt("User not found", []) }}; +serialize_error({user_error, {not_found, {chain, ?GLOBAL}}}) -> + {404, #{ + code => <<"NOT_FOUND">>, + message => <<"Authenticator not found in the 'global' scope">> + }}; +serialize_error({user_error, {not_found, {chain, Name}}}) -> + {400, #{ + code => <<"BAD_REQUEST">>, + message => binfmt("No authentication has been created for listener ~p", [Name]) + }}; serialize_error({user_error, already_exist}) -> {409, #{ code => <<"ALREADY_EXISTS">>, diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl index 45a605e6e..cceab0d54 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl @@ -435,6 +435,19 @@ test_authenticator_position(PathPrefix) -> PathPrefix ++ [?CONF_NS] ). +t_authenticator_users_not_found(_) -> + GlobalUser = #{user_id => <<"global_user">>, password => <<"p1">>}, + {ok, 404, _} = request( + get, + uri([?CONF_NS, "password_based:built_in_database", "users"]) + ), + {ok, 404, _} = request( + post, + uri([?CONF_NS, "password_based:built_in_database", "users"]), + GlobalUser + ), + ok. + %% listener authn api is not supported since 5.1.0 %% Don't support listener switch to global chain. ignore_switch_to_global_chain(_) -> From e95ec5b15052d17262a3b122e097b1fcdd07f51f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Nov 2023 09:24:21 -0300 Subject: [PATCH 9/9] test: fix another flaky test --- .../test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index b0e4e4ac8..d82a61fee 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -1477,7 +1477,7 @@ t_pull_worker_death(Config) -> [PullWorkerPid | _] = get_pull_worker_pids(Config), Ref = monitor(process, PullWorkerPid), - sys:terminate(PullWorkerPid, die), + sys:terminate(PullWorkerPid, die, 20_000), receive {'DOWN', Ref, process, PullWorkerPid, _} -> ok