refactor(schema_registry): refactor schema registry app and modules

Fixes https://emqx.atlassian.net/browse/EMQX-10361

- Moves `lib-ee/emqx_ee_schema_registry` to `apps/emqx_schema_registry`.
- Removes the `_ee_` segment from module names.
  - Exceptions are the table names which are kept to avoid backwards incompatibilities.
This commit is contained in:
Thales Macedo Garitezi 2023-07-10 11:31:47 -03:00
parent cacfb51280
commit 541d03a0ba
26 changed files with 232 additions and 125 deletions

View File

@ -10,7 +10,7 @@
-define(EE_SCHEMA_MODULES, [
emqx_license_schema,
emqx_ee_schema_registry_schema,
emqx_schema_registry_schema,
emqx_ft_schema
]).

View File

@ -158,7 +158,7 @@ basic_reboot_apps_edition(ee) ->
emqx_ft,
emqx_eviction_agent,
emqx_node_rebalance,
emqx_ee_schema_registry
emqx_schema_registry
];
%% unexcepted edition, should not happen
basic_reboot_apps_edition(_) ->

View File

@ -1148,7 +1148,7 @@ timezone_to_offset_seconds(TimeZone) ->
'$handle_undefined_function'(sprintf, [Format | Args]) ->
erlang:apply(fun sprintf_s/2, [Format, Args]);
%% This is for functions that should be handled in another module
%% (currently this module is emqx_ee_schema_registry_serde in the case of EE but
%% (currently this module is emqx_schema_registry_serde in the case of EE but
%% could be changed to another module in the future).
'$handle_undefined_function'(FunctionName, Args) ->
case emqx_rule_engine:extra_functions_module() of

View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2023
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2027-02-01
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this Licenses text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this Licenses text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

View File

@ -71,3 +71,18 @@ WHERE
- Register schema instance: adds a new instance of a schema of a
certain type. For example, when the user may have several Avro or
Protobuf schemas that they wish to use with different data flows.
# Documentation
- Refer to [Introduction to Schema Registry](https://docs.emqx.com/en/enterprise/v5.1/data-integration/schema-registry.html) for how to use the EMQX dashboard for configuring schemas.
- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html)
for the EMQX rules engine introduction.
# Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).
# License
EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt).

View File

@ -2,12 +2,13 @@
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-ifndef(EMQX_EE_SCHEMA_REGISTRY_HRL).
-define(EMQX_EE_SCHEMA_REGISTRY_HRL, true).
-ifndef(EMQX_SCHEMA_REGISTRY_HRL).
-define(EMQX_SCHEMA_REGISTRY_HRL, true).
-define(CONF_KEY_ROOT, schema_registry).
-define(CONF_KEY_PATH, [?CONF_KEY_ROOT]).
%% Note: this has the `_ee_' segment for backwards compatibility.
-define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard).
-define(SERDE_TAB, emqx_ee_schema_registry_serde_tab).
-define(PROTOBUF_CACHE_TAB, emqx_ee_schema_registry_protobuf_cache_tab).

View File

@ -2,14 +2,14 @@
{erl_opts, [debug_info]}.
{deps, [
{emqx, {path, "../../apps/emqx"}},
{emqx_utils, {path, "../../apps/emqx_utils"}},
{emqx_rule_engine, {path, "../../apps/emqx_rule_engine"}},
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}},
{emqx_rule_engine, {path, "../emqx_rule_engine"}},
{erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}},
{gpb, "4.19.7"}
]}.
{shell, [
% {config, "config/sys.config"},
{apps, [emqx_ee_schema_registry]}
{apps, [emqx_schema_registry]}
]}.

View File

@ -1,8 +1,8 @@
{application, emqx_ee_schema_registry, [
{application, emqx_schema_registry, [
{description, "EMQX Schema Registry"},
{vsn, "0.1.5"},
{registered, [emqx_ee_schema_registry_sup]},
{mod, {emqx_ee_schema_registry_app, []}},
{registered, [emqx_schema_registry_sup]},
{mod, {emqx_schema_registry_app, []}},
{included_applications, [
emqx_rule_engine
]},

View File

@ -1,13 +1,13 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry).
-module(emqx_schema_registry).
-behaviour(gen_server).
-behaviour(emqx_config_handler).
-behaviour(emqx_config_backup).
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -291,7 +291,7 @@ do_build_serde(Name, Serde) when not is_binary(Name) ->
do_build_serde(Name, #{type := Type, source := Source}) ->
try
{Serializer, Deserializer, Destructor} =
emqx_ee_schema_registry_serde:make_serde(Type, Name, Source),
emqx_schema_registry_serde:make_serde(Type, Name, Source),
Serde = #serde{
name = Name,
serializer = Serializer,

View File

@ -1,24 +1,24 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_app).
-module(emqx_schema_registry_app).
-behaviour(application).
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
%% Register rule engine extra functions module so that we can handle decode
%% and encode functions called from the rule engine SQL like language
ok = emqx_rule_engine:set_extra_functions_module(emqx_ee_schema_registry_serde),
ok = emqx_rule_engine:set_extra_functions_module(emqx_schema_registry_serde),
ok = mria_rlog:wait_for_shards([?SCHEMA_REGISTRY_SHARD], infinity),
%% HTTP API handler
emqx_conf:add_handler([?CONF_KEY_ROOT, schemas, '?'], emqx_ee_schema_registry),
emqx_conf:add_handler([?CONF_KEY_ROOT, schemas, '?'], emqx_schema_registry),
%% Conf load / data import handler
emqx_conf:add_handler(?CONF_KEY_PATH, emqx_ee_schema_registry),
emqx_ee_schema_registry_sup:start_link().
emqx_conf:add_handler(?CONF_KEY_PATH, emqx_schema_registry),
emqx_schema_registry_sup:start_link().
stop(_State) ->
emqx_conf:remove_handler([?CONF_KEY_ROOT, schemas, '?']),

View File

@ -1,11 +1,11 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_http_api).
-module(emqx_schema_registry_http_api).
-behaviour(minirest_api).
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_utils/include/emqx_utils_api.hrl").
@ -48,7 +48,7 @@ schema("/schema_registry") ->
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
hoconsc:array(emqx_ee_schema_registry_schema:api_schema("get")),
hoconsc:array(emqx_schema_registry_schema:api_schema("get")),
#{
sample =>
#{value => sample_list_schemas_response()}
@ -61,14 +61,14 @@ schema("/schema_registry") ->
summary => <<"Register a new schema">>,
description => ?DESC("desc_schema_registry_api_post"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_ee_schema_registry_schema:api_schema("post"),
emqx_schema_registry_schema:api_schema("post"),
post_examples()
),
responses =>
#{
201 =>
emqx_dashboard_swagger:schema_with_examples(
emqx_ee_schema_registry_schema:api_schema("post"),
emqx_schema_registry_schema:api_schema("post"),
post_examples()
),
400 => error_schema('ALREADY_EXISTS', "Schema already exists")
@ -87,7 +87,7 @@ schema("/schema_registry/:name") ->
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
emqx_ee_schema_registry_schema:api_schema("get"),
emqx_schema_registry_schema:api_schema("get"),
get_examples()
),
404 => error_schema('NOT_FOUND', "Schema not found")
@ -99,14 +99,14 @@ schema("/schema_registry/:name") ->
description => ?DESC("desc_schema_registry_api_put"),
parameters => [param_path_schema_name()],
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_ee_schema_registry_schema:api_schema("put"),
emqx_schema_registry_schema:api_schema("put"),
post_examples()
),
responses =>
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
emqx_ee_schema_registry_schema:api_schema("put"),
emqx_schema_registry_schema:api_schema("put"),
put_examples()
),
404 => error_schema('NOT_FOUND', "Schema not found")
@ -130,7 +130,7 @@ schema("/schema_registry/:name") ->
%%-------------------------------------------------------------------------------------------------
'/schema_registry'(get, _Params) ->
Schemas = emqx_ee_schema_registry:list_schemas(),
Schemas = emqx_schema_registry:list_schemas(),
Response =
lists:map(
fun({Name, Params}) ->
@ -141,11 +141,11 @@ schema("/schema_registry/:name") ->
?OK(Response);
'/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) ->
Params = maps:without([<<"name">>], Params0),
case emqx_ee_schema_registry:get_schema(Name) of
case emqx_schema_registry:get_schema(Name) of
{error, not_found} ->
case emqx_ee_schema_registry:add_schema(Name, Params) of
case emqx_schema_registry:add_schema(Name, Params) of
ok ->
{ok, Res} = emqx_ee_schema_registry:get_schema(Name),
{ok, Res} = emqx_schema_registry:get_schema(Name),
{201, Res#{name => Name}};
{error, Error} ->
?BAD_REQUEST(Error)
@ -155,31 +155,31 @@ schema("/schema_registry/:name") ->
end.
'/schema_registry/:name'(get, #{bindings := #{name := Name}}) ->
case emqx_ee_schema_registry:get_schema(Name) of
case emqx_schema_registry:get_schema(Name) of
{error, not_found} ->
?NOT_FOUND(<<"Schema not found">>);
{ok, Schema} ->
?OK(Schema#{name => Name})
end;
'/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) ->
case emqx_ee_schema_registry:get_schema(Name) of
case emqx_schema_registry:get_schema(Name) of
{error, not_found} ->
?NOT_FOUND(<<"Schema not found">>);
{ok, _} ->
case emqx_ee_schema_registry:add_schema(Name, Params) of
case emqx_schema_registry:add_schema(Name, Params) of
ok ->
{ok, Res} = emqx_ee_schema_registry:get_schema(Name),
{ok, Res} = emqx_schema_registry:get_schema(Name),
?OK(Res#{name => Name});
{error, Error} ->
?BAD_REQUEST(Error)
end
end;
'/schema_registry/:name'(delete, #{bindings := #{name := Name}}) ->
case emqx_ee_schema_registry:get_schema(Name) of
case emqx_schema_registry:get_schema(Name) of
{error, not_found} ->
?NOT_FOUND(<<"Schema not found">>);
{ok, _} ->
case emqx_ee_schema_registry:delete_schema(Name) of
case emqx_schema_registry:delete_schema(Name) of
ok ->
?NO_CONTENT;
{error, Error} ->

View File

@ -2,11 +2,11 @@
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_schema).
-module(emqx_schema_registry_schema).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
%% `hocon_schema' API
-export([

View File

@ -1,15 +1,15 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_serde).
-module(emqx_schema_registry_serde).
-behaviour(emqx_rule_funcs).
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ee_schema_registry_serde]}}]).
-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_schema_registry_serde]}}]).
%% API
-export([
@ -67,7 +67,7 @@ decode(SerdeName, RawData) ->
-spec decode(schema_name(), encoded_data(), [term()]) -> decoded_data().
decode(SerdeName, RawData, VarArgs) when is_list(VarArgs) ->
case emqx_ee_schema_registry:get_serde(SerdeName) of
case emqx_schema_registry:get_serde(SerdeName) of
{error, not_found} ->
error({serde_not_found, SerdeName});
{ok, #{deserializer := Deserializer}} ->
@ -80,7 +80,7 @@ encode(SerdeName, RawData) ->
-spec encode(schema_name(), decoded_data(), [term()]) -> encoded_data().
encode(SerdeName, EncodedData, VarArgs) when is_list(VarArgs) ->
case emqx_ee_schema_registry:get_serde(SerdeName) of
case emqx_schema_registry:get_serde(SerdeName) of
{error, not_found} ->
error({serde_not_found, SerdeName});
{ok, #{serializer := Serializer}} ->

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_sup).
-module(emqx_schema_registry_sup).
-behaviour(supervisor).
@ -29,7 +29,7 @@ init([]) ->
intensity => 10,
period => 100
},
ChildSpecs = [child_spec(emqx_ee_schema_registry)],
ChildSpecs = [child_spec(emqx_schema_registry)],
{ok, {SupFlags, ChildSpecs}}.
child_spec(Mod) ->
@ -38,6 +38,5 @@ child_spec(Mod) ->
start => {Mod, start_link, []},
restart => permanent,
shutdown => 5_000,
type => worker,
modules => [Mod]
type => worker
}.

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_SUITE).
-module(emqx_schema_registry_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
@ -10,11 +10,11 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-import(emqx_common_test_helpers, [on_exit/1]).
-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]).
-define(APPS, [emqx_conf, emqx_rule_engine, emqx_schema_registry]).
%%------------------------------------------------------------------------------
%% CT boilerplate
@ -46,7 +46,7 @@ sparkplug_tests() ->
].
init_per_suite(Config) ->
emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
emqx_mgmt_api_test_util:init_suite(?APPS),
Config.
@ -156,7 +156,7 @@ schema_params(protobuf) ->
create_serde(SerdeType, SerdeName) ->
Schema = schema_params(SerdeType),
ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema),
ok = emqx_schema_registry:add_schema(SerdeName, Schema),
ok.
test_params_for(avro, encode_decode1) ->
@ -313,9 +313,9 @@ test_params_for(Type, Name) ->
clear_schemas() ->
maps:foreach(
fun(Name, _Schema) ->
ok = emqx_ee_schema_registry:delete_schema(Name)
ok = emqx_schema_registry:delete_schema(Name)
end,
emqx_ee_schema_registry:list_schemas()
emqx_schema_registry:list_schemas()
).
receive_action_results() ->
@ -367,7 +367,7 @@ cluster(Config) ->
%% need to restart schema registry app in the tests so
%% that it re-registers the config handler that is lost
%% when emqx_conf restarts during join.
{env, [{emqx_machine, applications, [emqx_ee_schema_registry]}]},
{env, [{emqx_machine, applications, [emqx_schema_registry]}]},
{load_apps, [emqx_machine | ?APPS]},
{env_handler, fun
(emqx) ->
@ -453,11 +453,11 @@ protobuf_unique_cache_hit_spec(_Res, _Trace) ->
%%------------------------------------------------------------------------------
t_unknown_calls(_Config) ->
Ref = monitor(process, emqx_ee_schema_registry),
Ref = monitor(process, emqx_schema_registry),
%% for coverage
emqx_ee_schema_registry ! unknown,
gen_server:cast(emqx_ee_schema_registry, unknown),
?assertEqual({error, unknown_call}, gen_server:call(emqx_ee_schema_registry, unknown)),
emqx_schema_registry ! unknown,
gen_server:cast(emqx_schema_registry, unknown),
?assertEqual({error, unknown_call}, gen_server:call(emqx_schema_registry, unknown)),
receive
{'DOWN', Ref, process, _, _} ->
ct:fail("registry shouldn't have died")
@ -489,7 +489,7 @@ t_delete_serde(Config) ->
ok = create_serde(SerdeType, SerdeName),
{ok, {ok, _}} =
?wait_async_action(
emqx_ee_schema_registry:delete_schema(SerdeName),
emqx_schema_registry:delete_schema(SerdeName),
#{?snk_kind := schema_registry_serdes_deleted},
1_000
),
@ -522,7 +522,7 @@ t_encode(Config) ->
Published
),
#{payload := Encoded} = Published,
{ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
{ok, #{deserializer := Deserializer}} = emqx_schema_registry:get_serde(SerdeName),
?assertEqual(Payload, apply(Deserializer, [Encoded | ExtraArgs])),
ok.
@ -536,7 +536,7 @@ t_decode(Config) ->
extra_args := ExtraArgs
} = test_params_for(SerdeType, decode1),
{ok, _} = create_rule_http(#{sql => SQL}),
{ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
{ok, #{serializer := Serializer}} = emqx_schema_registry:get_serde(SerdeName),
EncodedBin = apply(Serializer, [Payload | ExtraArgs]),
emqx:publish(emqx_message:make(<<"t">>, EncodedBin)),
Published = receive_published(?LINE),
@ -559,7 +559,7 @@ t_protobuf_union_encode(Config) ->
extra_args := ExtraArgs
} = test_params_for(SerdeType, union1),
{ok, _} = create_rule_http(#{sql => SQL}),
{ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
{ok, #{serializer := Serializer}} = emqx_schema_registry:get_serde(SerdeName),
EncodedBinA = apply(Serializer, [PayloadA | ExtraArgs]),
emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)),
@ -594,7 +594,7 @@ t_protobuf_union_decode(Config) ->
extra_args := ExtraArgs
} = test_params_for(SerdeType, union2),
{ok, _} = create_rule_http(#{sql => SQL}),
{ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
{ok, #{deserializer := Deserializer}} = emqx_schema_registry:get_serde(SerdeName),
EncodedBinA = emqx_utils_json:encode(PayloadA),
emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)),
@ -639,9 +639,9 @@ t_fail_rollback(Config) ->
#{}
)
),
?assertMatch({ok, #{name := <<"a">>}}, emqx_ee_schema_registry:get_serde(<<"a">>)),
?assertMatch({ok, #{name := <<"a">>}}, emqx_schema_registry:get_serde(<<"a">>)),
%% no z serdes should be in the table
?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"z">>)),
?assertEqual({error, not_found}, emqx_schema_registry:get_serde(<<"z">>)),
ok.
t_cluster_serde_build(Config) ->
@ -660,13 +660,13 @@ t_cluster_serde_build(Config) ->
wait_for_cluster_rpc(N2),
?assertEqual(
ok,
erpc:call(N2, emqx_ee_schema_registry, add_schema, [SerdeName, Schema])
erpc:call(N2, emqx_schema_registry, add_schema, [SerdeName, Schema])
),
%% check that we can serialize/deserialize in all nodes
lists:foreach(
fun(N) ->
erpc:call(N, fun() ->
Res0 = emqx_ee_schema_registry:get_serde(SerdeName),
Res0 = emqx_schema_registry:get_serde(SerdeName),
?assertMatch({ok, #{}}, Res0, #{node => N}),
{ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0,
?assertEqual(
@ -691,7 +691,7 @@ t_cluster_serde_build(Config) ->
),
?assertEqual(
ok,
erpc:call(N1, emqx_ee_schema_registry, delete_schema, [SerdeName])
erpc:call(N1, emqx_schema_registry, delete_schema, [SerdeName])
),
{ok, _} = snabbkaffe:receive_events(SRef1),
lists:foreach(
@ -699,7 +699,7 @@ t_cluster_serde_build(Config) ->
erpc:call(N, fun() ->
?assertMatch(
{error, not_found},
emqx_ee_schema_registry:get_serde(SerdeName),
emqx_schema_registry:get_serde(SerdeName),
#{node => N}
),
ok
@ -726,7 +726,15 @@ t_import_config(_Config) ->
#{
<<"description">> => <<"My Avro Schema">>,
<<"source">> =>
<<"{\"type\":\"record\",\"fields\":[{\"type\":\"int\",\"name\":\"i\"},{\"type\":\"string\",\"name\":\"s\"}]}">>,
emqx_utils_json:encode(
#{
type => <<"record">>,
fields => [
#{type => <<"int">>, name => <<"i">>},
#{type => <<"string">>, name => <<"s">>}
]
}
),
<<"type">> => <<"avro">>
}
}
@ -740,15 +748,21 @@ t_import_config(_Config) ->
Path = [schema_registry, schemas, <<"my_avro_schema">>],
?assertEqual(
{ok, #{root_key => schema_registry, changed => []}},
emqx_ee_schema_registry:import_config(RawConf)
emqx_schema_registry:import_config(RawConf)
),
?assertEqual(
{ok, #{root_key => schema_registry, changed => [Path]}},
emqx_ee_schema_registry:import_config(RawConf1)
emqx_schema_registry:import_config(RawConf1)
).
sparkplug_example_data_base64() ->
<<"CPHh67HrMBIqChxjb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xc2VjGPXh67HrMCACUKgDEikKHGNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxXzVzZWMY9eHrseswIAJQVBIqCh1jb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xMHNlYxj14eux6zAgAlAqEigKG2NvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxX3J1bhj14eux6zAgBVABEioKHWNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxX3Jlc2V0GPXh67HrMCAFUAAYWA==">>.
<<
"CPHh67HrMBIqChxjb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xc2VjGPXh67HrMCACUKgD"
"EikKHGNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxXzVzZWMY9eHrseswIAJQVBIqCh1jb3Vu"
"dGVyX2dyb3VwMS9jb3VudGVyMV8xMHNlYxj14eux6zAgAlAqEigKG2NvdW50ZXJfZ3Jv"
"dXAxL2NvdW50ZXIxX3J1bhj14eux6zAgBVABEioKHWNvdW50ZXJfZ3JvdXAxL2NvdW50"
"ZXIxX3Jlc2V0GPXh67HrMCAFUAAYWA=="
>>.
sparkplug_example_data() ->
#{

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_http_api_SUITE).
-module(emqx_schema_registry_http_api_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
@ -12,9 +12,9 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-define(APPS, [emqx_conf, emqx_ee_schema_registry]).
-define(APPS, [emqx_conf, emqx_schema_registry]).
%%------------------------------------------------------------------------------
%% CT boilerplate
@ -34,7 +34,7 @@ groups() ->
].
init_per_suite(Config) ->
emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
emqx_mgmt_api_test_util:init_suite(?APPS),
Config.
@ -138,9 +138,9 @@ try_decode_error_message(Res) ->
clear_schemas() ->
maps:foreach(
fun(Name, _Schema) ->
ok = emqx_ee_schema_registry:delete_schema(Name)
ok = emqx_schema_registry:delete_schema(Name)
end,
emqx_ee_schema_registry:list_schemas()
emqx_schema_registry:list_schemas()
).
%%------------------------------------------------------------------------------
@ -249,7 +249,7 @@ t_crud(Config) ->
{ok, 400, #{
<<"code">> := <<"BAD_REQUEST">>,
<<"message">> :=
<<"{post_config_update,emqx_ee_schema_registry,", _/binary>>
<<"{post_config_update,emqx_schema_registry,", _/binary>>
}},
request({put, SchemaName, UpdateParams#{<<"source">> := InvalidSourceBin}})
),
@ -290,7 +290,7 @@ t_crud(Config) ->
{ok, 400, #{
<<"code">> := <<"BAD_REQUEST">>,
<<"message">> :=
<<"{post_config_update,emqx_ee_schema_registry,", _/binary>>
<<"{post_config_update,emqx_schema_registry,", _/binary>>
}},
request({post, Params#{<<"source">> := InvalidSourceBin}})
),

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_schema_registry_serde_SUITE).
-module(emqx_schema_registry_serde_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
@ -10,11 +10,11 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_ee_schema_registry.hrl").
-include("emqx_schema_registry.hrl").
-import(emqx_common_test_helpers, [on_exit/1]).
-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]).
-define(APPS, [emqx_conf, emqx_rule_engine, emqx_schema_registry]).
%%------------------------------------------------------------------------------
%% CT boilerplate
@ -24,7 +24,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
emqx_mgmt_api_test_util:init_suite(?APPS),
Config.
@ -46,9 +46,9 @@ end_per_testcase(_TestCase, _Config) ->
clear_schemas() ->
maps:foreach(
fun(Name, _Schema) ->
ok = emqx_ee_schema_registry:delete_schema(Name)
ok = emqx_schema_registry:delete_schema(Name)
end,
emqx_ee_schema_registry:list_schemas()
emqx_schema_registry:list_schemas()
).
schema_params(avro) ->
@ -81,13 +81,13 @@ schema_params(protobuf) ->
#{type => protobuf, source => SourceBin}.
assert_roundtrip(SerdeName, Original) ->
Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original),
Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded),
Encoded = emqx_schema_registry_serde:encode(SerdeName, Original),
Decoded = emqx_schema_registry_serde:decode(SerdeName, Encoded),
?assertEqual(Original, Decoded, #{original => Original}).
assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) ->
Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize),
Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize),
Encoded = emqx_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize),
Decoded = emqx_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize),
?assertEqual(Original, Decoded, #{original => Original}).
%%------------------------------------------------------------------------------
@ -97,7 +97,7 @@ assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) ->
t_roundtrip_avro(_Config) ->
SerdeName = my_serde,
Params = schema_params(avro),
ok = emqx_ee_schema_registry:add_schema(SerdeName, Params),
ok = emqx_schema_registry:add_schema(SerdeName, Params),
Original = #{<<"i">> => 10, <<"s">> => <<"hi">>},
%% for coverage
assert_roundtrip(SerdeName, Original, _ArgsSerialize = [], _ArgsDeserialize = []),
@ -110,7 +110,7 @@ t_avro_invalid_json_schema(_Config) ->
WrongParams = Params#{source := <<"{">>},
?assertMatch(
{error, #{reason := #{expected := _}}},
emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
emqx_schema_registry:add_schema(SerdeName, WrongParams)
),
ok.
@ -120,7 +120,7 @@ t_avro_invalid_schema(_Config) ->
WrongParams = Params#{source := <<"{}">>},
?assertMatch(
{error, {post_config_update, _, {not_found, <<"type">>}}},
emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
emqx_schema_registry:add_schema(SerdeName, WrongParams)
),
ok.
@ -130,18 +130,18 @@ t_serde_not_found(_Config) ->
Original = #{},
?assertError(
{serde_not_found, NonexistentSerde},
emqx_ee_schema_registry_serde:encode(NonexistentSerde, Original)
emqx_schema_registry_serde:encode(NonexistentSerde, Original)
),
?assertError(
{serde_not_found, NonexistentSerde},
emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original)
emqx_schema_registry_serde:decode(NonexistentSerde, Original)
),
ok.
t_roundtrip_protobuf(_Config) ->
SerdeName = my_serde,
Params = schema_params(protobuf),
ok = emqx_ee_schema_registry:add_schema(SerdeName, Params),
ok = emqx_schema_registry:add_schema(SerdeName, Params),
ExtraArgsPerson = [<<"Person">>],
Original0 = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>},
@ -167,6 +167,6 @@ t_protobuf_invalid_schema(_Config) ->
WrongParams = Params#{source := <<"xxxx">>},
?assertMatch(
{error, {post_config_update, _, {invalid_protobuf_schema, _}}},
emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
emqx_schema_registry:add_schema(SerdeName, WrongParams)
),
ok.

View File

@ -0,0 +1 @@
Schema Registry has been refactored to its own Erlang application. This allows for more flexibility in the future.

View File

@ -1,19 +0,0 @@
.rebar3
_*
.eunit
*.o
*.beam
*.plt
*.swp
*.swo
.erlang.cookie
ebin
log
erl_crash.dump
.rebar
logs
_build
.idea
*.iml
rebar3.crashdump
*~

View File

@ -190,6 +190,7 @@ defmodule EMQXUmbrella.MixProject do
:emqx_bridge_clickhouse,
:emqx_ft,
:emqx_s3,
:emqx_schema_registry,
:emqx_enterprise
])
end
@ -418,7 +419,7 @@ defmodule EMQXUmbrella.MixProject do
emqx_oracle: :permanent,
emqx_bridge_oracle: :permanent,
emqx_bridge_rabbitmq: :permanent,
emqx_ee_schema_registry: :permanent,
emqx_schema_registry: :permanent,
emqx_eviction_agent: :permanent,
emqx_node_rebalance: :permanent,
emqx_ft: :permanent

View File

@ -102,6 +102,7 @@ is_community_umbrella_app("apps/emqx_oracle") -> false;
is_community_umbrella_app("apps/emqx_bridge_rabbitmq") -> false;
is_community_umbrella_app("apps/emqx_ft") -> false;
is_community_umbrella_app("apps/emqx_s3") -> false;
is_community_umbrella_app("apps/emqx_schema_registry") -> false;
is_community_umbrella_app("apps/emqx_enterprise") -> false;
is_community_umbrella_app(_) -> true.
@ -486,7 +487,7 @@ relx_apps_per_edition(ee) ->
emqx_oracle,
emqx_bridge_oracle,
emqx_bridge_rabbitmq,
emqx_ee_schema_registry,
emqx_schema_registry,
emqx_eviction_agent,
emqx_node_rebalance,
emqx_ft

View File

@ -1,4 +1,4 @@
emqx_ee_schema_registry_http_api {
emqx_schema_registry_http_api {
desc_param_path_schema_name.desc:
"""The schema name"""

View File

@ -1,4 +1,4 @@
emqx_ee_schema_registry_schema {
emqx_schema_registry_schema {
avro_type.desc:
"""[Apache Avro](https://avro.apache.org/) serialization format."""