refactor(schema_registry): refactor schema registry app and modules

Fixes https://emqx.atlassian.net/browse/EMQX-10361

- Moves `lib-ee/emqx_ee_schema_registry` to `apps/emqx_schema_registry`.
- Removes the `_ee_` segment from module names.
  - Exceptions are the table names which are kept to avoid backwards incompatibilities.

apps/emqx_enterprise/src/emqx_enterprise_schema.erl

@@ -10,7 +10,7 @@
 
 -define(EE_SCHEMA_MODULES, [
     emqx_license_schema,
-    emqx_ee_schema_registry_schema,
+    emqx_schema_registry_schema,
     emqx_ft_schema
 ]).
 

apps/emqx_machine/src/emqx_machine_boot.erl

@@ -158,7 +158,7 @@ basic_reboot_apps_edition(ee) ->
         emqx_ft,
         emqx_eviction_agent,
         emqx_node_rebalance,
-        emqx_ee_schema_registry
+        emqx_schema_registry
     ];
 %% unexcepted edition, should not happen
 basic_reboot_apps_edition(_) ->

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -1148,7 +1148,7 @@ timezone_to_offset_seconds(TimeZone) ->
 '$handle_undefined_function'(sprintf, [Format | Args]) ->
     erlang:apply(fun sprintf_s/2, [Format, Args]);
 %% This is for functions that should be handled in another module
-%% (currently this module is emqx_ee_schema_registry_serde in the case of EE but
+%% (currently this module is emqx_schema_registry_serde in the case of EE but
 %% could be changed to another module in the future).
 '$handle_undefined_function'(FunctionName, Args) ->
     case emqx_rule_engine:extra_functions_module() of

apps/emqx_schema_registry/BSL.txt

@@ -0,0 +1,94 @@
+Business Source License 1.1
+
+Licensor:             Hangzhou EMQ Technologies Co., Ltd.
+Licensed Work:        EMQX Enterprise Edition
+                      The Licensed Work is (c) 2023
+                      Hangzhou EMQ Technologies Co., Ltd.
+Additional Use Grant: Students and educators are granted right to copy,
+                      modify, and create derivative work for research
+                      or education.
+Change Date:          2027-02-01
+Change License:       Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Software,
+please contact Licensor: https://www.emqx.com/en/contact
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

apps/emqx_schema_registry/README.md

@@ -71,3 +71,18 @@ WHERE
 - Register schema instance: adds a new instance of a schema of a
   certain type.  For example, when the user may have several Avro or
   Protobuf schemas that they wish to use with different data flows.
+
+# Documentation
+
+- Refer to [Introduction to Schema Registry](https://docs.emqx.com/en/enterprise/v5.1/data-integration/schema-registry.html) for how to use the EMQX dashboard for configuring schemas.
+
+- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html)
+  for the EMQX rules engine introduction.
+
+# Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+# License
+
+EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt).

apps/emqx_schema_registry/include/emqx_schema_registry.hrl

@@ -2,12 +2,13 @@
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--ifndef(EMQX_EE_SCHEMA_REGISTRY_HRL).
+-ifndef(EMQX_SCHEMA_REGISTRY_HRL).
--define(EMQX_EE_SCHEMA_REGISTRY_HRL, true).
+-define(EMQX_SCHEMA_REGISTRY_HRL, true).
 
 -define(CONF_KEY_ROOT, schema_registry).
 -define(CONF_KEY_PATH, [?CONF_KEY_ROOT]).
 
+%% Note: this has the `_ee_' segment for backwards compatibility.
 -define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard).
 -define(SERDE_TAB, emqx_ee_schema_registry_serde_tab).
 -define(PROTOBUF_CACHE_TAB, emqx_ee_schema_registry_protobuf_cache_tab).

apps/emqx_schema_registry/rebar.config

@@ -2,14 +2,14 @@
 
 {erl_opts, [debug_info]}.
 {deps, [
-  {emqx, {path, "../../apps/emqx"}},
+  {emqx, {path, "../emqx"}},
-  {emqx_utils, {path, "../../apps/emqx_utils"}},
+  {emqx_utils, {path, "../emqx_utils"}},
-  {emqx_rule_engine, {path, "../../apps/emqx_rule_engine"}},
+  {emqx_rule_engine, {path, "../emqx_rule_engine"}},
   {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}},
   {gpb, "4.19.7"}
 ]}.
 
 {shell, [
   % {config, "config/sys.config"},
-    {apps, [emqx_ee_schema_registry]}
+    {apps, [emqx_schema_registry]}
 ]}.

apps/emqx_schema_registry/src/emqx_schema_registry.app.src

@@ -1,8 +1,8 @@
-{application, emqx_ee_schema_registry, [
+{application, emqx_schema_registry, [
     {description, "EMQX Schema Registry"},
     {vsn, "0.1.5"},
-    {registered, [emqx_ee_schema_registry_sup]},
+    {registered, [emqx_schema_registry_sup]},
-    {mod, {emqx_ee_schema_registry_app, []}},
+    {mod, {emqx_schema_registry_app, []}},
     {included_applications, [
         emqx_rule_engine
     ]},

apps/emqx_schema_registry/src/emqx_schema_registry.erl

@@ -1,13 +1,13 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry).
+-module(emqx_schema_registry).
 
 -behaviour(gen_server).
 -behaviour(emqx_config_handler).
 -behaviour(emqx_config_backup).
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -291,7 +291,7 @@ do_build_serde(Name, Serde) when not is_binary(Name) ->
 do_build_serde(Name, #{type := Type, source := Source}) ->
     try
         {Serializer, Deserializer, Destructor} =
-            emqx_ee_schema_registry_serde:make_serde(Type, Name, Source),
+            emqx_schema_registry_serde:make_serde(Type, Name, Source),
         Serde = #serde{
             name = Name,
             serializer = Serializer,

apps/emqx_schema_registry/src/emqx_schema_registry_app.erl

@@ -1,24 +1,24 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_app).
+-module(emqx_schema_registry_app).
 
 -behaviour(application).
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 
 -export([start/2, stop/1]).
 
 start(_StartType, _StartArgs) ->
     %% Register rule engine extra functions module so that we can handle decode
     %% and encode functions called from the rule engine SQL like language
-    ok = emqx_rule_engine:set_extra_functions_module(emqx_ee_schema_registry_serde),
+    ok = emqx_rule_engine:set_extra_functions_module(emqx_schema_registry_serde),
     ok = mria_rlog:wait_for_shards([?SCHEMA_REGISTRY_SHARD], infinity),
     %% HTTP API handler
-    emqx_conf:add_handler([?CONF_KEY_ROOT, schemas, '?'], emqx_ee_schema_registry),
+    emqx_conf:add_handler([?CONF_KEY_ROOT, schemas, '?'], emqx_schema_registry),
     %% Conf load / data import handler
-    emqx_conf:add_handler(?CONF_KEY_PATH, emqx_ee_schema_registry),
+    emqx_conf:add_handler(?CONF_KEY_PATH, emqx_schema_registry),
-    emqx_ee_schema_registry_sup:start_link().
+    emqx_schema_registry_sup:start_link().
 
 stop(_State) ->
     emqx_conf:remove_handler([?CONF_KEY_ROOT, schemas, '?']),

apps/emqx_schema_registry/src/emqx_schema_registry_http_api.erl

@@ -1,11 +1,11 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_http_api).
+-module(emqx_schema_registry_http_api).
 
 -behaviour(minirest_api).
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_utils/include/emqx_utils_api.hrl").
@@ -48,7 +48,7 @@ schema("/schema_registry") ->
                 #{
                     200 =>
                         emqx_dashboard_swagger:schema_with_examples(
-                            hoconsc:array(emqx_ee_schema_registry_schema:api_schema("get")),
+                            hoconsc:array(emqx_schema_registry_schema:api_schema("get")),
                             #{
                                 sample =>
                                     #{value => sample_list_schemas_response()}
@@ -61,14 +61,14 @@ schema("/schema_registry") ->
             summary => <<"Register a new schema">>,
             description => ?DESC("desc_schema_registry_api_post"),
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_ee_schema_registry_schema:api_schema("post"),
+                emqx_schema_registry_schema:api_schema("post"),
                 post_examples()
             ),
             responses =>
                 #{
                     201 =>
                         emqx_dashboard_swagger:schema_with_examples(
-                            emqx_ee_schema_registry_schema:api_schema("post"),
+                            emqx_schema_registry_schema:api_schema("post"),
                             post_examples()
                         ),
                     400 => error_schema('ALREADY_EXISTS', "Schema already exists")
@@ -87,7 +87,7 @@ schema("/schema_registry/:name") ->
                 #{
                     200 =>
                         emqx_dashboard_swagger:schema_with_examples(
-                            emqx_ee_schema_registry_schema:api_schema("get"),
+                            emqx_schema_registry_schema:api_schema("get"),
                             get_examples()
                         ),
                     404 => error_schema('NOT_FOUND', "Schema not found")
@@ -99,14 +99,14 @@ schema("/schema_registry/:name") ->
             description => ?DESC("desc_schema_registry_api_put"),
             parameters => [param_path_schema_name()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_ee_schema_registry_schema:api_schema("put"),
+                emqx_schema_registry_schema:api_schema("put"),
                 post_examples()
             ),
             responses =>
                 #{
                     200 =>
                         emqx_dashboard_swagger:schema_with_examples(
-                            emqx_ee_schema_registry_schema:api_schema("put"),
+                            emqx_schema_registry_schema:api_schema("put"),
                             put_examples()
                         ),
                     404 => error_schema('NOT_FOUND', "Schema not found")
@@ -130,7 +130,7 @@ schema("/schema_registry/:name") ->
 %%-------------------------------------------------------------------------------------------------
 
 '/schema_registry'(get, _Params) ->
-    Schemas = emqx_ee_schema_registry:list_schemas(),
+    Schemas = emqx_schema_registry:list_schemas(),
     Response =
         lists:map(
             fun({Name, Params}) ->
@@ -141,11 +141,11 @@ schema("/schema_registry/:name") ->
     ?OK(Response);
 '/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) ->
     Params = maps:without([<<"name">>], Params0),
-    case emqx_ee_schema_registry:get_schema(Name) of
+    case emqx_schema_registry:get_schema(Name) of
         {error, not_found} ->
-            case emqx_ee_schema_registry:add_schema(Name, Params) of
+            case emqx_schema_registry:add_schema(Name, Params) of
                 ok ->
-                    {ok, Res} = emqx_ee_schema_registry:get_schema(Name),
+                    {ok, Res} = emqx_schema_registry:get_schema(Name),
                     {201, Res#{name => Name}};
                 {error, Error} ->
                     ?BAD_REQUEST(Error)
@@ -155,31 +155,31 @@ schema("/schema_registry/:name") ->
     end.
 
 '/schema_registry/:name'(get, #{bindings := #{name := Name}}) ->
-    case emqx_ee_schema_registry:get_schema(Name) of
+    case emqx_schema_registry:get_schema(Name) of
         {error, not_found} ->
             ?NOT_FOUND(<<"Schema not found">>);
         {ok, Schema} ->
             ?OK(Schema#{name => Name})
     end;
 '/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) ->
-    case emqx_ee_schema_registry:get_schema(Name) of
+    case emqx_schema_registry:get_schema(Name) of
         {error, not_found} ->
             ?NOT_FOUND(<<"Schema not found">>);
         {ok, _} ->
-            case emqx_ee_schema_registry:add_schema(Name, Params) of
+            case emqx_schema_registry:add_schema(Name, Params) of
                 ok ->
-                    {ok, Res} = emqx_ee_schema_registry:get_schema(Name),
+                    {ok, Res} = emqx_schema_registry:get_schema(Name),
                     ?OK(Res#{name => Name});
                 {error, Error} ->
                     ?BAD_REQUEST(Error)
             end
     end;
 '/schema_registry/:name'(delete, #{bindings := #{name := Name}}) ->
-    case emqx_ee_schema_registry:get_schema(Name) of
+    case emqx_schema_registry:get_schema(Name) of
         {error, not_found} ->
             ?NOT_FOUND(<<"Schema not found">>);
         {ok, _} ->
-            case emqx_ee_schema_registry:delete_schema(Name) of
+            case emqx_schema_registry:delete_schema(Name) of
                 ok ->
                     ?NO_CONTENT;
                 {error, Error} ->

apps/emqx_schema_registry/src/emqx_schema_registry_schema.erl

@@ -2,11 +2,11 @@
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_schema_registry_schema).
+-module(emqx_schema_registry_schema).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 
 %% `hocon_schema' API
 -export([

apps/emqx_schema_registry/src/emqx_schema_registry_serde.erl

@@ -1,15 +1,15 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_serde).
+-module(emqx_schema_registry_serde).
 
 -behaviour(emqx_rule_funcs).
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ee_schema_registry_serde]}}]).
+-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_schema_registry_serde]}}]).
 
 %% API
 -export([
@@ -67,7 +67,7 @@ decode(SerdeName, RawData) ->
 
 -spec decode(schema_name(), encoded_data(), [term()]) -> decoded_data().
 decode(SerdeName, RawData, VarArgs) when is_list(VarArgs) ->
-    case emqx_ee_schema_registry:get_serde(SerdeName) of
+    case emqx_schema_registry:get_serde(SerdeName) of
         {error, not_found} ->
             error({serde_not_found, SerdeName});
         {ok, #{deserializer := Deserializer}} ->
@@ -80,7 +80,7 @@ encode(SerdeName, RawData) ->
 
 -spec encode(schema_name(), decoded_data(), [term()]) -> encoded_data().
 encode(SerdeName, EncodedData, VarArgs) when is_list(VarArgs) ->
-    case emqx_ee_schema_registry:get_serde(SerdeName) of
+    case emqx_schema_registry:get_serde(SerdeName) of
         {error, not_found} ->
             error({serde_not_found, SerdeName});
         {ok, #{serializer := Serializer}} ->

apps/emqx_schema_registry/src/emqx_schema_registry_sup.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_sup).
+-module(emqx_schema_registry_sup).
 
 -behaviour(supervisor).
 
@@ -29,7 +29,7 @@ init([]) ->
         intensity => 10,
         period => 100
     },
-    ChildSpecs = [child_spec(emqx_ee_schema_registry)],
+    ChildSpecs = [child_spec(emqx_schema_registry)],
     {ok, {SupFlags, ChildSpecs}}.
 
 child_spec(Mod) ->
@@ -38,6 +38,5 @@ child_spec(Mod) ->
         start => {Mod, start_link, []},
         restart => permanent,
         shutdown => 5_000,
-        type => worker,
+        type => worker
-        modules => [Mod]
     }.

apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_SUITE).
+-module(emqx_schema_registry_SUITE).
 
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -10,11 +10,11 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 
 -import(emqx_common_test_helpers, [on_exit/1]).
 
--define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]).
+-define(APPS, [emqx_conf, emqx_rule_engine, emqx_schema_registry]).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -46,7 +46,7 @@ sparkplug_tests() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
+    emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
     emqx_mgmt_api_test_util:init_suite(?APPS),
     Config.
 
@@ -156,7 +156,7 @@ schema_params(protobuf) ->
 
 create_serde(SerdeType, SerdeName) ->
     Schema = schema_params(SerdeType),
-    ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema),
+    ok = emqx_schema_registry:add_schema(SerdeName, Schema),
     ok.
 
 test_params_for(avro, encode_decode1) ->
@@ -313,9 +313,9 @@ test_params_for(Type, Name) ->
 clear_schemas() ->
     maps:foreach(
         fun(Name, _Schema) ->
-            ok = emqx_ee_schema_registry:delete_schema(Name)
+            ok = emqx_schema_registry:delete_schema(Name)
         end,
-        emqx_ee_schema_registry:list_schemas()
+        emqx_schema_registry:list_schemas()
     ).
 
 receive_action_results() ->
@@ -367,7 +367,7 @@ cluster(Config) ->
             %% need to restart schema registry app in the tests so
             %% that it re-registers the config handler that is lost
             %% when emqx_conf restarts during join.
-            {env, [{emqx_machine, applications, [emqx_ee_schema_registry]}]},
+            {env, [{emqx_machine, applications, [emqx_schema_registry]}]},
             {load_apps, [emqx_machine | ?APPS]},
             {env_handler, fun
                 (emqx) ->
@@ -453,11 +453,11 @@ protobuf_unique_cache_hit_spec(_Res, _Trace) ->
 %%------------------------------------------------------------------------------
 
 t_unknown_calls(_Config) ->
-    Ref = monitor(process, emqx_ee_schema_registry),
+    Ref = monitor(process, emqx_schema_registry),
     %% for coverage
-    emqx_ee_schema_registry ! unknown,
+    emqx_schema_registry ! unknown,
-    gen_server:cast(emqx_ee_schema_registry, unknown),
+    gen_server:cast(emqx_schema_registry, unknown),
-    ?assertEqual({error, unknown_call}, gen_server:call(emqx_ee_schema_registry, unknown)),
+    ?assertEqual({error, unknown_call}, gen_server:call(emqx_schema_registry, unknown)),
     receive
         {'DOWN', Ref, process, _, _} ->
             ct:fail("registry shouldn't have died")
@@ -489,7 +489,7 @@ t_delete_serde(Config) ->
             ok = create_serde(SerdeType, SerdeName),
             {ok, {ok, _}} =
                 ?wait_async_action(
-                    emqx_ee_schema_registry:delete_schema(SerdeName),
+                    emqx_schema_registry:delete_schema(SerdeName),
                     #{?snk_kind := schema_registry_serdes_deleted},
                     1_000
                 ),
@@ -522,7 +522,7 @@ t_encode(Config) ->
         Published
     ),
     #{payload := Encoded} = Published,
-    {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
+    {ok, #{deserializer := Deserializer}} = emqx_schema_registry:get_serde(SerdeName),
     ?assertEqual(Payload, apply(Deserializer, [Encoded | ExtraArgs])),
     ok.
 
@@ -536,7 +536,7 @@ t_decode(Config) ->
         extra_args := ExtraArgs
     } = test_params_for(SerdeType, decode1),
     {ok, _} = create_rule_http(#{sql => SQL}),
-    {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
+    {ok, #{serializer := Serializer}} = emqx_schema_registry:get_serde(SerdeName),
     EncodedBin = apply(Serializer, [Payload | ExtraArgs]),
     emqx:publish(emqx_message:make(<<"t">>, EncodedBin)),
     Published = receive_published(?LINE),
@@ -559,7 +559,7 @@ t_protobuf_union_encode(Config) ->
         extra_args := ExtraArgs
     } = test_params_for(SerdeType, union1),
     {ok, _} = create_rule_http(#{sql => SQL}),
-    {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
+    {ok, #{serializer := Serializer}} = emqx_schema_registry:get_serde(SerdeName),
 
     EncodedBinA = apply(Serializer, [PayloadA | ExtraArgs]),
     emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)),
@@ -594,7 +594,7 @@ t_protobuf_union_decode(Config) ->
         extra_args := ExtraArgs
     } = test_params_for(SerdeType, union2),
     {ok, _} = create_rule_http(#{sql => SQL}),
-    {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName),
+    {ok, #{deserializer := Deserializer}} = emqx_schema_registry:get_serde(SerdeName),
 
     EncodedBinA = emqx_utils_json:encode(PayloadA),
     emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)),
@@ -639,9 +639,9 @@ t_fail_rollback(Config) ->
             #{}
         )
     ),
-    ?assertMatch({ok, #{name := <<"a">>}}, emqx_ee_schema_registry:get_serde(<<"a">>)),
+    ?assertMatch({ok, #{name := <<"a">>}}, emqx_schema_registry:get_serde(<<"a">>)),
     %% no z serdes should be in the table
-    ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"z">>)),
+    ?assertEqual({error, not_found}, emqx_schema_registry:get_serde(<<"z">>)),
     ok.
 
 t_cluster_serde_build(Config) ->
@@ -660,13 +660,13 @@ t_cluster_serde_build(Config) ->
             wait_for_cluster_rpc(N2),
             ?assertEqual(
                 ok,
-                erpc:call(N2, emqx_ee_schema_registry, add_schema, [SerdeName, Schema])
+                erpc:call(N2, emqx_schema_registry, add_schema, [SerdeName, Schema])
             ),
             %% check that we can serialize/deserialize in all nodes
             lists:foreach(
                 fun(N) ->
                     erpc:call(N, fun() ->
-                        Res0 = emqx_ee_schema_registry:get_serde(SerdeName),
+                        Res0 = emqx_schema_registry:get_serde(SerdeName),
                         ?assertMatch({ok, #{}}, Res0, #{node => N}),
                         {ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0,
                         ?assertEqual(
@@ -691,7 +691,7 @@ t_cluster_serde_build(Config) ->
             ),
             ?assertEqual(
                 ok,
-                erpc:call(N1, emqx_ee_schema_registry, delete_schema, [SerdeName])
+                erpc:call(N1, emqx_schema_registry, delete_schema, [SerdeName])
             ),
             {ok, _} = snabbkaffe:receive_events(SRef1),
             lists:foreach(
@@ -699,7 +699,7 @@ t_cluster_serde_build(Config) ->
                     erpc:call(N, fun() ->
                         ?assertMatch(
                             {error, not_found},
-                            emqx_ee_schema_registry:get_serde(SerdeName),
+                            emqx_schema_registry:get_serde(SerdeName),
                             #{node => N}
                         ),
                         ok
@@ -726,7 +726,15 @@ t_import_config(_Config) ->
                             #{
                                 <<"description">> => <<"My Avro Schema">>,
                                 <<"source">> =>
-                                    <<"{\"type\":\"record\",\"fields\":[{\"type\":\"int\",\"name\":\"i\"},{\"type\":\"string\",\"name\":\"s\"}]}">>,
+                                    emqx_utils_json:encode(
+                                        #{
+                                            type => <<"record">>,
+                                            fields => [
+                                                #{type => <<"int">>, name => <<"i">>},
+                                                #{type => <<"string">>, name => <<"s">>}
+                                            ]
+                                        }
+                                    ),
                                 <<"type">> => <<"avro">>
                             }
                     }
@@ -740,15 +748,21 @@ t_import_config(_Config) ->
     Path = [schema_registry, schemas, <<"my_avro_schema">>],
     ?assertEqual(
         {ok, #{root_key => schema_registry, changed => []}},
-        emqx_ee_schema_registry:import_config(RawConf)
+        emqx_schema_registry:import_config(RawConf)
     ),
     ?assertEqual(
         {ok, #{root_key => schema_registry, changed => [Path]}},
-        emqx_ee_schema_registry:import_config(RawConf1)
+        emqx_schema_registry:import_config(RawConf1)
     ).
 
 sparkplug_example_data_base64() ->
-    <<"CPHh67HrMBIqChxjb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xc2VjGPXh67HrMCACUKgDEikKHGNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxXzVzZWMY9eHrseswIAJQVBIqCh1jb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xMHNlYxj14eux6zAgAlAqEigKG2NvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxX3J1bhj14eux6zAgBVABEioKHWNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxX3Jlc2V0GPXh67HrMCAFUAAYWA==">>.
+    <<
+        "CPHh67HrMBIqChxjb3VudGVyX2dyb3VwMS9jb3VudGVyMV8xc2VjGPXh67HrMCACUKgD"
+        "EikKHGNvdW50ZXJfZ3JvdXAxL2NvdW50ZXIxXzVzZWMY9eHrseswIAJQVBIqCh1jb3Vu"
+        "dGVyX2dyb3VwMS9jb3VudGVyMV8xMHNlYxj14eux6zAgAlAqEigKG2NvdW50ZXJfZ3Jv"
+        "dXAxL2NvdW50ZXIxX3J1bhj14eux6zAgBVABEioKHWNvdW50ZXJfZ3JvdXAxL2NvdW50"
+        "ZXIxX3Jlc2V0GPXh67HrMCAFUAAYWA=="
+    >>.
 
 sparkplug_example_data() ->
     #{

apps/emqx_schema_registry/test/emqx_schema_registry_http_api_SUITE.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_http_api_SUITE).
+-module(emqx_schema_registry_http_api_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -12,9 +12,9 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 
--define(APPS, [emqx_conf, emqx_ee_schema_registry]).
+-define(APPS, [emqx_conf, emqx_schema_registry]).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -34,7 +34,7 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
+    emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
     emqx_mgmt_api_test_util:init_suite(?APPS),
     Config.
 
@@ -138,9 +138,9 @@ try_decode_error_message(Res) ->
 clear_schemas() ->
     maps:foreach(
         fun(Name, _Schema) ->
-            ok = emqx_ee_schema_registry:delete_schema(Name)
+            ok = emqx_schema_registry:delete_schema(Name)
         end,
-        emqx_ee_schema_registry:list_schemas()
+        emqx_schema_registry:list_schemas()
     ).
 
 %%------------------------------------------------------------------------------
@@ -249,7 +249,7 @@ t_crud(Config) ->
         {ok, 400, #{
             <<"code">> := <<"BAD_REQUEST">>,
             <<"message">> :=
-                <<"{post_config_update,emqx_ee_schema_registry,", _/binary>>
+                <<"{post_config_update,emqx_schema_registry,", _/binary>>
         }},
         request({put, SchemaName, UpdateParams#{<<"source">> := InvalidSourceBin}})
     ),
@@ -290,7 +290,7 @@ t_crud(Config) ->
         {ok, 400, #{
             <<"code">> := <<"BAD_REQUEST">>,
             <<"message">> :=
-                <<"{post_config_update,emqx_ee_schema_registry,", _/binary>>
+                <<"{post_config_update,emqx_schema_registry,", _/binary>>
         }},
         request({post, Params#{<<"source">> := InvalidSourceBin}})
     ),

apps/emqx_schema_registry/test/emqx_schema_registry_serde_SUITE.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_schema_registry_serde_SUITE).
+-module(emqx_schema_registry_serde_SUITE).
 
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -10,11 +10,11 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--include("emqx_ee_schema_registry.hrl").
+-include("emqx_schema_registry.hrl").
 
 -import(emqx_common_test_helpers, [on_exit/1]).
 
--define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]).
+-define(APPS, [emqx_conf, emqx_rule_engine, emqx_schema_registry]).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -24,7 +24,7 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema),
+    emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
     emqx_mgmt_api_test_util:init_suite(?APPS),
     Config.
 
@@ -46,9 +46,9 @@ end_per_testcase(_TestCase, _Config) ->
 clear_schemas() ->
     maps:foreach(
         fun(Name, _Schema) ->
-            ok = emqx_ee_schema_registry:delete_schema(Name)
+            ok = emqx_schema_registry:delete_schema(Name)
         end,
-        emqx_ee_schema_registry:list_schemas()
+        emqx_schema_registry:list_schemas()
     ).
 
 schema_params(avro) ->
@@ -81,13 +81,13 @@ schema_params(protobuf) ->
     #{type => protobuf, source => SourceBin}.
 
 assert_roundtrip(SerdeName, Original) ->
-    Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original),
+    Encoded = emqx_schema_registry_serde:encode(SerdeName, Original),
-    Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded),
+    Decoded = emqx_schema_registry_serde:decode(SerdeName, Encoded),
     ?assertEqual(Original, Decoded, #{original => Original}).
 
 assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) ->
-    Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize),
+    Encoded = emqx_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize),
-    Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize),
+    Decoded = emqx_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize),
     ?assertEqual(Original, Decoded, #{original => Original}).
 
 %%------------------------------------------------------------------------------
@@ -97,7 +97,7 @@ assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) ->
 t_roundtrip_avro(_Config) ->
     SerdeName = my_serde,
     Params = schema_params(avro),
-    ok = emqx_ee_schema_registry:add_schema(SerdeName, Params),
+    ok = emqx_schema_registry:add_schema(SerdeName, Params),
     Original = #{<<"i">> => 10, <<"s">> => <<"hi">>},
     %% for coverage
     assert_roundtrip(SerdeName, Original, _ArgsSerialize = [], _ArgsDeserialize = []),
@@ -110,7 +110,7 @@ t_avro_invalid_json_schema(_Config) ->
     WrongParams = Params#{source := <<"{">>},
     ?assertMatch(
         {error, #{reason := #{expected := _}}},
-        emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
+        emqx_schema_registry:add_schema(SerdeName, WrongParams)
     ),
     ok.
 
@@ -120,7 +120,7 @@ t_avro_invalid_schema(_Config) ->
     WrongParams = Params#{source := <<"{}">>},
     ?assertMatch(
         {error, {post_config_update, _, {not_found, <<"type">>}}},
-        emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
+        emqx_schema_registry:add_schema(SerdeName, WrongParams)
     ),
     ok.
 
@@ -130,18 +130,18 @@ t_serde_not_found(_Config) ->
     Original = #{},
     ?assertError(
         {serde_not_found, NonexistentSerde},
-        emqx_ee_schema_registry_serde:encode(NonexistentSerde, Original)
+        emqx_schema_registry_serde:encode(NonexistentSerde, Original)
     ),
     ?assertError(
         {serde_not_found, NonexistentSerde},
-        emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original)
+        emqx_schema_registry_serde:decode(NonexistentSerde, Original)
     ),
     ok.
 
 t_roundtrip_protobuf(_Config) ->
     SerdeName = my_serde,
     Params = schema_params(protobuf),
-    ok = emqx_ee_schema_registry:add_schema(SerdeName, Params),
+    ok = emqx_schema_registry:add_schema(SerdeName, Params),
     ExtraArgsPerson = [<<"Person">>],
 
     Original0 = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>},
@@ -167,6 +167,6 @@ t_protobuf_invalid_schema(_Config) ->
     WrongParams = Params#{source := <<"xxxx">>},
     ?assertMatch(
         {error, {post_config_update, _, {invalid_protobuf_schema, _}}},
-        emqx_ee_schema_registry:add_schema(SerdeName, WrongParams)
+        emqx_schema_registry:add_schema(SerdeName, WrongParams)
     ),
     ok.

changes/ee/feat-11241.en.md

@@ -0,0 +1 @@
+Schema Registry has been refactored to its own Erlang application. This allows for more flexibility in the future.

lib-ee/emqx_ee_schema_registry/.gitignore

@@ -1,19 +0,0 @@
-.rebar3
-_*
-.eunit
-*.o
-*.beam
-*.plt
-*.swp
-*.swo
-.erlang.cookie
-ebin
-log
-erl_crash.dump
-.rebar
-logs
-_build
-.idea
-*.iml
-rebar3.crashdump
-*~

mix.exs

@@ -190,6 +190,7 @@ defmodule EMQXUmbrella.MixProject do
       :emqx_bridge_clickhouse,
       :emqx_ft,
       :emqx_s3,
+      :emqx_schema_registry,
       :emqx_enterprise
     ])
   end
@@ -418,7 +419,7 @@ defmodule EMQXUmbrella.MixProject do
           emqx_oracle: :permanent,
           emqx_bridge_oracle: :permanent,
           emqx_bridge_rabbitmq: :permanent,
-          emqx_ee_schema_registry: :permanent,
+          emqx_schema_registry: :permanent,
           emqx_eviction_agent: :permanent,
           emqx_node_rebalance: :permanent,
           emqx_ft: :permanent

rebar.config.erl

@@ -102,6 +102,7 @@ is_community_umbrella_app("apps/emqx_oracle") -> false;
 is_community_umbrella_app("apps/emqx_bridge_rabbitmq") -> false;
 is_community_umbrella_app("apps/emqx_ft") -> false;
 is_community_umbrella_app("apps/emqx_s3") -> false;
+is_community_umbrella_app("apps/emqx_schema_registry") -> false;
 is_community_umbrella_app("apps/emqx_enterprise") -> false;
 is_community_umbrella_app(_) -> true.
 
@@ -486,7 +487,7 @@ relx_apps_per_edition(ee) ->
         emqx_oracle,
         emqx_bridge_oracle,
         emqx_bridge_rabbitmq,
-        emqx_ee_schema_registry,
+        emqx_schema_registry,
         emqx_eviction_agent,
         emqx_node_rebalance,
         emqx_ft

rel/i18n/emqx_schema_registry_http_api.hocon

@@ -1,4 +1,4 @@
-emqx_ee_schema_registry_http_api {
+emqx_schema_registry_http_api {
 
 desc_param_path_schema_name.desc:
 """The schema name"""

rel/i18n/emqx_schema_registry_schema.hocon

@@ -1,4 +1,4 @@
-emqx_ee_schema_registry_schema {
+emqx_schema_registry_schema {
 
 avro_type.desc:
 """[Apache Avro](https://avro.apache.org/) serialization format."""