Merge remote-tracking branch 'origin/release-54' into sync-r54-m-20231221
This commit is contained in:
commit
035f5f977e
4
Makefile
4
Makefile
|
@ -20,8 +20,8 @@ endif
|
||||||
|
|
||||||
# Dashboard version
|
# Dashboard version
|
||||||
# from https://github.com/emqx/emqx-dashboard5
|
# from https://github.com/emqx/emqx-dashboard5
|
||||||
export EMQX_DASHBOARD_VERSION ?= v1.5.2
|
export EMQX_DASHBOARD_VERSION ?= v1.6.0
|
||||||
export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0-beta.8
|
export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0
|
||||||
|
|
||||||
PROFILE ?= emqx
|
PROFILE ?= emqx
|
||||||
REL_PROFILES := emqx emqx-enterprise
|
REL_PROFILES := emqx emqx-enterprise
|
||||||
|
|
|
@ -32,10 +32,10 @@
|
||||||
%% `apps/emqx/src/bpapi/README.md'
|
%% `apps/emqx/src/bpapi/README.md'
|
||||||
|
|
||||||
%% Opensource edition
|
%% Opensource edition
|
||||||
-define(EMQX_RELEASE_CE, "5.4.0-rc.1").
|
-define(EMQX_RELEASE_CE, "5.4.0").
|
||||||
|
|
||||||
%% Enterprise edition
|
%% Enterprise edition
|
||||||
-define(EMQX_RELEASE_EE, "5.4.0-rc.1").
|
-define(EMQX_RELEASE_EE, "5.4.0").
|
||||||
|
|
||||||
%% The HTTP API version
|
%% The HTTP API version
|
||||||
-define(EMQX_API_VERSION, "5.0").
|
-define(EMQX_API_VERSION, "5.0").
|
||||||
|
|
|
@ -416,7 +416,7 @@ uninstall_bridge_v2(
|
||||||
{error, _} ->
|
{error, _} ->
|
||||||
ok;
|
ok;
|
||||||
ok ->
|
ok ->
|
||||||
%% Deinstall from connector
|
%% uninstall from connector
|
||||||
ConnectorId = emqx_connector_resource:resource_id(
|
ConnectorId = emqx_connector_resource:resource_id(
|
||||||
connector_type(BridgeV2Type), ConnectorName
|
connector_type(BridgeV2Type), ConnectorName
|
||||||
),
|
),
|
||||||
|
@ -869,6 +869,8 @@ config_key_path() ->
|
||||||
config_key_path_leaf() ->
|
config_key_path_leaf() ->
|
||||||
[?ROOT_KEY, '?', '?'].
|
[?ROOT_KEY, '?', '?'].
|
||||||
|
|
||||||
|
pre_config_update(_, {force_update, Conf}, _OldConf) ->
|
||||||
|
{ok, Conf};
|
||||||
%% NOTE: We depend on the `emqx_bridge:pre_config_update/3` to restart/stop the
|
%% NOTE: We depend on the `emqx_bridge:pre_config_update/3` to restart/stop the
|
||||||
%% underlying resources.
|
%% underlying resources.
|
||||||
pre_config_update(_, {_Oper, _, _}, undefined) ->
|
pre_config_update(_, {_Oper, _, _}, undefined) ->
|
||||||
|
@ -882,55 +884,15 @@ pre_config_update(_Path, Conf, _OldConfig) when is_map(Conf) ->
|
||||||
operation_to_enable(disable) -> false;
|
operation_to_enable(disable) -> false;
|
||||||
operation_to_enable(enable) -> true.
|
operation_to_enable(enable) -> true.
|
||||||
|
|
||||||
|
%% A public API that can trigger this is:
|
||||||
|
%% bin/emqx ctl conf load data/configs/cluster.hocon
|
||||||
|
post_config_update([?ROOT_KEY], {force_update, _Req}, NewConf, OldConf, _AppEnv) ->
|
||||||
|
do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => false});
|
||||||
%% This top level handler will be triggered when the actions path is updated
|
%% This top level handler will be triggered when the actions path is updated
|
||||||
%% with calls to emqx_conf:update([actions], BridgesConf, #{}).
|
%% with calls to emqx_conf:update([actions], BridgesConf, #{}).
|
||||||
%%
|
%%
|
||||||
%% A public API that can trigger this is:
|
|
||||||
%% bin/emqx ctl conf load data/configs/cluster.hocon
|
|
||||||
post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
|
post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
|
||||||
#{added := Added, removed := Removed, changed := Updated} =
|
do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => true});
|
||||||
diff_confs(NewConf, OldConf),
|
|
||||||
%% new and updated bridges must have their connector references validated
|
|
||||||
UpdatedConfigs =
|
|
||||||
lists:map(
|
|
||||||
fun({{Type, BridgeName}, {_Old, New}}) ->
|
|
||||||
{Type, BridgeName, New}
|
|
||||||
end,
|
|
||||||
maps:to_list(Updated)
|
|
||||||
),
|
|
||||||
AddedConfigs =
|
|
||||||
lists:map(
|
|
||||||
fun({{Type, BridgeName}, AddedConf}) ->
|
|
||||||
{Type, BridgeName, AddedConf}
|
|
||||||
end,
|
|
||||||
maps:to_list(Added)
|
|
||||||
),
|
|
||||||
ToValidate = UpdatedConfigs ++ AddedConfigs,
|
|
||||||
case multi_validate_referenced_connectors(ToValidate) of
|
|
||||||
ok ->
|
|
||||||
%% The config update will be failed if any task in `perform_bridge_changes` failed.
|
|
||||||
RemoveFun = fun uninstall_bridge_v2/3,
|
|
||||||
CreateFun = fun install_bridge_v2/3,
|
|
||||||
UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) ->
|
|
||||||
uninstall_bridge_v2(Type, Name, OldBridgeConf),
|
|
||||||
install_bridge_v2(Type, Name, Conf)
|
|
||||||
end,
|
|
||||||
Result = perform_bridge_changes([
|
|
||||||
#{action => RemoveFun, data => Removed},
|
|
||||||
#{
|
|
||||||
action => CreateFun,
|
|
||||||
data => Added,
|
|
||||||
on_exception_fn => fun emqx_bridge_resource:remove/4
|
|
||||||
},
|
|
||||||
#{action => UpdateFun, data => Updated}
|
|
||||||
]),
|
|
||||||
ok = unload_message_publish_hook(),
|
|
||||||
ok = load_message_publish_hook(NewConf),
|
|
||||||
?tp(bridge_post_config_update_done, #{}),
|
|
||||||
Result;
|
|
||||||
{error, Error} ->
|
|
||||||
{error, Error}
|
|
||||||
end;
|
|
||||||
post_config_update([?ROOT_KEY, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) ->
|
post_config_update([?ROOT_KEY, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) ->
|
||||||
Conf = emqx:get_config([?ROOT_KEY, BridgeType, BridgeName]),
|
Conf = emqx:get_config([?ROOT_KEY, BridgeType, BridgeName]),
|
||||||
ok = uninstall_bridge_v2(BridgeType, BridgeName, Conf),
|
ok = uninstall_bridge_v2(BridgeType, BridgeName, Conf),
|
||||||
|
@ -970,6 +932,50 @@ post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf,
|
||||||
{error, Error}
|
{error, Error}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
|
do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors := NeedValidate}) ->
|
||||||
|
#{added := Added, removed := Removed, changed := Updated} =
|
||||||
|
diff_confs(NewConf, OldConf),
|
||||||
|
UpdatedConfigs =
|
||||||
|
lists:map(
|
||||||
|
fun({{Type, BridgeName}, {_Old, New}}) ->
|
||||||
|
{Type, BridgeName, New}
|
||||||
|
end,
|
||||||
|
maps:to_list(Updated)
|
||||||
|
),
|
||||||
|
AddedConfigs =
|
||||||
|
lists:map(
|
||||||
|
fun({{Type, BridgeName}, AddedConf}) ->
|
||||||
|
{Type, BridgeName, AddedConf}
|
||||||
|
end,
|
||||||
|
maps:to_list(Added)
|
||||||
|
),
|
||||||
|
ToValidate = UpdatedConfigs ++ AddedConfigs,
|
||||||
|
case multi_validate_referenced_connectors(NeedValidate, ToValidate) of
|
||||||
|
ok ->
|
||||||
|
%% The config update will be failed if any task in `perform_bridge_changes` failed.
|
||||||
|
RemoveFun = fun uninstall_bridge_v2/3,
|
||||||
|
CreateFun = fun install_bridge_v2/3,
|
||||||
|
UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) ->
|
||||||
|
uninstall_bridge_v2(Type, Name, OldBridgeConf),
|
||||||
|
install_bridge_v2(Type, Name, Conf)
|
||||||
|
end,
|
||||||
|
Result = perform_bridge_changes([
|
||||||
|
#{action => RemoveFun, data => Removed},
|
||||||
|
#{
|
||||||
|
action => CreateFun,
|
||||||
|
data => Added,
|
||||||
|
on_exception_fn => fun emqx_bridge_resource:remove/4
|
||||||
|
},
|
||||||
|
#{action => UpdateFun, data => Updated}
|
||||||
|
]),
|
||||||
|
ok = unload_message_publish_hook(),
|
||||||
|
ok = load_message_publish_hook(NewConf),
|
||||||
|
?tp(bridge_post_config_update_done, #{}),
|
||||||
|
Result;
|
||||||
|
{error, Error} ->
|
||||||
|
{error, Error}
|
||||||
|
end.
|
||||||
|
|
||||||
diff_confs(NewConfs, OldConfs) ->
|
diff_confs(NewConfs, OldConfs) ->
|
||||||
emqx_utils_maps:diff_maps(
|
emqx_utils_maps:diff_maps(
|
||||||
flatten_confs(NewConfs),
|
flatten_confs(NewConfs),
|
||||||
|
@ -1600,7 +1606,9 @@ to_connector(ConnectorNameBin, BridgeType) ->
|
||||||
throw(not_found)
|
throw(not_found)
|
||||||
end.
|
end.
|
||||||
|
|
||||||
multi_validate_referenced_connectors(Configs) ->
|
multi_validate_referenced_connectors(false, _Configs) ->
|
||||||
|
ok;
|
||||||
|
multi_validate_referenced_connectors(true, Configs) ->
|
||||||
Pipeline =
|
Pipeline =
|
||||||
lists:map(
|
lists:map(
|
||||||
fun({Type, BridgeName, #{connector := ConnectorName}}) ->
|
fun({Type, BridgeName, #{connector := ConnectorName}}) ->
|
||||||
|
|
|
@ -11,8 +11,8 @@
|
||||||
action_type_name/0,
|
action_type_name/0,
|
||||||
connector_type_name/0,
|
connector_type_name/0,
|
||||||
schema_module/0,
|
schema_module/0,
|
||||||
bridge_v1_config_to_action_config/2,
|
|
||||||
connector_action_config_to_bridge_v1_config/2,
|
connector_action_config_to_bridge_v1_config/2,
|
||||||
|
bridge_v1_config_to_action_config/2,
|
||||||
bridge_v1_config_to_connector_config/1,
|
bridge_v1_config_to_connector_config/1,
|
||||||
bridge_v1_type_name_fun/1
|
bridge_v1_type_name_fun/1
|
||||||
]).
|
]).
|
||||||
|
@ -28,14 +28,25 @@ connector_type_name() -> redis.
|
||||||
|
|
||||||
schema_module() -> ?SCHEMA_MODULE.
|
schema_module() -> ?SCHEMA_MODULE.
|
||||||
|
|
||||||
|
%% redis_cluster don't have batch options
|
||||||
connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) ->
|
connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) ->
|
||||||
maps:merge(
|
Config0 = emqx_utils_maps:deep_merge(
|
||||||
maps:without(
|
maps:without(
|
||||||
[<<"connector">>],
|
[<<"connector">>],
|
||||||
map_unindent(<<"parameters">>, ActionConfig)
|
emqx_utils_maps:unindent(<<"parameters">>, ActionConfig)
|
||||||
),
|
),
|
||||||
map_unindent(<<"parameters">>, ConnectorConfig)
|
emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig)
|
||||||
).
|
),
|
||||||
|
Config1 =
|
||||||
|
case Config0 of
|
||||||
|
#{<<"resource_opts">> := ResOpts0, <<"redis_type">> := Type} ->
|
||||||
|
Schema = emqx_bridge_redis:fields("creation_opts_redis_" ++ binary_to_list(Type)),
|
||||||
|
ResOpts = maps:with(schema_keys(Schema), ResOpts0),
|
||||||
|
Config0#{<<"resource_opts">> => ResOpts};
|
||||||
|
_ ->
|
||||||
|
Config0
|
||||||
|
end,
|
||||||
|
maps:without([<<"description">>], Config1).
|
||||||
|
|
||||||
bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) ->
|
bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) ->
|
||||||
ActionTopLevelKeys = schema_keys(?SCHEMA_MODULE:fields(redis_action)),
|
ActionTopLevelKeys = schema_keys(?SCHEMA_MODULE:fields(redis_action)),
|
||||||
|
@ -81,22 +92,9 @@ v1_type(<<"cluster">>) -> redis_cluster.
|
||||||
|
|
||||||
bridge_v1_type_names() -> [redis_single, redis_sentinel, redis_cluster].
|
bridge_v1_type_names() -> [redis_single, redis_sentinel, redis_cluster].
|
||||||
|
|
||||||
map_unindent(Key, Map) ->
|
|
||||||
maps:merge(
|
|
||||||
maps:get(Key, Map),
|
|
||||||
maps:remove(Key, Map)
|
|
||||||
).
|
|
||||||
|
|
||||||
map_indent(IndentKey, PickKeys, Map) ->
|
|
||||||
maps:put(
|
|
||||||
IndentKey,
|
|
||||||
maps:with(PickKeys, Map),
|
|
||||||
maps:without(PickKeys, Map)
|
|
||||||
).
|
|
||||||
|
|
||||||
schema_keys(Schema) ->
|
schema_keys(Schema) ->
|
||||||
[bin(Key) || {Key, _} <- Schema].
|
[bin(Key) || {Key, _} <- Schema].
|
||||||
|
|
||||||
make_config_map(PickKeys, IndentKeys, Config) ->
|
make_config_map(PickKeys, IndentKeys, Config) ->
|
||||||
Conf0 = maps:with(PickKeys, Config),
|
Conf0 = maps:with(PickKeys, Config),
|
||||||
map_indent(<<"parameters">>, IndentKeys, Conf0).
|
emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0).
|
||||||
|
|
|
@ -28,7 +28,7 @@
|
||||||
-export([remove/2, remove/3]).
|
-export([remove/2, remove/3]).
|
||||||
-export([tombstone/2]).
|
-export([tombstone/2]).
|
||||||
-export([reset/2, reset/3]).
|
-export([reset/2, reset/3]).
|
||||||
-export([dump_schema/2, reformat_schema_dump/1]).
|
-export([dump_schema/2, reformat_schema_dump/2]).
|
||||||
-export([schema_module/0]).
|
-export([schema_module/0]).
|
||||||
|
|
||||||
%% TODO: move to emqx_dashboard when we stop building api schema at build time
|
%% TODO: move to emqx_dashboard when we stop building api schema at build time
|
||||||
|
@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) ->
|
||||||
ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
|
ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
|
||||||
|
|
||||||
gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
|
gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
|
||||||
NestedStruct = reformat_schema_dump(StructsJsonArray),
|
NestedStruct = reformat_schema_dump(StructsJsonArray, Lang),
|
||||||
%% write to files
|
%% write to files
|
||||||
NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]),
|
NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]),
|
||||||
io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
|
io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
|
||||||
|
@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
%% @doc This function is exported for scripts/schema-dump-reformat.escript
|
%% @doc This function is exported for scripts/schema-dump-reformat.escript
|
||||||
reformat_schema_dump(StructsJsonArray0) ->
|
reformat_schema_dump(StructsJsonArray0, Lang) ->
|
||||||
%% prepare
|
%% prepare
|
||||||
|
DescResolver = make_desc_resolver(Lang),
|
||||||
StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
|
StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
|
||||||
#{fields := RootFields} = hd(StructsJsonArray),
|
#{fields := RootFields} = hd(StructsJsonArray),
|
||||||
RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
|
RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
|
||||||
RootNames = lists:map(fun to_bin/1, RootNames0),
|
RootNames = lists:map(fun to_bin/1, RootNames0),
|
||||||
%% reformat
|
%% reformat
|
||||||
[Root | FlatStructs0] = lists:map(
|
[Root | FlatStructs0] = lists:map(
|
||||||
fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray
|
fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end,
|
||||||
|
StructsJsonArray
|
||||||
),
|
),
|
||||||
FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
|
FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
|
||||||
gen_nested_doc(FlatStructs).
|
gen_nested_doc(FlatStructs).
|
||||||
|
@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) ->
|
||||||
|
|
||||||
%% generate flat docs for each struct.
|
%% generate flat docs for each struct.
|
||||||
%% using references to link to other structs.
|
%% using references to link to other structs.
|
||||||
gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
|
gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) ->
|
||||||
ShortName = short_name(FullName),
|
ShortName = short_name(FullName),
|
||||||
case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
|
case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
|
||||||
true ->
|
true ->
|
||||||
|
@ -314,18 +316,19 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
|
||||||
text => short_name(FullName),
|
text => short_name(FullName),
|
||||||
hash => format_hash(FullName),
|
hash => format_hash(FullName),
|
||||||
doc => maps:get(desc, S, <<"">>),
|
doc => maps:get(desc, S, <<"">>),
|
||||||
fields => format_fields(Fields)
|
fields => format_fields(Fields, DescResolver)
|
||||||
}.
|
}.
|
||||||
|
|
||||||
format_fields([]) ->
|
format_fields(Fields, DescResolver) ->
|
||||||
[];
|
[format_field(F, DescResolver) || F <- Fields].
|
||||||
format_fields([Field | Fields]) ->
|
|
||||||
[format_field(Field) | format_fields(Fields)].
|
|
||||||
|
|
||||||
format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
|
format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) ->
|
||||||
|
TypeDoc = format_type_desc(Type, DescResolver),
|
||||||
L = [
|
L = [
|
||||||
{text, Name},
|
{text, Name},
|
||||||
{type, format_type(Type)},
|
{type, format_type(Type)},
|
||||||
|
%% TODO: Make it into a separate field.
|
||||||
|
%% {typedoc, format_type_desc(Type, DescResolver)},
|
||||||
{refs, format_refs(Type)},
|
{refs, format_refs(Type)},
|
||||||
{aliases,
|
{aliases,
|
||||||
case Aliases of
|
case Aliases of
|
||||||
|
@ -333,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
|
||||||
_ -> Aliases
|
_ -> Aliases
|
||||||
end},
|
end},
|
||||||
{default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
|
{default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
|
||||||
{doc, maps:get(desc, F, undefined)}
|
{doc, join_format([maps:get(desc, F, undefined), TypeDoc])}
|
||||||
],
|
],
|
||||||
maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
|
maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
|
||||||
|
|
||||||
|
@ -393,10 +396,26 @@ format_union_members([Member | Members], Acc) ->
|
||||||
NewAcc = [format_type(Member) | Acc],
|
NewAcc = [format_type(Member) | Acc],
|
||||||
format_union_members(Members, NewAcc).
|
format_union_members(Members, NewAcc).
|
||||||
|
|
||||||
|
format_type_desc(#{kind := primitive, name := Name}, DescResolver) ->
|
||||||
|
format_primitive_type_desc(Name, DescResolver);
|
||||||
|
format_type_desc(#{}, _DescResolver) ->
|
||||||
|
undefined.
|
||||||
|
|
||||||
format_primitive_type(TypeStr) ->
|
format_primitive_type(TypeStr) ->
|
||||||
Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr),
|
Spec = get_primitive_typespec(TypeStr),
|
||||||
to_bin(maps:get(type, Spec)).
|
to_bin(maps:get(type, Spec)).
|
||||||
|
|
||||||
|
format_primitive_type_desc(TypeStr, DescResolver) ->
|
||||||
|
case get_primitive_typespec(TypeStr) of
|
||||||
|
#{desc := Desc} ->
|
||||||
|
DescResolver(Desc);
|
||||||
|
#{} ->
|
||||||
|
undefined
|
||||||
|
end.
|
||||||
|
|
||||||
|
get_primitive_typespec(TypeStr) ->
|
||||||
|
emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr).
|
||||||
|
|
||||||
%% All types should have a namespace to avlid name clashing.
|
%% All types should have a namespace to avlid name clashing.
|
||||||
is_missing_namespace(ShortName, FullName, RootNames) ->
|
is_missing_namespace(ShortName, FullName, RootNames) ->
|
||||||
case lists:member(ShortName, RootNames) of
|
case lists:member(ShortName, RootNames) of
|
||||||
|
@ -560,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
|
||||||
typename_to_spec(TypeStr, Module) ->
|
typename_to_spec(TypeStr, Module) ->
|
||||||
emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
|
emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
|
||||||
|
|
||||||
|
join_format(Snippets) ->
|
||||||
|
case [S || S <- Snippets, S =/= undefined] of
|
||||||
|
[] ->
|
||||||
|
undefined;
|
||||||
|
NonEmpty ->
|
||||||
|
to_bin(lists:join("<br/>", NonEmpty))
|
||||||
|
end.
|
||||||
|
|
||||||
to_bin(List) when is_list(List) -> iolist_to_binary(List);
|
to_bin(List) when is_list(List) -> iolist_to_binary(List);
|
||||||
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
|
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
|
||||||
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
|
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
|
||||||
|
|
|
@ -286,9 +286,16 @@ update_config_cluster(
|
||||||
check_res(Key, emqx_authn:merge_config(Conf), Conf, Opts);
|
check_res(Key, emqx_authn:merge_config(Conf), Conf, Opts);
|
||||||
update_config_cluster(Key, NewConf, #{mode := merge} = Opts) ->
|
update_config_cluster(Key, NewConf, #{mode := merge} = Opts) ->
|
||||||
Merged = merge_conf(Key, NewConf),
|
Merged = merge_conf(Key, NewConf),
|
||||||
check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Opts);
|
Request = make_request(Key, Merged),
|
||||||
|
check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), NewConf, Opts);
|
||||||
update_config_cluster(Key, Value, #{mode := replace} = Opts) ->
|
update_config_cluster(Key, Value, #{mode := replace} = Opts) ->
|
||||||
check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Opts).
|
Request = make_request(Key, Value),
|
||||||
|
check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), Value, Opts).
|
||||||
|
|
||||||
|
make_request(Key, Value) when Key =:= <<"connectors">> orelse Key =:= <<"actions">> ->
|
||||||
|
{force_update, Value};
|
||||||
|
make_request(_Key, Value) ->
|
||||||
|
Value.
|
||||||
|
|
||||||
-define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}).
|
-define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}).
|
||||||
update_config_local(
|
update_config_local(
|
||||||
|
@ -305,9 +312,11 @@ update_config_local(
|
||||||
check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Opts);
|
check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Opts);
|
||||||
update_config_local(Key, NewConf, #{mode := merge} = Opts) ->
|
update_config_local(Key, NewConf, #{mode := merge} = Opts) ->
|
||||||
Merged = merge_conf(Key, NewConf),
|
Merged = merge_conf(Key, NewConf),
|
||||||
check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Opts);
|
Request = make_request(Key, Merged),
|
||||||
|
check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), NewConf, Opts);
|
||||||
update_config_local(Key, Value, #{mode := replace} = Opts) ->
|
update_config_local(Key, Value, #{mode := replace} = Opts) ->
|
||||||
check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Opts).
|
Request = make_request(Key, Value),
|
||||||
|
check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), Value, Opts).
|
||||||
|
|
||||||
check_res(Key, Res, Conf, Opts) -> check_res(cluster, Key, Res, Conf, Opts).
|
check_res(Key, Res, Conf, Opts) -> check_res(cluster, Key, Res, Conf, Opts).
|
||||||
check_res(Node, Key, {ok, _}, _Conf, Opts) ->
|
check_res(Node, Key, {ok, _}, _Conf, Opts) ->
|
||||||
|
|
|
@ -16,6 +16,8 @@
|
||||||
|
|
||||||
-module(emqx_conf_schema_types).
|
-module(emqx_conf_schema_types).
|
||||||
|
|
||||||
|
-include_lib("hocon/include/hocon_types.hrl").
|
||||||
|
|
||||||
-export([readable/2]).
|
-export([readable/2]).
|
||||||
-export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
|
-export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
|
||||||
|
|
||||||
|
@ -165,37 +167,37 @@ readable("duration()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"12m">>},
|
swagger => #{type => string, example => <<"12m">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"12m">>}
|
docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("duration_s()") ->
|
readable("duration_s()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"1h">>},
|
swagger => #{type => string, example => <<"1h">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"1h">>}
|
docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("duration_ms()") ->
|
readable("duration_ms()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"32s">>},
|
swagger => #{type => string, example => <<"32s">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"32s">>}
|
docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("timeout_duration()") ->
|
readable("timeout_duration()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"12m">>},
|
swagger => #{type => string, example => <<"12m">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"12m">>}
|
docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("timeout_duration_s()") ->
|
readable("timeout_duration_s()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"1h">>},
|
swagger => #{type => string, example => <<"1h">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"1h">>}
|
docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("timeout_duration_ms()") ->
|
readable("timeout_duration_ms()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"32s">>},
|
swagger => #{type => string, example => <<"32s">>},
|
||||||
dashboard => #{type => duration},
|
dashboard => #{type => duration},
|
||||||
docgen => #{type => "String", example => <<"32s">>}
|
docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
|
||||||
};
|
};
|
||||||
readable("percent()") ->
|
readable("percent()") ->
|
||||||
#{
|
#{
|
||||||
|
@ -219,13 +221,13 @@ readable("bytesize()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"32MB">>},
|
swagger => #{type => string, example => <<"32MB">>},
|
||||||
dashboard => #{type => 'byteSize'},
|
dashboard => #{type => 'byteSize'},
|
||||||
docgen => #{type => "String", example => <<"32MB">>}
|
docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)}
|
||||||
};
|
};
|
||||||
readable("wordsize()") ->
|
readable("wordsize()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"1024KB">>},
|
swagger => #{type => string, example => <<"1024KB">>},
|
||||||
dashboard => #{type => 'wordSize'},
|
dashboard => #{type => 'wordSize'},
|
||||||
docgen => #{type => "String", example => <<"1024KB">>}
|
docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)}
|
||||||
};
|
};
|
||||||
readable("map(" ++ Map) ->
|
readable("map(" ++ Map) ->
|
||||||
[$) | _MapArgs] = lists:reverse(Map),
|
[$) | _MapArgs] = lists:reverse(Map),
|
||||||
|
@ -287,7 +289,11 @@ readable("secret()") ->
|
||||||
#{
|
#{
|
||||||
swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>},
|
swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>},
|
||||||
dashboard => #{type => string},
|
dashboard => #{type => string},
|
||||||
docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}
|
docgen => #{
|
||||||
|
type => "Secret",
|
||||||
|
example => <<"R4ND0M/S∃CЯ∃T"/utf8>>,
|
||||||
|
desc => ?DESC(secret)
|
||||||
|
}
|
||||||
};
|
};
|
||||||
readable(TypeStr0) ->
|
readable(TypeStr0) ->
|
||||||
case string:split(TypeStr0, ":") of
|
case string:split(TypeStr0, ":") of
|
||||||
|
|
|
@ -107,6 +107,8 @@ config_key_path() ->
|
||||||
|
|
||||||
pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
|
pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
|
||||||
{ok, RawConf};
|
{ok, RawConf};
|
||||||
|
pre_config_update([?ROOT_KEY], {force_update, NewConf}, RawConf) ->
|
||||||
|
pre_config_update([?ROOT_KEY], NewConf, RawConf);
|
||||||
pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
|
pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
|
||||||
case multi_validate_connector_names(NewConf) of
|
case multi_validate_connector_names(NewConf) of
|
||||||
ok ->
|
ok ->
|
||||||
|
@ -135,23 +137,16 @@ pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) ->
|
||||||
operation_to_enable(disable) -> false;
|
operation_to_enable(disable) -> false;
|
||||||
operation_to_enable(enable) -> true.
|
operation_to_enable(enable) -> true.
|
||||||
|
|
||||||
|
post_config_update([?ROOT_KEY], {force_update, _}, NewConf, OldConf, _AppEnv) ->
|
||||||
|
#{added := Added, removed := Removed, changed := Updated} =
|
||||||
|
diff_confs(NewConf, OldConf),
|
||||||
|
perform_connector_changes(Removed, Added, Updated);
|
||||||
post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
|
post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
|
||||||
#{added := Added, removed := Removed, changed := Updated} =
|
#{added := Added, removed := Removed, changed := Updated} =
|
||||||
diff_confs(NewConf, OldConf),
|
diff_confs(NewConf, OldConf),
|
||||||
case ensure_no_channels(Removed) of
|
case ensure_no_channels(Removed) of
|
||||||
ok ->
|
ok ->
|
||||||
%% The config update will be failed if any task in `perform_connector_changes` failed.
|
perform_connector_changes(Removed, Added, Updated);
|
||||||
Result = perform_connector_changes([
|
|
||||||
#{action => fun emqx_connector_resource:remove/4, data => Removed},
|
|
||||||
#{
|
|
||||||
action => fun emqx_connector_resource:create/4,
|
|
||||||
data => Added,
|
|
||||||
on_exception_fn => fun emqx_connector_resource:remove/4
|
|
||||||
},
|
|
||||||
#{action => fun emqx_connector_resource:update/4, data => Updated}
|
|
||||||
]),
|
|
||||||
?tp(connector_post_config_update_done, #{}),
|
|
||||||
Result;
|
|
||||||
{error, Error} ->
|
{error, Error} ->
|
||||||
{error, Error}
|
{error, Error}
|
||||||
end;
|
end;
|
||||||
|
@ -175,6 +170,20 @@ post_config_update([?ROOT_KEY, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
|
||||||
?tp(connector_post_config_update_done, #{}),
|
?tp(connector_post_config_update_done, #{}),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
|
%% The config update will be failed if any task in `perform_connector_changes` failed.
|
||||||
|
perform_connector_changes(Removed, Added, Updated) ->
|
||||||
|
Result = perform_connector_changes([
|
||||||
|
#{action => fun emqx_connector_resource:remove/4, data => Removed},
|
||||||
|
#{
|
||||||
|
action => fun emqx_connector_resource:create/4,
|
||||||
|
data => Added,
|
||||||
|
on_exception_fn => fun emqx_connector_resource:remove/4
|
||||||
|
},
|
||||||
|
#{action => fun emqx_connector_resource:update/4, data => Updated}
|
||||||
|
]),
|
||||||
|
?tp(connector_post_config_update_done, #{}),
|
||||||
|
Result.
|
||||||
|
|
||||||
list() ->
|
list() ->
|
||||||
maps:fold(
|
maps:fold(
|
||||||
fun(Type, NameAndConf, Connectors) ->
|
fun(Type, NameAndConf, Connectors) ->
|
||||||
|
|
|
@ -356,10 +356,13 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) ->
|
||||||
case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of
|
case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of
|
||||||
ok ->
|
ok ->
|
||||||
{200};
|
{200};
|
||||||
{error, MsgList} ->
|
%% bad hocon format
|
||||||
|
{error, MsgList = [{_, _} | _]} ->
|
||||||
JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end,
|
JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end,
|
||||||
JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun),
|
JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun),
|
||||||
{400, #{<<"content-type">> => <<"text/plain">>}, JsonMap}
|
{400, #{<<"content-type">> => <<"text/plain">>}, JsonMap};
|
||||||
|
{error, Msg} ->
|
||||||
|
{400, #{<<"content-type">> => <<"text/plain">>}, Msg}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 ->
|
find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 ->
|
||||||
|
|
|
@ -378,8 +378,13 @@ t_get_configs_in_different_accept(_Config) ->
|
||||||
?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)).
|
?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)).
|
||||||
|
|
||||||
t_create_webhook_v1_bridges_api({'init', Config}) ->
|
t_create_webhook_v1_bridges_api({'init', Config}) ->
|
||||||
application:ensure_all_started(emqx_connector),
|
lists:foreach(
|
||||||
application:ensure_all_started(emqx_bridge),
|
fun(App) ->
|
||||||
|
_ = application:stop(App),
|
||||||
|
{ok, [App]} = application:ensure_all_started(App)
|
||||||
|
end,
|
||||||
|
[emqx_connector, emqx_bridge]
|
||||||
|
),
|
||||||
Config;
|
Config;
|
||||||
t_create_webhook_v1_bridges_api({'end', _}) ->
|
t_create_webhook_v1_bridges_api({'end', _}) ->
|
||||||
application:stop(emqx_bridge),
|
application:stop(emqx_bridge),
|
||||||
|
|
|
@ -0,0 +1,120 @@
|
||||||
|
# e5.4.0
|
||||||
|
|
||||||
|
## Breaking Changes
|
||||||
|
|
||||||
|
- [#11994](https://github.com/emqx/emqx/pull/11994) Stop releasing packages for Windows.
|
||||||
|
|
||||||
|
- [#11998](https://github.com/emqx/emqx/pull/11998) Stop releasing packages for MacOS 11 (BigSur).
|
||||||
|
|
||||||
|
- [#12112](https://github.com/emqx/emqx/pull/12112) Stop supporting UDP multicast based clustering strategy.
|
||||||
|
|
||||||
|
- [#10976](https://github.com/emqx/emqx/pull/10976) Fix topic-filter overlapping handling in shared subscription.
|
||||||
|
* Hook callback `session.subscribed` and `client.subscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from `subopts`.
|
||||||
|
* Hook callback `session.unsubscribed` and `client.unsubscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#` instead of just `topic1/#`.
|
||||||
|
* ExHook Proto changed. The `share` field in message `SubOpts` was deprecated.
|
||||||
|
ExHook Server will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from message `SubOpts`.
|
||||||
|
* `session.subscribed` and `session.unsubscribed` rule-engine events will have shared subscriptions in their full representation for `topic`, e.g. `$share/group1/topic1/#` instead of just `topic1/#`.
|
||||||
|
|
||||||
|
## Enhancements
|
||||||
|
|
||||||
|
- [#11884](https://github.com/emqx/emqx/pull/11884) Modified the Prometheus API and configuration to implement the following improvements:
|
||||||
|
|
||||||
|
- Restructured configuration sections to group related settings, improving readability and maintainability.
|
||||||
|
- Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security.
|
||||||
|
- Maintained backwards compatibility while refactoring code, avoiding breaking changes.
|
||||||
|
|
||||||
|
- [#11896](https://github.com/emqx/emqx/pull/11896) Introduced an enhancement for configuring sensitive authentication fields in bridges, such as passwords, tokens, and secret keys. This improvement allows the use of secrets stored as files in the file system. These secrets can be securely referenced in configuration files using the special `file://` prefix, enhancing the security of sensitive data handling in bridge configurations.
|
||||||
|
|
||||||
|
- [#11921](https://github.com/emqx/emqx/pull/11921) Introduced Open Telemetry Logs Handler that allows to format log events in alignment with the Open Telemetry log data model. This handler facilitates the exportation of formatted log events to a configured Open Telemetry collector or back-end, thereby enhancing log management and integration capabilities.
|
||||||
|
|
||||||
|
- [#11935](https://github.com/emqx/emqx/pull/11935) Switched to the new `v2` routing store schema by default. New schema improves both subscription and routing performance, especially in scenarios with concurrent subscriptions to topic filters sharing common wildcard prefixes. However, it does come with a minor increase in memory usage. This schema also eliminates the need for a separate index, thus inconsistencies in the routing state rarely encountered in previous versions should no longer be possible.
|
||||||
|
|
||||||
|
If a cluster is rolling upgraded from older version, the cluster will continue to use `v1` store until a full cluster (non-rolling) restart happens.
|
||||||
|
|
||||||
|
Users can still opt for the previous schema by configuring the `broker.routing.storage_schema` option to `v1`. However, this also requires a complete, non-rolling restart of the cluster to take effect.
|
||||||
|
|
||||||
|
- [#11984](https://github.com/emqx/emqx/pull/11984) Implemented Open Telemetry distributed tracing feature.
|
||||||
|
|
||||||
|
- [#12017](https://github.com/emqx/emqx/pull/12017) Implemented a dedicated HTTP API for the import and export of configuration and user data.
|
||||||
|
|
||||||
|
- [#12040](https://github.com/emqx/emqx/pull/12040) Upgraded QUIC protocol stack.
|
||||||
|
|
||||||
|
- [#11766](https://github.com/emqx/emqx/pull/11766) Implemented a preliminary Role-Based Access Control for the REST API. In this version, there are three predefined roles:
|
||||||
|
- Administrator: This role can access all resources.
|
||||||
|
- Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API.
|
||||||
|
- Publisher: Specifically tailored for MQTT message publishing, this role is confined to accessing endpoints related to message publication.
|
||||||
|
|
||||||
|
- [#12201](https://github.com/emqx/emqx/pull/11994) Support hot update of TCP/SSL/WS/WSS MQTT listeners configuration.
|
||||||
|
This allows changing most of the configuration parameters without restarting the listener and disconnecting the clients. The limitations are:
|
||||||
|
- For TCP/SSL listeners, changes to the following parameters still require listener restart and clients reconnect:
|
||||||
|
* `bind`
|
||||||
|
* `tcp_options.backlog`
|
||||||
|
- For WS/WSS (WebSocket) listeners, changing transport related parameters (listed below) will cause listening socket to be re-opened, but established connections will stay uninterrupted.
|
||||||
|
* `bind`
|
||||||
|
* `tcp_options.*`
|
||||||
|
* `ssl_options.*`
|
||||||
|
|
||||||
|
- [#11608](https://github.com/emqx/emqx/pull/11608) Integrated LDAP bind operation as a new authenticator, providing a more flexible and secure method for user authentication.
|
||||||
|
|
||||||
|
- [#11773](https://github.com/emqx/emqx/pull/11773) Implemented Dashboard support for audit log management. Users can utilize this page to view all change operations performed on EMQX devices and data, such as kicking out devices, creating/deleting rules, etc.
|
||||||
|
|
||||||
|
- [#11778](https://github.com/emqx/emqx/pull/11778) Integrated Microsoft Entra Identity (formerly known as Azure Active Directory) support into the SAML single sign-on (SSO) process.
|
||||||
|
|
||||||
|
|
||||||
|
- [#11811](https://github.com/emqx/emqx/pull/11811) Improved the format for the REST API key bootstrap file to support initializing key with a role.
|
||||||
|
|
||||||
|
The new form is:`api_key:api_secret:role`.
|
||||||
|
|
||||||
|
`role` is optional and its default value is `administrator`.
|
||||||
|
|
||||||
|
- [#11852](https://github.com/emqx/emqx/pull/11852) Introduced a new GB/T 32960 gateway, enabling vehicles to connect with EMQX via the GBT32960 vehicular networking protocol.
|
||||||
|
|
||||||
|
- [#11883](https://github.com/emqx/emqx/pull/11883) Introduced a new JT/T808 gateway, enabling vehicles to connect with EMQX via the JT/T 808 vehicular networking protocol.
|
||||||
|
|
||||||
|
- [#11885](https://github.com/emqx/emqx/pull/11885) Introduced a new OCPP gateway for Electric vehicle (EV) charging stations to access EMQX through the OCPP (Open Charge Point Protocol).
|
||||||
|
|
||||||
|
- [#11971](https://github.com/emqx/emqx/pull/11971) Made `/api/v5/load_rebalance/availability_check` public, meaning it no longer requires authentication. This change simplifies the setup of load balancers.
|
||||||
|
|
||||||
|
It improved the gracefulness of the rebalance/evacuation process during the wait health check phase. The connections to nodes marked for eviction are now not prohibited during this phase.
|
||||||
|
During this phase it is unknown whether these nodes are all marked unhealthy by the load balancer, so prohibiting connections to them may cause multiple unsuccessful reconnection attempts.
|
||||||
|
|
||||||
|
- [#12013](https://github.com/emqx/emqx/pull/12013) The data bridging design has been adjusted to split it into connectors and actions (Sinks). Connectors are used to manage the integration of data with external systems and can be reused across multiple actions, while actions are used to configure how data is processed. This design provides greater flexibility and scalability, resulting in clearer data integration configuration and management.
|
||||||
|
|
||||||
|
The adjusted data bridges includes PostgreSQL, Timescale, and Matrix, which have now been split into connectors and actions APIs, but they remain backward compatible with the old data bridge API.
|
||||||
|
|
||||||
|
- [#12016](https://github.com/emqx/emqx/pull/12016) Enhanced license key management.
|
||||||
|
|
||||||
|
EMQX can now load the license key from a specified file. This is enabled by setting the `license.key` configuration to a file path, which should be prefixed with `"file://"`.
|
||||||
|
Also added the ability to revert to the default trial license by setting `license.key = default`. This option simplifies the process of returning to the trial license if needed.
|
||||||
|
|
||||||
|
- [#12129](https://github.com/emqx/emqx/pull/12129) Default license renewal. Replaced old license issued in Jan 2023. New license supports up to 25 concurrent connections.
|
||||||
|
|
||||||
|
### Bug Fixes
|
||||||
|
|
||||||
|
- [#10976](https://github.com/emqx/emqx/pull/10976) Fixed topic-filter overlapping handling in shared subscription.
|
||||||
|
In the previous implementation, the storage method for subscription options did not provide adequate support for shared subscriptions. This resulted in message routing failures and leakage of routing tables between nodes during the "subscribe-unsubscribe" process with specific order and topics.
|
||||||
|
|
||||||
|
- [#12048](https://github.com/emqx/emqx/pull/12048) Fixed COAP gateway bug that caused it to ignore subscription options.
|
||||||
|
|
||||||
|
- [#12078](https://github.com/emqx/emqx/pull/12078) Upgraded grpc-erl to 0.6.12. This update addresses a potential deadlock issue where the grpc client started dependent apps lazily.
|
||||||
|
|
||||||
|
- [#12081](https://github.com/emqx/emqx/pull/12081) Updated `gen_rpc` library to version 3.3.1. The new version includes several performance improvements:
|
||||||
|
|
||||||
|
- Avoiding allocating extra memory for the packets before they are sent to the wire in some cases.
|
||||||
|
|
||||||
|
- Bypassing network for the local calls.
|
||||||
|
|
||||||
|
- Avoid senstive data leaking in debug logs [#12202](https://github.com/emqx/emqx/pull/12202)
|
||||||
|
|
||||||
|
- [#12111](https://github.com/emqx/emqx/pull/12111) Fixed an issue when API tokens were sometimes unavailable immediately after login due to race condition.
|
||||||
|
|
||||||
|
- [#12121](https://github.com/emqx/emqx/pull/12121) Fixed an issue where nodes in the cluster would occasionally return a stale view when updating configurations on different nodes concurrently.
|
||||||
|
|
||||||
|
- [#12158](https://github.com/emqx/emqx/pull/12158) Fixed an issue when the rule engine cannot connect to Redis hosted by Upstash.
|
||||||
|
|
||||||
|
Before the fix, after establishing a TCP connection with the Redis service, the Redis driver of EMQX used [Inline Commands](https://redis.io/docs/reference/protocol-spec/#inline-commands) to send AUTH and SELECT commands. However, the `upstash` Redis service does not support Inline Commands, which causes the rule engine to fail to connect to the `upstash` Redis service.
|
||||||
|
After the fix, the Redis driver of EMQX uses RESP (REdis Serialization Protocol) to send AUTH and SELECT commands.
|
||||||
|
|
||||||
|
- [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before.
|
||||||
|
|
||||||
|
- [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation.
|
|
@ -0,0 +1,82 @@
|
||||||
|
# v5.4.0
|
||||||
|
|
||||||
|
## Breaking Changes
|
||||||
|
|
||||||
|
- [#11994](https://github.com/emqx/emqx/pull/11994) Stop releasing packages for Windows.
|
||||||
|
|
||||||
|
- [#11998](https://github.com/emqx/emqx/pull/11998) Stop releasing packages for MacOS 11 (BigSur).
|
||||||
|
|
||||||
|
- [#12112](https://github.com/emqx/emqx/pull/12112) Stop supporting UDP multicast based clustering strategy.
|
||||||
|
|
||||||
|
- [#10976](https://github.com/emqx/emqx/pull/10976) Fix topic-filter overlapping handling in shared subscription.
|
||||||
|
* Hook callback `session.subscribed` and `client.subscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from `subopts`.
|
||||||
|
* Hook callback `session.unsubscribed` and `client.unsubscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#` instead of just `topic1/#`.
|
||||||
|
* ExHook Proto changed. The `share` field in message `SubOpts` was deprecated.
|
||||||
|
ExHook Server will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from message `SubOpts`.
|
||||||
|
* `session.subscribed` and `session.unsubscribed` rule-engine events will have shared subscriptions in their full representation for `topic`, e.g. `$share/group1/topic1/#` instead of just `topic1/#`.
|
||||||
|
|
||||||
|
## Enhancements
|
||||||
|
|
||||||
|
- [#11884](https://github.com/emqx/emqx/pull/11884) Modified the Prometheus API and configuration to implement the following improvements:
|
||||||
|
|
||||||
|
- Restructured configuration sections to group related settings, improving readability and maintainability.
|
||||||
|
- Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security.
|
||||||
|
- Maintained backwards compatibility while refactoring code, avoiding breaking changes.
|
||||||
|
|
||||||
|
- [#11896](https://github.com/emqx/emqx/pull/11896) Introduced an enhancement for configuring sensitive authentication fields in bridges, such as passwords, tokens, and secret keys. This improvement allows the use of secrets stored as files in the file system. These secrets can be securely referenced in configuration files using the special `file://` prefix, enhancing the security of sensitive data handling in bridge configurations.
|
||||||
|
|
||||||
|
- [#11921](https://github.com/emqx/emqx/pull/11921) Introduced Open Telemetry Logs Handler that allows to format log events in alignment with the Open Telemetry log data model. This handler facilitates the exportation of formatted log events to a configured Open Telemetry collector or back-end, thereby enhancing log management and integration capabilities.
|
||||||
|
|
||||||
|
- [#11935](https://github.com/emqx/emqx/pull/11935) Switched to the new `v2` routing store schema by default. New schema improves both subscription and routing performance, especially in scenarios with concurrent subscriptions to topic filters sharing common wildcard prefixes. However, it does come with a minor increase in memory usage. This schema also eliminates the need for a separate index, thus inconsistencies in the routing state rarely encountered in previous versions should no longer be possible.
|
||||||
|
|
||||||
|
If a cluster is rolling upgraded from older version, the cluster will continue to use `v1` store until a full cluster (non-rolling) restart happens.
|
||||||
|
|
||||||
|
Users can still opt for the previous schema by configuring the `broker.routing.storage_schema` option to `v1`. However, this also requires a complete, non-rolling restart of the cluster to take effect.
|
||||||
|
|
||||||
|
- [#11984](https://github.com/emqx/emqx/pull/11984) Implemented Open Telemetry distributed tracing feature.
|
||||||
|
|
||||||
|
- [#12017](https://github.com/emqx/emqx/pull/12017) Implemented a dedicated HTTP API for the import and export of configuration and user data.
|
||||||
|
|
||||||
|
- [#12040](https://github.com/emqx/emqx/pull/12040) Upgraded QUIC protocol stack.
|
||||||
|
|
||||||
|
- [#11766](https://github.com/emqx/emqx/pull/11766) Implemented a preliminary Role-Based Access Control for the REST API. In this version, there are three predefined roles:
|
||||||
|
- Administrator: This role can access all resources.
|
||||||
|
- Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API.
|
||||||
|
- Publisher: Specifically tailored for MQTT message publishing, this role is confined to accessing endpoints related to message publication.
|
||||||
|
|
||||||
|
- [#12201](https://github.com/emqx/emqx/pull/11994) Support hot update of TCP/SSL/WS/WSS MQTT listeners configuration.
|
||||||
|
This allows changing most of the configuration parameters without restarting the listener and disconnecting the clients. The limitations are:
|
||||||
|
- For TCP/SSL listeners, changes to the following parameters still require listener restart and clients reconnect:
|
||||||
|
* `bind`
|
||||||
|
* `tcp_options.backlog`
|
||||||
|
- For WS/WSS (WebSocket) listeners, changing transport related parameters (listed below) will cause listening socket to be re-opened, but established connections will stay uninterrupted.
|
||||||
|
* `bind`
|
||||||
|
* `tcp_options.*`
|
||||||
|
* `ssl_options.*`
|
||||||
|
|
||||||
|
## Bug Fixes
|
||||||
|
|
||||||
|
- [#12048](https://github.com/emqx/emqx/pull/12048) Fixed COAP gateway bug that caused it to ignore subscription options.
|
||||||
|
|
||||||
|
- [#12078](https://github.com/emqx/emqx/pull/12078) Upgraded grpc-erl to 0.6.12. This update addresses a potential deadlock issue where the grpc client started dependent apps lazily.
|
||||||
|
|
||||||
|
- [#12081](https://github.com/emqx/emqx/pull/12081) Updated `gen_rpc` library to version 3.3.1. The new version includes several performance improvements:
|
||||||
|
|
||||||
|
- Avoiding allocating extra memory for the packets before they are sent to the wire in some cases.
|
||||||
|
|
||||||
|
- Bypassing network for the local calls.
|
||||||
|
|
||||||
|
- Avoid senstive data leaking in debug logs [#12202](https://github.com/emqx/emqx/pull/12202)
|
||||||
|
|
||||||
|
- [#12111](https://github.com/emqx/emqx/pull/12111) Fixed an issue when API tokens were sometimes unavailable immediately after login due to race condition.
|
||||||
|
|
||||||
|
- [#12121](https://github.com/emqx/emqx/pull/12121) Fixed an issue where nodes in the cluster would occasionally return a stale view when updating configurations on different nodes concurrently.
|
||||||
|
|
||||||
|
- [#12158](https://github.com/emqx/emqx/pull/12158) Fixed an issue when the rule engine cannot connect to Redis hosted by Upstash.
|
||||||
|
|
||||||
|
Before the fix, after establishing a TCP connection with the Redis service, the Redis driver of EMQX used [Inline Commands](https://redis.io/docs/reference/protocol-spec/#inline-commands) to send AUTH and SELECT commands. However, the `upstash` Redis service does not support Inline Commands, which causes the rule engine to fail to connect to the `upstash` Redis service.
|
||||||
|
After the fix, the Redis driver of EMQX uses RESP (REdis Serialization Protocol) to send AUTH and SELECT commands.
|
||||||
|
|
||||||
|
- [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before.
|
||||||
|
|
||||||
|
- [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation.
|
|
@ -14,8 +14,8 @@ type: application
|
||||||
|
|
||||||
# This is the chart version. This version number should be incremented each time you make changes
|
# This is the chart version. This version number should be incremented each time you make changes
|
||||||
# to the chart and its templates, including the app version.
|
# to the chart and its templates, including the app version.
|
||||||
version: 5.4.0-rc.1
|
version: 5.4.0
|
||||||
|
|
||||||
# This is the version number of the application being deployed. This version number should be
|
# This is the version number of the application being deployed. This version number should be
|
||||||
# incremented each time you make changes to the application.
|
# incremented each time you make changes to the application.
|
||||||
appVersion: 5.4.0-rc.1
|
appVersion: 5.4.0
|
||||||
|
|
|
@ -14,8 +14,8 @@ type: application
|
||||||
|
|
||||||
# This is the chart version. This version number should be incremented each time you make changes
|
# This is the chart version. This version number should be incremented each time you make changes
|
||||||
# to the chart and its templates, including the app version.
|
# to the chart and its templates, including the app version.
|
||||||
version: 5.4.0-rc.1
|
version: 5.4.0
|
||||||
|
|
||||||
# This is the version number of the application being deployed. This version number should be
|
# This is the version number of the application being deployed. This version number should be
|
||||||
# incremented each time you make changes to the application.
|
# incremented each time you make changes to the application.
|
||||||
appVersion: 5.4.0-rc.1
|
appVersion: 5.4.0
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
emqx_conf_schema_types {
|
||||||
|
|
||||||
|
duration.desc:
|
||||||
|
"""A string that represents a time duration, for example: <code>10s</code>, <code>2.5m</code>, <code>1h30m</code>, <code>1W2D</code>, or <code>2345ms</code>, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing <code>1200ms</code> for <code>Duration(s)</code> is equivalent to writing <code>1s</code>. The unit part is case-insensitive."""
|
||||||
|
|
||||||
|
bytesize.desc:
|
||||||
|
"""A string that represents a number of bytes, for example: <code>10B</code>, <code>640kb</code>, <code>4MB</code>, <code>1GB</code>. Units are interpreted as powers of 1024, and the unit part is case-insensitive."""
|
||||||
|
|
||||||
|
secret.desc:
|
||||||
|
"""A string holding some sensitive information, such as a password. When secret starts with <code>file://</code>, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using <code>file://</code> secrets."""
|
||||||
|
|
||||||
|
}
|
|
@ -1,132 +0,0 @@
|
||||||
#!/usr/bin/env escript
|
|
||||||
|
|
||||||
%% This script translates the hocon_schema_json's schema dump to a new format.
|
|
||||||
%% It is used to convert older version EMQX's schema dumps to the new format
|
|
||||||
%% after all files are upgraded to the new format, this script can be removed.
|
|
||||||
|
|
||||||
-mode(compile).
|
|
||||||
|
|
||||||
main([Input]) ->
|
|
||||||
ok = add_libs(),
|
|
||||||
_ = atoms(),
|
|
||||||
{ok, Data} = file:read_file(Input),
|
|
||||||
Json = jsx:decode(Data),
|
|
||||||
NewJson = reformat(Json),
|
|
||||||
io:format("~s~n", [jsx:encode(NewJson)]);
|
|
||||||
main(_) ->
|
|
||||||
io:format("Usage: schema-dump-reformat.escript <input.json>~n"),
|
|
||||||
halt(1).
|
|
||||||
|
|
||||||
reformat(Json) ->
|
|
||||||
emqx_conf:reformat_schema_dump(fix(Json)).
|
|
||||||
|
|
||||||
%% fix old type specs to make them compatible with new type specs
|
|
||||||
fix(#{
|
|
||||||
<<"kind">> := <<"union">>,
|
|
||||||
<<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}]
|
|
||||||
}) ->
|
|
||||||
%% s3_exporter.secret_access_key
|
|
||||||
#{
|
|
||||||
kind => primitive,
|
|
||||||
name => <<"string()">>
|
|
||||||
};
|
|
||||||
fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) ->
|
|
||||||
#{
|
|
||||||
kind => enum,
|
|
||||||
symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all]
|
|
||||||
};
|
|
||||||
fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) ->
|
|
||||||
#{kind => enum, symbols => [random, hash]};
|
|
||||||
fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) ->
|
|
||||||
#{kind => enum, symbols => [random, hash]};
|
|
||||||
fix(Map) when is_map(Map) ->
|
|
||||||
maps:from_list(fix(maps:to_list(Map)));
|
|
||||||
fix(List) when is_list(List) ->
|
|
||||||
lists:map(fun fix/1, List);
|
|
||||||
fix({<<"kind">>, Kind}) ->
|
|
||||||
{kind, binary_to_atom(Kind, utf8)};
|
|
||||||
fix({<<"name">>, Type}) ->
|
|
||||||
{name, fix_type(Type)};
|
|
||||||
fix({K, V}) ->
|
|
||||||
{binary_to_atom(K, utf8), fix(V)};
|
|
||||||
fix(V) when is_number(V) ->
|
|
||||||
V;
|
|
||||||
fix(V) when is_atom(V) ->
|
|
||||||
V;
|
|
||||||
fix(V) when is_binary(V) ->
|
|
||||||
V.
|
|
||||||
|
|
||||||
%% ensure below ebin dirs are added to code path:
|
|
||||||
%% _build/default/lib/*/ebin
|
|
||||||
%% _build/emqx/lib/*/ebin
|
|
||||||
%% _build/emqx-enterprise/lib/*/ebin
|
|
||||||
add_libs() ->
|
|
||||||
Profile = os:getenv("PROFILE"),
|
|
||||||
case Profile of
|
|
||||||
"emqx" ->
|
|
||||||
ok;
|
|
||||||
"emqx-enterprise" ->
|
|
||||||
ok;
|
|
||||||
_ ->
|
|
||||||
io:format("PROFILE is not set~n"),
|
|
||||||
halt(1)
|
|
||||||
end,
|
|
||||||
Dirs =
|
|
||||||
filelib:wildcard("_build/default/lib/*/ebin") ++
|
|
||||||
filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"),
|
|
||||||
lists:foreach(fun add_lib/1, Dirs).
|
|
||||||
|
|
||||||
add_lib(Dir) ->
|
|
||||||
code:add_patha(Dir),
|
|
||||||
Beams = filelib:wildcard(Dir ++ "/*.beam"),
|
|
||||||
_ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
load_beam(BeamFile) ->
|
|
||||||
ModuleName = filename:basename(BeamFile, ".beam"),
|
|
||||||
Module = list_to_atom(ModuleName),
|
|
||||||
%% load the beams to make sure the atoms are existing
|
|
||||||
code:ensure_loaded(Module),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
fix_type(<<"[{string(), string()}]">>) ->
|
|
||||||
<<"map()">>;
|
|
||||||
fix_type(<<"[{binary(), binary()}]">>) ->
|
|
||||||
<<"map()">>;
|
|
||||||
fix_type(<<"emqx_limiter_schema:rate()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_limiter_schema:burst_rate()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_limiter_schema:capacity()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_limiter_schema:initial()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_limiter_schema:failure_strategy()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_conf_schema:file()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"#{term() => binary()}">>) ->
|
|
||||||
<<"map()">>;
|
|
||||||
fix_type(<<"[term()]">>) ->
|
|
||||||
%% jwt claims
|
|
||||||
<<"map()">>;
|
|
||||||
fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) ->
|
|
||||||
<<"non_neg_integer()">>;
|
|
||||||
fix_type(<<"emqx_s3_schema:secret_access_key()">>) ->
|
|
||||||
<<"string()">>;
|
|
||||||
fix_type(Type) ->
|
|
||||||
Type.
|
|
||||||
|
|
||||||
%% ensure atoms are loaded
|
|
||||||
%% these atoms are from older version of emqx
|
|
||||||
atoms() ->
|
|
||||||
[
|
|
||||||
emqx_ee_connector_clickhouse,
|
|
||||||
emqx_ee_bridge_gcp_pubsub,
|
|
||||||
emqx_ee_bridge_influxdb,
|
|
||||||
emqx_connector_http
|
|
||||||
].
|
|
Loading…
Reference in New Issue