yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10974 from keynslug/ft/EMQX-9523/backend-enable-flag 

fix(ft): add `enable` flag to every backend
This commit is contained in:
Ilya Averyanov 2023-06-09 11:44:37 +03:00 committed by GitHub
parent 2e9ac8f1ec ce8cc05cc8
commit c2635e938b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 75 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/emqx_ft/README.md
View File

@ -29,6 +29,7 @@ file_transfer {
enable = true
storage {
local {
enable = true
exporter {
local { root = "/var/lib/emqx/transfers" }
}
@ -50,7 +51,9 @@ file_transfer {
enable = true
storage {
local {
enable = true
exporter {
enable = true
s3 {
host = "s3.us-east-1.amazonaws.com"
port = "443"

56
apps/emqx_ft/src/emqx_ft_schema.erl
View File

@ -25,6 +25,10 @@
-export([schema/1]).
%% Utilities
-export([backend/1]).
%% Test-only helpers
-export([translate/1]).
-type json_value() ::
@ -142,7 +146,7 @@ fields(local_storage) ->
}
}
)}
];
] ++ common_backend_fields();
fields(local_storage_segments) ->
[
{root,
@ -190,9 +194,9 @@ fields(local_storage_exporter) ->
required => false
}
)}
];
] ++ common_backend_fields();
fields(s3_exporter) ->
emqx_s3_schema:fields(s3);
emqx_s3_schema:fields(s3) ++ common_backend_fields();
fields(local_storage_segments_gc) ->
[
{interval,
@ -229,6 +233,18 @@ fields(local_storage_segments_gc) ->
)}
].
common_backend_fields() ->
[
{enable,
mk(
boolean(), #{
desc => ?DESC("backend_enable"),
required => false,
default => true
}
)}
].
desc(file_transfer) ->
"File transfer settings";
desc(local_storage) ->
@ -275,11 +291,14 @@ validator(filename) ->
];
validator(backend) ->
fun(Config) ->
case maps:keys(Config) of
[_Type] ->
Enabled = maps:filter(fun(_, #{<<"enable">> := E}) -> E end, Config),
case maps:to_list(Enabled) of
[{_Type, _BackendConfig}] ->
ok;
_Conflicts = [_ | _] ->
{error, multiple_conflicting_backends}
{error, multiple_enabled_backends};
_None = [] ->
{error, no_enabled_backend}
end
end.
@ -311,11 +330,24 @@ converter(unicode_string) ->
ref(Ref) ->
ref(?MODULE, Ref).
%% Utilities
-spec backend(emqx_config:config()) ->
{_Type :: atom(), emqx_config:config()}.
backend(Config) ->
catch maps:foreach(fun emit_enabled/2, Config).
-spec emit_enabled(atom(), emqx_config:config()) ->
no_return().
emit_enabled(Type, BConf = #{enable := Enabled}) ->
Enabled andalso throw({Type, BConf}).
%% Test-only helpers
-spec translate(emqx_config:raw_config()) ->
emqx_config:config().
translate(Conf) ->
[Root] = roots(),
maps:get(
Root,
hocon_tconf:check_plain(
?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root]
)
).
RootRaw = atom_to_binary(Root),
ConfChecked = hocon_tconf:check_plain(?MODULE, #{RootRaw => Conf}, #{}, [Root]),
emqx_utils_maps:unsafe_atom_key_map(maps:get(RootRaw, ConfChecked)).

4
apps/emqx_ft/src/emqx_ft_storage.erl
View File

@ -182,8 +182,8 @@ on_backend_update(BackendOld, BackendNew) when
%%--------------------------------------------------------------------
-spec backend(config()) -> backend().
backend(#{local := Storage}) ->
{local, Storage}.
backend(Config) ->
emqx_ft_schema:backend(Config).
on_storage_start({Type, Storage}) ->
(mod(Type)):start(Storage).

6
apps/emqx_ft/src/emqx_ft_storage_exporter.erl
View File

@ -175,10 +175,10 @@ stop({ExporterMod, ExporterOpts}) ->
%%------------------------------------------------------------------------------
exporter(Storage) ->
case maps:get(exporter, Storage) of
#{local := Options} ->
case emqx_ft_schema:backend(maps:get(exporter, Storage)) of
{local, Options} ->
{emqx_ft_storage_exporter_fs, Options};
#{s3 := Options} ->
{s3, Options} ->
{emqx_ft_storage_exporter_s3, Options}
end.

1
apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl
View File

@ -46,6 +46,7 @@
-export_type([options/0]).
-type options() :: #{
enable := true,
root => file:name(),
_ => _
}.

1
apps/emqx_ft/src/emqx_ft_storage_fs.erl
View File

@ -102,6 +102,7 @@
-type storage() :: #{
type := 'local',
enable := true,
segments := segments(),
exporter := emqx_ft_storage_exporter:exporter()
}.

1
apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl
View File

@ -256,6 +256,7 @@ storage(Config) ->
},
<<"exporter">> => #{
<<"local">> => #{
<<"enable">> => true,
<<"root">> => ?config(exports_root, Config)
}
}

9
apps/emqx_ft/test/emqx_ft_conf_SUITE.erl
View File

@ -77,6 +77,7 @@ t_update_config(_Config) ->
},
<<"exporter">> => #{
<<"local">> => #{
<<"enable">> => true,
<<"root">> => <<"/tmp/exports">>
}
}
@ -104,7 +105,10 @@ t_disable_restore_config(Config) ->
{ok, _},
emqx_conf:update(
[file_transfer],
#{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}},
#{
<<"enable">> => true,
<<"storage">> => #{<<"local">> => #{}}
},
#{}
)
),
@ -207,6 +211,7 @@ t_switch_exporter(_Config) ->
[file_transfer, storage, local, exporter],
#{
<<"s3">> => #{
<<"enable">> => true,
<<"bucket">> => <<"emqx">>,
<<"host">> => <<"https://localhost">>,
<<"port">> => 9000,
@ -234,7 +239,7 @@ t_switch_exporter(_Config) ->
{ok, _},
emqx_conf:update(
[file_transfer, storage, local, exporter],
#{<<"local">> => #{}},
#{<<"local">> => #{<<"enable">> => true}},
#{}
)
),

3
apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl
View File

@ -45,9 +45,10 @@ init_per_testcase(TC, Config) ->
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"enable">> => true,
<<"segments">> => #{<<"root">> => SegmentsRoot},
<<"exporter">> => #{
<<"local">> => #{<<"root">> => ExportsRoot}
<<"local">> => #{<<"enable">> => true, <<"root">> => ExportsRoot}
}
}
}

9
apps/emqx_ft/test/emqx_ft_test_helpers.erl
View File

@ -55,17 +55,24 @@ local_storage(Config) ->
local_storage(Config, Opts) ->
#{
<<"local">> => #{
<<"enable">> => true,
<<"segments">> => #{<<"root">> => root(Config, node(), [segments])},
<<"exporter">> => exporter(Config, Opts)
}
}.
exporter(Config, #{exporter := local}) ->
#{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}};
#{
<<"local">> => #{
<<"enable">> => true,
<<"root">> => root(Config, node(), [exports])
}
};
exporter(_Config, #{exporter := s3, bucket_name := BucketName}) ->
BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp),
#{
<<"s3">> => BaseConfig#{
<<"enable">> => true,
<<"bucket">> => list_to_binary(BucketName),
<<"host">> => ?S3_HOST,
<<"port">> => ?S3_PORT

3
rel/i18n/emqx_ft_schema.hocon
View File

@ -18,6 +18,9 @@ store_segment_timeout.desc:
"""Timeout for storing a file segment.<br/>
After reaching the timeout, message with the segment will be acked with an error"""
backend_enable.desc:
"""Whether to enable this backend."""
storage_backend.desc:
"""Storage settings for file transfer."""