diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index 75fb47c27..df9135816 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -17,12 +17,14 @@ -module(emqx_ft_fs_util). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("kernel/include/file.hrl"). -export([is_filename_safe/1]). -export([escape_filename/1]). -export([unescape_filename/1]). -export([read_decode_file/2]). +-export([read_info/1]). -export([fold/4]). @@ -144,13 +146,20 @@ safe_decode(Content, DecodeFun) -> {error, corrupted} end. +-spec read_info(file:name_all()) -> + {ok, file:file_info()} | {error, file:posix() | badarg}. +read_info(AbsPath) -> + % NOTE + % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. + file:read_link_info(AbsPath, [{time, posix}, raw]). + -spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> Acc. fold(Fun, Acc, Root, Glob) -> fold(Fun, Acc, [], Root, Glob, []). fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> - case file:list_dir(filename:join(Root, Path)) of + case list_dir(filename:join(Root, Path)) of {ok, Filenames} -> lists:foldl( fun(FN, Acc) -> @@ -172,7 +181,7 @@ fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_fu Fun(Path, {error, Reason}, Stack, AccIn) end; fold(Fun, AccIn, Filepath, Root, [], Stack) -> - case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of + case ?MODULE:read_info(filename:join(Root, Filepath)) of {ok, Info} -> Fun(Filepath, Info, Stack, AccIn); {error, Reason} -> @@ -183,3 +192,13 @@ matches_glob('*', _) -> true; matches_glob(FilterFun, Filename) when is_function(FilterFun) -> FilterFun(Filename). + +list_dir(AbsPath) -> + case ?MODULE:read_info(AbsPath) of + {ok, #file_info{type = directory}} -> + file:list_dir(AbsPath); + {ok, #file_info{}} -> + {error, enotdir}; + {error, Reason} -> + {error, Reason} + end. diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl new file mode 100644 index 000000000..81a483651 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -0,0 +1,159 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +t_fold_single_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a", #file_info{type = directory}, ["a"]}, + {"c", #file_info{type = directory}, ["c"]}, + {"d", #file_info{type = directory}, ["d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*'])) + ). + +t_fold_multi_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*'])) + ). + +t_fold_no_glob(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [{"", #file_info{type = directory}, []}], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [])) + ). + +t_fold_glob_too_deep(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + ). + +t_fold_invalid_root(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + ), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + ). + +t_fold_filter_unicode(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + ), + ?assertMatch( + [ + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + ). + +t_fold_filter_levels(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + ). + +t_fold_errors(Config) -> + Root = ?config(data_dir, Config), + ok = meck:new(emqx_ft_fs_util, [passthrough]), + ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) -> + ct:pal("read_info(~p)", [AbsFilepath]), + Filename = filename:basename(AbsFilepath), + case Filename of + "b" -> {error, eacces}; + "link" -> {error, enotsup}; + "bar" -> {error, enotdir}; + "needle" -> {error, ebusy}; + _ -> meck:passthrough([AbsFilepath]) + end + end), + ?assertMatch( + [ + {"a/b", {error, eacces}, ["b", "a"]}, + {"a/link", {error, enotsup}, ["link", "a"]}, + {"c/link", {error, enotsup}, ["link", "c"]}, + {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +%% + +is_not(F) -> + fun(X) -> not F(X) end. + +is_latin1(Filename) -> + case unicode:characters_to_binary(Filename, unicode, latin1) of + {error, _, _} -> + false; + _ -> + true + end. + +is_letter(Filename) -> + case Filename of + [_] -> + true; + _ -> + false + end. + +cons(Path, Info, Stack, Acc) -> + [{Path, Info, Stack} | Acc]. + +sort(L) when is_list(L) -> + lists:sort(L). diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я new file mode 100644 index 000000000..ac31ffd53 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я @@ -0,0 +1 @@ +Ты diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link new file mode 120000 index 000000000..1b271d838 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link @@ -0,0 +1 @@ +../c \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 new file mode 100644 index 000000000..2e11eb72f --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 @@ -0,0 +1 @@ +Zhōngwén diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link new file mode 120000 index 000000000..82f488f26 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link @@ -0,0 +1 @@ +../a \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle new file mode 100644 index 000000000..d755762d1 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle @@ -0,0 +1 @@ +haystack diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack new file mode 100644 index 000000000..a6b681bf4 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack @@ -0,0 +1 @@ +needle