From dcd59e4f1b069498dd669d9f4efbffa51a624f79 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 6 Jun 2023 10:40:20 +0300 Subject: [PATCH] fix(ft): set more conservative filename length limit Otherwise, local fs exporter will have a hard time preserving the filemeta, because its filename is even 13 bytes longer. --- apps/emqx_ft/src/emqx_ft_schema.erl | 4 +++- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 12 +++++++++++- apps/emqx_ft/test/emqx_ft_SUITE.erl | 4 +++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 2b98562b4..c1ee41d0d 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -42,7 +42,9 @@ %% on most filesystems. Even though, say, S3 does not have such limitations, it's %% still useful to have a limit on the filename length, to avoid having to deal with %% limits in the storage backends. --define(MAX_FILENAME_BYTELEN, 255). +%% Usual realistic limit is 255 bytes actually, but we leave some room for backends +%% to spare. +-define(MAX_FILENAME_BYTELEN, 240). -import(hoconsc, [ref/2, mk/2]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index ae709dd52..e211cb421 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -128,7 +128,17 @@ complete( Filemeta = FilemetaIn#{checksum => Checksum}, ok = file:close(Handle), _ = filelib:ensure_dir(ResultFilepath), - _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + ManifestFilepath = mk_manifest_filename(ResultFilepath), + case file:write_file(ManifestFilepath, encode_filemeta(Filemeta)) of + ok -> + ok; + {error, Reason} -> + ?SLOG(warning, "filemeta_write_failed", #{ + path => ManifestFilepath, + meta => Filemeta, + reason => Reason + }) + end, file:rename(Filepath, ResultFilepath). -spec discard(export_st()) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 6861038e9..c48c77d93 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -261,6 +261,7 @@ t_nasty_clientids_fileids(_Config) -> fun({ClientId, FileId}) -> ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), [Export] = list_files(ClientId), + ?assertMatch(#{meta := #{name := "justfile"}}, Export), ?assertEqual({ok, ClientId}, read_export(Export)) end, Transfers @@ -271,13 +272,14 @@ t_nasty_filenames(_Config) -> {<<"nasty1">>, "146%"}, {<<"nasty2">>, "🌚"}, {<<"nasty3">>, "δΈ­ζ–‡.txt"}, - {<<"nasty4">>, _254Bytes = string:join(lists:duplicate(255 div 5, "LONG"), ".")} + {<<"nasty4">>, _239Bytes = string:join(lists:duplicate(240 div 5, "LONG"), ".")} ], ok = lists:foreach( fun({ClientId, Filename}) -> FileId = unicode:characters_to_binary(Filename), ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Filename, FileId), [Export] = list_files(ClientId), + ?assertMatch(#{meta := #{name := Filename}}, Export), ?assertEqual({ok, FileId}, read_export(Export)) end, Filenames