Thales Macedo Garitezi
GitHub
commit
770dd188b1
|
|
@ -43,8 +43,18 @@ runs:
|
|||
echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT
|
||||
echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT
|
||||
mkdir -p "$OTP_SOURCE_PATH" "$OTP_INSTALL_PATH"
|
||||
# we need this to skip using cache for self-hosted runners
|
||||
case ${{ inputs.os }} in
|
||||
*arm64)
|
||||
echo "SELF_HOSTED=true" >> $GITHUB_OUTPUT
|
||||
;;
|
||||
*)
|
||||
echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT
|
||||
;;
|
||||
esac
|
||||
- uses: actions/cache@v3
|
||||
id: cache
|
||||
if: steps.prepare.outputs.SELF_HOSTED != 'true'
|
||||
with:
|
||||
path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }}
|
||||
key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
|
||||
|
|
@ -54,22 +64,36 @@ runs:
|
|||
run: |
|
||||
OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}"
|
||||
OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}"
|
||||
SELF_HOSTED="${{ steps.prepare.outputs.SELF_HOSTED }}"
|
||||
# when it's self-hosted, it never hits the cache,
|
||||
# skip rebuild if it's self-hosted and the install path already has a 'bin'
|
||||
if [ "${SELF_HOSTED:-false}" = 'true' ]; then
|
||||
if [ -n "$OTP_INSTALL_PATH" ] && [ -d "$OTP_INSTALL_PATH/bin" ]; then
|
||||
echo "Skip rebuilding OTP, found $OTP_INSTALL_PATH"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
## when it's not self-hosted, or the install path is not found,
|
||||
## build otp from source code.
|
||||
if [ -d "$OTP_SOURCE_PATH" ]; then
|
||||
rm -rf "$OTP_SOURCE_PATH"
|
||||
fi
|
||||
git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH"
|
||||
cd "$OTP_SOURCE_PATH"
|
||||
if [ "$(arch)" = arm64 ]; then
|
||||
export CFLAGS="-O2 -g -I$(brew --prefix unixodbc)/include"
|
||||
export LDFLAGS="-L$(brew --prefix unixodbc)/lib"
|
||||
export CC="/usr/bin/gcc -I$(brew --prefix unixodbc)/include"
|
||||
WITH_ODBC="--with-odbc=$(brew --prefix unixodbc)"
|
||||
else
|
||||
WITH_ODBC=""
|
||||
fi
|
||||
./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH"
|
||||
./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) ${WITH_ODBC} --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH"
|
||||
make -j$(nproc)
|
||||
rm -rf "$OTP_INSTALL_PATH"
|
||||
make install
|
||||
if [ "$(arch)" = arm64 ]; then
|
||||
unset CFLAGS
|
||||
unset LDFLAGS
|
||||
unset CC
|
||||
fi
|
||||
- name: build
|
||||
env:
|
||||
|
|
@ -87,6 +111,10 @@ runs:
|
|||
shell: bash
|
||||
run: |
|
||||
export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
|
||||
# inspec erl in PATH
|
||||
which erl
|
||||
# inspec erl command banner
|
||||
erl -s init stop
|
||||
make ensure-rebar3
|
||||
mkdir -p $HOME/bin
|
||||
cp rebar3 $HOME/bin/rebar3
|
||||
|
|
|
|||
|
|
@ -32,10 +32,10 @@
|
|||
%% `apps/emqx/src/bpapi/README.md'
|
||||
|
||||
%% Community edition
|
||||
-define(EMQX_RELEASE_CE, "5.1.0-alpha.2").
|
||||
-define(EMQX_RELEASE_CE, "5.1.0-alpha.3").
|
||||
|
||||
%% Enterprise edition
|
||||
-define(EMQX_RELEASE_EE, "5.1.0-alpha.2").
|
||||
-define(EMQX_RELEASE_EE, "5.1.0-alpha.3").
|
||||
|
||||
%% the HTTP API version
|
||||
-define(EMQX_API_VERSION, "5.0").
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ choose_ingress_pool_size(
|
|||
{_Filter, #{share := _Name}} ->
|
||||
% NOTE: this is shared subscription, many workers may subscribe
|
||||
PoolSize;
|
||||
{_Filter, #{}} ->
|
||||
{_Filter, #{}} when PoolSize > 1 ->
|
||||
% NOTE: this is regular subscription, only one worker should subscribe
|
||||
?SLOG(warning, #{
|
||||
msg => "mqtt_bridge_ingress_pool_size_ignored",
|
||||
|
|
@ -110,6 +110,8 @@ choose_ingress_pool_size(
|
|||
config_pool_size => PoolSize,
|
||||
pool_size => 1
|
||||
}),
|
||||
1;
|
||||
{_Filter, #{}} when PoolSize == 1 ->
|
||||
1
|
||||
end.
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@
|
|||
|
||||
-export([fold/4]).
|
||||
|
||||
-export([mk_temp_filename/1]).
|
||||
|
||||
-type foldfun(Acc) ::
|
||||
fun(
|
||||
(
|
||||
|
|
@ -178,3 +180,24 @@ fold(FoldFun, Acc, It) ->
|
|||
none ->
|
||||
Acc
|
||||
end.
|
||||
|
||||
-spec mk_temp_filename(file:filename()) ->
|
||||
file:filename().
|
||||
mk_temp_filename(Filename) ->
|
||||
% NOTE
|
||||
% Using only the first 200 characters of the filename to avoid making filenames
|
||||
% exceeding 255 bytes in UTF-8. It's actually too conservative, `Suffix` can be
|
||||
% at most 16 bytes.
|
||||
Unique = erlang:unique_integer([positive]),
|
||||
Suffix = binary:encode_hex(<<Unique:64>>),
|
||||
mk_filename([string:slice(Filename, 0, 200), ".", Suffix]).
|
||||
|
||||
mk_filename(Comps) ->
|
||||
lists:append(lists:map(fun mk_filename_component/1, Comps)).
|
||||
|
||||
mk_filename_component(A) when is_atom(A) ->
|
||||
atom_to_list(A);
|
||||
mk_filename_component(B) when is_binary(B) ->
|
||||
unicode:characters_to_list(B);
|
||||
mk_filename_component(S) when is_list(S) ->
|
||||
S.
|
||||
|
|
|
|||
|
|
@ -42,7 +42,9 @@
|
|||
%% on most filesystems. Even though, say, S3 does not have such limitations, it's
|
||||
%% still useful to have a limit on the filename length, to avoid having to deal with
|
||||
%% limits in the storage backends.
|
||||
-define(MAX_FILENAME_BYTELEN, 255).
|
||||
%% Usual realistic limit is 255 bytes actually, but we leave some room for backends
|
||||
%% to spare.
|
||||
-define(MAX_FILENAME_BYTELEN, 240).
|
||||
|
||||
-import(hoconsc, [ref/2, mk/2]).
|
||||
|
||||
|
|
@ -145,7 +147,7 @@ fields(local_storage_segments) ->
|
|||
[
|
||||
{root,
|
||||
mk(
|
||||
binary(),
|
||||
string(),
|
||||
#{
|
||||
desc => ?DESC("local_storage_segments_root"),
|
||||
required => false
|
||||
|
|
@ -182,7 +184,7 @@ fields(local_storage_exporter) ->
|
|||
[
|
||||
{root,
|
||||
mk(
|
||||
binary(),
|
||||
string(),
|
||||
#{
|
||||
desc => ?DESC("local_storage_exporter_root"),
|
||||
required => false
|
||||
|
|
|
|||
|
|
@ -128,7 +128,17 @@ complete(
|
|||
Filemeta = FilemetaIn#{checksum => Checksum},
|
||||
ok = file:close(Handle),
|
||||
_ = filelib:ensure_dir(ResultFilepath),
|
||||
_ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)),
|
||||
ManifestFilepath = mk_manifest_filename(ResultFilepath),
|
||||
case file:write_file(ManifestFilepath, encode_filemeta(Filemeta)) of
|
||||
ok ->
|
||||
ok;
|
||||
{error, Reason} ->
|
||||
?SLOG(warning, "filemeta_write_failed", #{
|
||||
path => ManifestFilepath,
|
||||
meta => Filemeta,
|
||||
reason => Reason
|
||||
})
|
||||
end,
|
||||
file:rename(Filepath, ResultFilepath).
|
||||
|
||||
-spec discard(export_st()) ->
|
||||
|
|
@ -452,8 +462,7 @@ mk_manifest_filename(Filename) when is_binary(Filename) ->
|
|||
<<Filename/binary, ?MANIFEST>>.
|
||||
|
||||
mk_temp_absfilepath(Options, Transfer, Filename) ->
|
||||
Unique = erlang:unique_integer([positive]),
|
||||
TempFilename = integer_to_list(Unique) ++ "." ++ Filename,
|
||||
TempFilename = emqx_ft_fs_util:mk_temp_filename(Filename),
|
||||
filename:join(mk_absdir(Options, Transfer, temporary), TempFilename).
|
||||
|
||||
mk_absdir(Options, _Transfer, temporary) ->
|
||||
|
|
|
|||
|
|
@ -445,16 +445,8 @@ write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content)
|
|||
end.
|
||||
|
||||
mk_temp_filepath(Storage, Transfer, Filename) ->
|
||||
Unique = erlang:unique_integer([positive]),
|
||||
filename:join(get_subdir(Storage, Transfer, temporary), mk_filename([Unique, ".", Filename])).
|
||||
|
||||
mk_filename(Comps) ->
|
||||
lists:append(lists:map(fun mk_filename_component/1, Comps)).
|
||||
|
||||
mk_filename_component(I) when is_integer(I) -> integer_to_list(I);
|
||||
mk_filename_component(A) when is_atom(A) -> atom_to_list(A);
|
||||
mk_filename_component(B) when is_binary(B) -> unicode:characters_to_list(B);
|
||||
mk_filename_component(S) when is_list(S) -> S.
|
||||
TempFilename = emqx_ft_fs_util:mk_temp_filename(Filename),
|
||||
filename:join(get_subdir(Storage, Transfer, temporary), TempFilename).
|
||||
|
||||
write_contents(Filepath, Content) ->
|
||||
file:write_file(Filepath, Content).
|
||||
|
|
|
|||
|
|
@ -261,6 +261,7 @@ t_nasty_clientids_fileids(_Config) ->
|
|||
fun({ClientId, FileId}) ->
|
||||
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId),
|
||||
[Export] = list_files(ClientId),
|
||||
?assertMatch(#{meta := #{name := "justfile"}}, Export),
|
||||
?assertEqual({ok, ClientId}, read_export(Export))
|
||||
end,
|
||||
Transfers
|
||||
|
|
@ -270,13 +271,15 @@ t_nasty_filenames(_Config) ->
|
|||
Filenames = [
|
||||
{<<"nasty1">>, "146%"},
|
||||
{<<"nasty2">>, "🌚"},
|
||||
{<<"nasty3">>, "中文.txt"}
|
||||
{<<"nasty3">>, "中文.txt"},
|
||||
{<<"nasty4">>, _239Bytes = string:join(lists:duplicate(240 div 5, "LONG"), ".")}
|
||||
],
|
||||
ok = lists:foreach(
|
||||
fun({ClientId, Filename}) ->
|
||||
FileId = unicode:characters_to_binary(Filename),
|
||||
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Filename, FileId),
|
||||
[Export] = list_files(ClientId),
|
||||
?assertMatch(#{meta := #{name := Filename}}, Export),
|
||||
?assertEqual({ok, FileId}, read_export(Export))
|
||||
end,
|
||||
Filenames
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ t_update_config(_Config) ->
|
|||
)
|
||||
),
|
||||
?assertEqual(
|
||||
<<"/tmp/path">>,
|
||||
"/tmp/path",
|
||||
emqx_config:get([file_transfer, storage, local, segments, root])
|
||||
),
|
||||
?assertEqual(
|
||||
|
|
@ -150,7 +150,7 @@ t_disable_restore_config(Config) ->
|
|||
),
|
||||
ok = emqtt:stop(Client),
|
||||
% Restore local storage backend
|
||||
Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])),
|
||||
Root = emqx_ft_test_helpers:root(Config, node(), [segments]),
|
||||
?assertMatch(
|
||||
{ok, _},
|
||||
emqx_conf:update(
|
||||
|
|
@ -177,7 +177,7 @@ t_disable_restore_config(Config) ->
|
|||
[
|
||||
#{
|
||||
?snk_kind := garbage_collection,
|
||||
storage := #{segments := #{root := Root}}
|
||||
storage := #{segments := #{gc := #{interval := 1000}}}
|
||||
}
|
||||
],
|
||||
?of_kind(garbage_collection, Trace)
|
||||
|
|
|
|||
|
|
@ -63,3 +63,24 @@ unescape_filename_test_() ->
|
|||
?_assertEqual(Input, emqx_ft_fs_util:unescape_filename(Filename))
|
||||
|| {Filename, Input} <- ?NAMES
|
||||
].
|
||||
|
||||
mk_temp_filename_test_() ->
|
||||
[
|
||||
?_assertMatch(
|
||||
"." ++ Suffix when length(Suffix) == 16,
|
||||
emqx_ft_fs_util:mk_temp_filename(<<>>)
|
||||
),
|
||||
?_assertMatch(
|
||||
"file.name." ++ Suffix when length(Suffix) == 16,
|
||||
emqx_ft_fs_util:mk_temp_filename("file.name")
|
||||
),
|
||||
?_assertMatch(
|
||||
"safe.🦺." ++ Suffix when length(Suffix) == 16,
|
||||
emqx_ft_fs_util:mk_temp_filename(<<"safe.🦺"/utf8>>)
|
||||
),
|
||||
?_assertEqual(
|
||||
% FilenameSlice + Dot + Suffix
|
||||
200 + 1 + 16,
|
||||
length(emqx_ft_fs_util:mk_temp_filename(lists:duplicate(63, "LONG")))
|
||||
)
|
||||
].
|
||||
|
|
|
|||
Loading…
Reference in New Issue