diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config index 92ab7895e..b1483e028 100644 --- a/apps/emqx_s3/rebar.config +++ b/apps/emqx_s3/rebar.config @@ -1,6 +1,6 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}} + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c01b1b240..6bb6c2e3b 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -189,7 +189,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> -spec uri(client(), key()) -> iodata(). uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> - erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig). + erlcloud_s3:make_presigned_v4_url(ExpireTime, Bucket, get, erlcloud_key(Key), [], AwsConfig). -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index 5fda42fbe..e09ea2773 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -54,6 +54,8 @@ groups() -> t_happy_path_multi, t_abort_multi, t_abort_simple_put, + t_signed_url_download, + t_signed_nonascii_url_download, {group, noconn_errors}, {group, timeout_errors}, @@ -193,6 +195,40 @@ t_happy_path_multi(Config) -> Key ). +t_signed_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/ascii.txt", + + {ok, Data} = upload(Key, 1024, 5), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + +t_signed_nonascii_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/unicode-🫠.txt", + + {ok, Data} = upload(Key, 1024 * 1024, 8), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + t_abort_multi(Config) -> Key = emqx_s3_test_helpers:unique_key(), {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), @@ -532,3 +568,24 @@ data(Byte, ChunkSize, ChunkCount) -> list_objects(Config) -> Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), proplists:get_value(contents, Props). + +upload(Key, ChunkSize, ChunkCount) -> + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, ChunkSize, ChunkCount), + + ok = lists:foreach( + fun(Chunk) -> ?assertEqual(ok, emqx_s3_uploader:write(Pid, Chunk)) end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ok = ?assertProcessExited( + normal, + Pid + ), + + {ok, Data}. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index f36eb0ad6..654b1bf54 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -4,7 +4,7 @@ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/mix.exs b/mix.exs index e5e91b491..4f3d5a217 100644 --- a/mix.exs +++ b/mix.exs @@ -94,10 +94,10 @@ defmodule EMQXUmbrella.MixProject do # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, "1.6.2", override: true}, + {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true} # end of erlcloud's deps