Merge pull request #12241 from keynslug/fix/EMQX-11665/s3-headers

fix(s3): handle extra configured headers correctly

apps/emqx_s3/src/emqx_s3.app.src

@@ -1,6 +1,6 @@
 {application, emqx_s3, [
     {description, "EMQX S3"},
-    {vsn, "5.0.12"},
+    {vsn, "5.0.13"},
     {modules, []},
     {registered, [emqx_s3_sup]},
     {applications, [
@@ -8,8 +8,7 @@
         stdlib,
         gproc,
         erlcloud,
-        ehttpc,
-        emqx_bridge_http
+        ehttpc
     ]},
     {mod, {emqx_s3_app, []}}
 ]}.

apps/emqx_s3/src/emqx_s3_client.erl

@@ -388,7 +388,7 @@ with_path_and_query_only(Url, Fun) ->
 
 %% Users provide headers as a map, but erlcloud expects a list of tuples with string keys and values.
 headers_user_to_erlcloud_request(UserHeaders) ->
-    [{to_list_string(K), V} || {K, V} <- maps:to_list(UserHeaders)].
+    [{string:to_lower(to_list_string(K)), V} || {K, V} <- maps:to_list(UserHeaders)].
 
 %% Ehttpc returns operates on headers as a list of tuples with binary keys.
 %% Erlcloud expects a list of tuples with string values and lowcase string keys
@@ -409,6 +409,8 @@ to_binary(Val) when is_binary(Val) -> Val.
 
 to_list_string(Val) when is_binary(Val) ->
     binary_to_list(Val);
+to_list_string(Val) when is_atom(Val) ->
+    atom_to_list(Val);
 to_list_string(Val) when is_list(Val) ->
     Val.
 

apps/emqx_s3/test/emqx_s3_client_SUITE.erl

@@ -142,6 +142,22 @@ t_no_acl(Config) ->
 
     ok = emqx_s3_client:put_object(Client, Key, <<"data">>).
 
+t_extra_headers(Config0) ->
+    Config = [{extra_headers, #{'Content-Type' => <<"application/json">>}} | Config0],
+    Key = ?config(key, Config),
+
+    Client = client(Config),
+    Data = #{foo => bar},
+    ok = emqx_s3_client:put_object(Client, Key, emqx_utils_json:encode(Data)),
+
+    Url = emqx_s3_client:uri(Client, Key),
+
+    {ok, {{_StatusLine, 200, "OK"}, _Headers, Content}} = httpc:request(Url),
+    ?_assertEqual(
+        Data,
+        emqx_utils_json:decode(Content)
+    ).
+
 %%--------------------------------------------------------------------
 %% Helpers
 %%--------------------------------------------------------------------
@@ -154,17 +170,22 @@ client(Config) ->
 
 profile_config(Config) ->
     ProfileConfig0 = emqx_s3_test_helpers:base_config(?config(conn_type, Config)),
-    ProfileConfig1 = maps:put(
-        bucket,
-        ?config(bucket, Config),
-        ProfileConfig0
-    ),
-    ProfileConfig2 = emqx_utils_maps:deep_put(
-        [transport_options, pool_type],
-        ProfileConfig1,
-        ?config(pool_type, Config)
-    ),
-    ProfileConfig2.
+    maps:fold(
+        fun inject_config/3,
+        ProfileConfig0,
+        #{
+            bucket => ?config(bucket, Config),
+            [transport_options, pool_type] => ?config(pool_type, Config),
+            [transport_options, headers] => ?config(extra_headers, Config)
+        }
+    ).
+
+inject_config(_Key, undefined, ProfileConfig) ->
+    ProfileConfig;
+inject_config(KeyPath, Value, ProfileConfig) when is_list(KeyPath) ->
+    emqx_utils_maps:deep_put(KeyPath, ProfileConfig, Value);
+inject_config(Key, Value, ProfileConfig) ->
+    maps:put(Key, Value, ProfileConfig).
 
 data(Size) ->
     iolist_to_binary([$a || _ <- lists:seq(1, Size)]).

changes/ee/fix-12241.en.md

@@ -0,0 +1 @@
+Fix an issue where setting up extra HTTP headers for communication with S3 API would break File Transfers using S3 storage backend.