Merge pull request #6195 from tigercl/fix/multi-http-headers

fix(http): fix duplicate http headers

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_http,
  [{description, "EMQ X Authentication/ACL with HTTP API"},
-  {vsn, "4.3.2"}, % strict semver, bump manually!
+  {vsn, "4.3.3"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_http_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_auth_http/src/emqx_auth_http.appup.src

@@ -2,12 +2,22 @@
 
 {VSN,
   [
+    {"4.3.2", [
+     {apply, {application, stop, [emqx_auth_http]}},
+     {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]},
+     {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]}
+    ]},
     {<<"4.3.[0-1]">>, [
      {restart_application, emqx_auth_http}
     ]},
     {<<".*">>, []}
   ],
   [
+    {"4.3.2", [
+     {apply, {application, stop, [emqx_auth_http]}},
+     {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]},
+     {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]}
+    ]},
     {<<"4.3.[0-1]">>, [
      {restart_application, emqx_auth_http}
     ]},

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -150,7 +150,7 @@ ensure_content_type_header(Method, Headers)
   when Method =:= post orelse Method =:= put ->
     Headers;
 ensure_content_type_header(_Method, Headers) ->
-    lists:keydelete("content-type", 1, Headers).
+    lists:keydelete(<<"content-type">>, 1, Headers).
 
 path(#{path := "", 'query' := Query}) ->
     "?" ++ Query;

apps/emqx_auth_http/src/emqx_auth_http_cli.erl

@@ -32,7 +32,7 @@ request(PoolName, get, Path, Headers, Params, Timeout) ->
     reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout));
 
 request(PoolName, post, Path, Headers, Params, Timeout) ->
-    Body = case proplists:get_value("content-type", Headers) of
+    Body = case proplists:get_value(<<"content-type">>, Headers) of
                "application/x-www-form-urlencoded" ->
                    cow_qs:qs(bin_kw(Params));
                "application/json" -> 

apps/emqx_web_hook/src/emqx_web_hook.app.src

@@ -1,6 +1,6 @@
 {application, emqx_web_hook,
  [{description, "EMQ X WebHook Plugin"},
-  {vsn, "4.3.7"}, % strict semver, bump manually!
+  {vsn, "4.3.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_web_hook_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_web_hook/src/emqx_web_hook.appup.src

@@ -1,6 +1,10 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
+  [{"4.3.7",
+    [{apply,{application,stop,[emqx_web_hook]}},
+     {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {<<"4.3.[0-2]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
      {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
@@ -9,7 +13,11 @@
    {<<"4.3.[3-4]">>,
     [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
+  [{"4.3.7",
+    [{apply,{application,stop,[emqx_web_hook]}},
+     {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {<<"4.3.[0-2]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
      {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},

apps/emqx_web_hook/src/emqx_web_hook_actions.erl

@@ -295,7 +295,7 @@ create_req(_, Path, Headers, Body) ->
 parse_action_params(Params = #{<<"url">> := URL}) ->
     {ok, #{path := CommonPath}} = emqx_http_lib:uri_parse(URL),
     Method = method(maps:get(<<"method">>, Params, <<"POST">>)),
-    Headers = headers(maps:get(<<"headers">>, Params, undefined)),
+    Headers = headers(maps:get(<<"headers">>, Params, #{})),
     NHeaders = ensure_content_type_header(Headers, Method),
     #{method => Method,
       path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)),
@@ -307,7 +307,7 @@ parse_action_params(Params = #{<<"url">> := URL}) ->
 ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put ->
     Headers;
 ensure_content_type_header(Headers, _Method) ->
-    lists:keydelete("content-type", 1, Headers).
+    lists:keydelete(<<"content-type">>, 1, Headers).
 
 merge_path(CommonPath, <<>>) ->
     l2b(CommonPath);
@@ -326,11 +326,8 @@ method(POST) when POST == <<"POST">>; POST == <<"post">> -> post;
 method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put;
 method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete.
 
-headers(undefined) -> [];
-headers(Headers) when is_map(Headers) ->
-    headers(maps:to_list(Headers));
-headers(Headers) when is_list(Headers) ->
-    [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers].
+headers(Headers) ->
+    emqx_http_lib:normalise_headers(maps:to_list(Headers)).
 
 str(Str) when is_list(Str) -> Str;
 str(Atom) when is_atom(Atom) -> atom_to_list(Atom);

apps/emqx_web_hook/src/emqx_web_hook_app.erl

@@ -87,7 +87,7 @@ translate_env() ->
     application:set_env(?APP, path, Path),
     application:set_env(?APP, pool_opts, PoolOpts),
     Headers = application:get_env(?APP, headers, []),
-    NHeaders = set_content_type(Headers),
+    NHeaders = set_content_type(emqx_http_lib:normalise_headers(Headers)),
     application:set_env(?APP, headers, NHeaders).
 
 path(#{path := "", 'query' := Query}) ->
@@ -100,5 +100,5 @@ path(#{path := Path}) ->
     Path.
 
 set_content_type(Headers) ->
-    NHeaders = proplists:delete(<<"Content-Type">>, proplists:delete(<<"content-type">>, Headers)),
+    NHeaders = proplists:delete(<<"content-type">>, Headers),
     [{<<"content-type">>, <<"application/json">>} | NHeaders].

src/emqx.app.src

@@ -1,7 +1,7 @@
 {application, emqx,
  [{id, "emqx"},
   {description, "EMQ X"},
-  {vsn, "4.3.11"}, % strict semver, bump manually!
+  {vsn, "4.3.12"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]},

src/emqx.appup.src

@@ -1,10 +1,14 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.10",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+  [{"4.3.11",
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
+   {"4.3.10",
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -14,7 +18,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -24,7 +29,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -36,7 +42,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -49,7 +56,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
@@ -63,7 +71,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
@@ -78,7 +87,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
@@ -157,11 +167,15 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.10",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+  [{"4.3.11",
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
+   {"4.3.10",
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -171,7 +185,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -181,7 +196,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -193,7 +209,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
@@ -206,7 +223,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
@@ -220,7 +238,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
@@ -235,7 +254,8 @@
      {load_module,emqx_rpc,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
-    [{load_module,emqx_connection,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},

src/emqx_http_lib.erl

@@ -96,11 +96,11 @@ do_parse(URI) ->
 %% underscores replaced with hyphens
 %% NOTE: assuming the input Headers list is a proplists,
 %% that is, when a key is duplicated, list header overrides tail
-%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}]
+%% e.g. [{"Content_Type", "applicaiton/binary"}, {<<"content-type">>, "applicaiton/json"}]
 %% results in: [{"content-type", "applicaiton/binary"}]
 normalise_headers(Headers0) ->
     F = fun({K0, V}) ->
-                K = re:replace(K0, "_", "-", [{return,list}]),
+                K = re:replace(K0, "_", "-", [{return,binary}]),
                 {string:lowercase(K), V}
         end,
     Headers = lists:map(F, Headers0),

test/emqx_http_lib_tests.erl

@@ -89,6 +89,6 @@ uri_parse_test_() ->
     ].
 
 normalise_headers_test() ->
-    ?assertEqual([{"content-type", "applicaiton/binary"}],
+    ?assertEqual([{<<"content-type">>, "applicaiton/binary"}],
                  emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"},
                                                   {"content-type", "applicaiton/json"}])).