diff --git a/rebar.config b/rebar.config index cf2a46aac..3b0cdb905 100644 --- a/rebar.config +++ b/rebar.config @@ -57,6 +57,7 @@ , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.4.0"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index a72f57933..6f2816b3a 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -246,6 +246,7 @@ relx_apps(ReleaseType) -> , {ekka, load} , {emqx_plugin_libs, load} , observer_cli + , emqx_http_lib ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl deleted file mode 100644 index 893c260ee..000000000 --- a/src/emqx_http_lib.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib). - --export([ uri_encode/1 - , uri_decode/1 - , uri_parse/1 - , normalise_headers/1 - ]). - --export_type([uri_map/0]). - --type uri_map() :: #{scheme := http | https, - host := unicode:chardata(), - port := non_neg_integer(), - path => unicode:chardata(), - query => unicode:chardata(), - fragment => unicode:chardata(), - userinfo => unicode:chardata()}. - --type hex_uri() :: string() | binary(). --type maybe_hex_uri() :: string() | binary(). %% A possibly hexadecimal encoded URI. --type uri() :: string() | binary(). - -%% @doc Decode percent-encoded URI. -%% This is copied from http_uri.erl which has been deprecated since OTP-23 -%% The recommended replacement uri_string function is not quite equivalent -%% and not backward compatible. --spec uri_decode(maybe_hex_uri()) -> uri(). -uri_decode(String) when is_list(String) -> - do_uri_decode(String); -uri_decode(String) when is_binary(String) -> - do_uri_decode_binary(String). - -do_uri_decode([$%,Hex1,Hex2|Rest]) -> - [hex2dec(Hex1)*16+hex2dec(Hex2)|do_uri_decode(Rest)]; -do_uri_decode([First|Rest]) -> - [First|do_uri_decode(Rest)]; -do_uri_decode([]) -> - []. - -do_uri_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> - <<(binary_to_integer(Hex, 16)), (do_uri_decode_binary(Rest))/binary>>; -do_uri_decode_binary(<>) -> - <>; -do_uri_decode_binary(<<>>) -> - <<>>. - -%% @doc Encode URI. --spec uri_encode(uri()) -> hex_uri(). -uri_encode(URI) when is_list(URI) -> - lists:append([do_uri_encode(Char) || Char <- URI]); -uri_encode(URI) when is_binary(URI) -> - << <<(do_uri_encode_binary(Char))/binary>> || <> <= URI >>. - -%% @doc Parse URI into a map as uri_string:uri_map(), but with two fields -%% normalised: (1): port number is never 'undefined', default ports are used -%% if missing. (2): scheme is always atom. --spec uri_parse(string() | binary()) -> {ok, uri_map()} | {error, any()}. -uri_parse(URI) -> - try - {ok, do_parse(uri_string:normalize(URI))} - catch - throw : Reason -> - {error, Reason} - end. - -do_parse({error, Reason, Which}) -> throw({Reason, Which}); -do_parse(URI) -> - %% ensure we return string() instead of binary() in uri_map() values. - Map = uri_string:parse(unicode:characters_to_list(URI)), - case maps:is_key(scheme, Map) of - true -> - normalise_parse_result(Map); - false -> - %% missing scheme, add "http://" and try again - Map2 = uri_string:parse(unicode:characters_to_list(["http://", URI])), - normalise_parse_result(Map2) - end. - -%% @doc Return HTTP headers list with keys lower-cased and -%% underscores replaced with hyphens -%% NOTE: assuming the input Headers list is a proplists, -%% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] -%% results in: [{"content-type", "applicaiton/binary"}] -normalise_headers(Headers0) -> - F = fun({K0, V}) -> - K = re:replace(K0, "_", "-", [{return,list}]), - {string:lowercase(K), V} - end, - Headers = lists:map(F, Headers0), - Keys = proplists:get_keys(Headers), - [{K, proplists:get_value(K, Headers)} || K <- Keys]. - -normalise_parse_result(#{host := Host, scheme := Scheme0} = Map) -> - {Scheme, DefaultPort} = atom_scheme_and_default_port(Scheme0), - Port = case maps:get(port, Map, undefined) of - N when is_number(N) -> N; - _ -> DefaultPort - end, - Map#{ scheme := Scheme - , host := emqx_misc:maybe_parse_ip(Host) - , port => Port - }. - -%% NOTE: so far we only support http/coap schemes. -atom_scheme_and_default_port(Scheme) when is_list(Scheme) -> - atom_scheme_and_default_port(list_to_binary(Scheme)); -atom_scheme_and_default_port(<<"http">> ) -> {http, 80}; -atom_scheme_and_default_port(<<"https">>) -> {https, 443}; -atom_scheme_and_default_port(<<"coap">> ) -> {coap, 5683}; -atom_scheme_and_default_port(<<"coaps">>) -> {coaps, 5684}; -atom_scheme_and_default_port(Other) -> throw({unsupported_scheme, Other}). - -do_uri_encode(Char) -> - case reserved(Char) of - true -> - [ $% | integer_to_hexlist(Char)]; - false -> - [Char] - end. - -do_uri_encode_binary(Char) -> - case reserved(Char) of - true -> - << $%, (integer_to_binary(Char, 16))/binary >>; - false -> - <> - end. - -reserved($;) -> true; -reserved($:) -> true; -reserved($@) -> true; -reserved($&) -> true; -reserved($=) -> true; -reserved($+) -> true; -reserved($,) -> true; -reserved($/) -> true; -reserved($?) -> true; -reserved($#) -> true; -reserved($[) -> true; -reserved($]) -> true; -reserved($<) -> true; -reserved($>) -> true; -reserved($\") -> true; -reserved(${) -> true; -reserved($}) -> true; -reserved($|) -> true; -reserved($\\) -> true; -reserved($') -> true; -reserved($^) -> true; -reserved($%) -> true; -reserved($\s) -> true; -reserved(_) -> false. - -integer_to_hexlist(Int) -> - integer_to_list(Int, 16). - -hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; -hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; -hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl deleted file mode 100644 index 7bcb7d056..000000000 --- a/test/emqx_http_lib_tests.erl +++ /dev/null @@ -1,94 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib_tests). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - -uri_encode_decode_test_() -> - Opts = [{numtests, 1000}, {to_file, user}], - {timeout, 10, - fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}. - -prop_run() -> - ?FORALL(Generated, prop_uri(), test_prop_uri(iolist_to_binary(Generated))). - -prop_uri() -> - proper_types:non_empty(proper_types:list(proper_types:union([prop_char(), prop_reserved()]))). - -prop_char() -> proper_types:integer(32, 126). - -prop_reserved() -> - proper_types:oneof([$;, $:, $@, $&, $=, $+, $,, $/, $?, - $#, $[, $], $<, $>, $\", ${, $}, $|, - $\\, $', $^, $%, $ ]). - -test_prop_uri(URI) -> - Encoded = emqx_http_lib:uri_encode(URI), - Decoded1 = emqx_http_lib:uri_decode(Encoded), - ?assertEqual(URI, Decoded1), - Decoded2 = uri_string:percent_decode(Encoded), - ?assertEqual(URI, Decoded2), - true. - -uri_parse_test_() -> - [ {"default port http", - fun() -> ?assertMatch({ok, #{port := 80, scheme := http, host := "localhost"}}, - emqx_http_lib:uri_parse("localhost")) - end - } - , {"default port https", - fun() -> ?assertMatch({ok, #{port := 443, scheme := https}}, - emqx_http_lib:uri_parse("https://localhost")) - end - } - , {"bad url", - fun() -> ?assertMatch({error, {invalid_uri, _}}, - emqx_http_lib:uri_parse("https://localhost:notnumber")) - end - } - , {"normalise", - fun() -> ?assertMatch({ok, #{scheme := https, host := {127, 0, 0, 1}}}, - emqx_http_lib:uri_parse("HTTPS://127.0.0.1")) - end - } - , {"coap default port", - fun() -> ?assertMatch({ok, #{scheme := coap, port := 5683}}, - emqx_http_lib:uri_parse("coap://127.0.0.1")) - end - } - , {"coaps default port", - fun() -> ?assertMatch({ok, #{scheme := coaps, port := 5684}}, - emqx_http_lib:uri_parse("coaps://127.0.0.1")) - end - } - , {"unsupported_scheme", - fun() -> ?assertEqual({error, {unsupported_scheme, <<"wss">>}}, - emqx_http_lib:uri_parse("wss://127.0.0.1")) - end - } - , {"ipv6 host", - fun() -> ?assertMatch({ok, #{scheme := http, host := T}} when size(T) =:= 8, - emqx_http_lib:uri_parse("http://[::1]:80")) - end - } - ]. - -normalise_headers_test() -> - ?assertEqual([{"content-type", "applicaiton/binary"}], - emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"}, - {"content-type", "applicaiton/json"}])).