From da6a8e39913a2dc1da51cdbae2e19e51e884ba9b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 5 Jun 2021 10:16:12 +0200 Subject: [PATCH 1/2] chore(http_lib): use lib app --- rebar.config | 1 + rebar.config.erl | 1 + src/emqx_http_lib.erl | 176 ----------------------------------- test/emqx_http_lib_tests.erl | 94 ------------------- 4 files changed, 2 insertions(+), 270 deletions(-) delete mode 100644 src/emqx_http_lib.erl delete mode 100644 test/emqx_http_lib_tests.erl diff --git a/rebar.config b/rebar.config index cf2a46aac..3b0cdb905 100644 --- a/rebar.config +++ b/rebar.config @@ -57,6 +57,7 @@ , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.4.0"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index a72f57933..6f2816b3a 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -246,6 +246,7 @@ relx_apps(ReleaseType) -> , {ekka, load} , {emqx_plugin_libs, load} , observer_cli + , emqx_http_lib ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl deleted file mode 100644 index 893c260ee..000000000 --- a/src/emqx_http_lib.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib). - --export([ uri_encode/1 - , uri_decode/1 - , uri_parse/1 - , normalise_headers/1 - ]). - --export_type([uri_map/0]). - --type uri_map() :: #{scheme := http | https, - host := unicode:chardata(), - port := non_neg_integer(), - path => unicode:chardata(), - query => unicode:chardata(), - fragment => unicode:chardata(), - userinfo => unicode:chardata()}. - --type hex_uri() :: string() | binary(). --type maybe_hex_uri() :: string() | binary(). %% A possibly hexadecimal encoded URI. --type uri() :: string() | binary(). - -%% @doc Decode percent-encoded URI. -%% This is copied from http_uri.erl which has been deprecated since OTP-23 -%% The recommended replacement uri_string function is not quite equivalent -%% and not backward compatible. --spec uri_decode(maybe_hex_uri()) -> uri(). -uri_decode(String) when is_list(String) -> - do_uri_decode(String); -uri_decode(String) when is_binary(String) -> - do_uri_decode_binary(String). - -do_uri_decode([$%,Hex1,Hex2|Rest]) -> - [hex2dec(Hex1)*16+hex2dec(Hex2)|do_uri_decode(Rest)]; -do_uri_decode([First|Rest]) -> - [First|do_uri_decode(Rest)]; -do_uri_decode([]) -> - []. - -do_uri_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> - <<(binary_to_integer(Hex, 16)), (do_uri_decode_binary(Rest))/binary>>; -do_uri_decode_binary(<>) -> - <>; -do_uri_decode_binary(<<>>) -> - <<>>. - -%% @doc Encode URI. --spec uri_encode(uri()) -> hex_uri(). -uri_encode(URI) when is_list(URI) -> - lists:append([do_uri_encode(Char) || Char <- URI]); -uri_encode(URI) when is_binary(URI) -> - << <<(do_uri_encode_binary(Char))/binary>> || <> <= URI >>. - -%% @doc Parse URI into a map as uri_string:uri_map(), but with two fields -%% normalised: (1): port number is never 'undefined', default ports are used -%% if missing. (2): scheme is always atom. --spec uri_parse(string() | binary()) -> {ok, uri_map()} | {error, any()}. -uri_parse(URI) -> - try - {ok, do_parse(uri_string:normalize(URI))} - catch - throw : Reason -> - {error, Reason} - end. - -do_parse({error, Reason, Which}) -> throw({Reason, Which}); -do_parse(URI) -> - %% ensure we return string() instead of binary() in uri_map() values. - Map = uri_string:parse(unicode:characters_to_list(URI)), - case maps:is_key(scheme, Map) of - true -> - normalise_parse_result(Map); - false -> - %% missing scheme, add "http://" and try again - Map2 = uri_string:parse(unicode:characters_to_list(["http://", URI])), - normalise_parse_result(Map2) - end. - -%% @doc Return HTTP headers list with keys lower-cased and -%% underscores replaced with hyphens -%% NOTE: assuming the input Headers list is a proplists, -%% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] -%% results in: [{"content-type", "applicaiton/binary"}] -normalise_headers(Headers0) -> - F = fun({K0, V}) -> - K = re:replace(K0, "_", "-", [{return,list}]), - {string:lowercase(K), V} - end, - Headers = lists:map(F, Headers0), - Keys = proplists:get_keys(Headers), - [{K, proplists:get_value(K, Headers)} || K <- Keys]. - -normalise_parse_result(#{host := Host, scheme := Scheme0} = Map) -> - {Scheme, DefaultPort} = atom_scheme_and_default_port(Scheme0), - Port = case maps:get(port, Map, undefined) of - N when is_number(N) -> N; - _ -> DefaultPort - end, - Map#{ scheme := Scheme - , host := emqx_misc:maybe_parse_ip(Host) - , port => Port - }. - -%% NOTE: so far we only support http/coap schemes. -atom_scheme_and_default_port(Scheme) when is_list(Scheme) -> - atom_scheme_and_default_port(list_to_binary(Scheme)); -atom_scheme_and_default_port(<<"http">> ) -> {http, 80}; -atom_scheme_and_default_port(<<"https">>) -> {https, 443}; -atom_scheme_and_default_port(<<"coap">> ) -> {coap, 5683}; -atom_scheme_and_default_port(<<"coaps">>) -> {coaps, 5684}; -atom_scheme_and_default_port(Other) -> throw({unsupported_scheme, Other}). - -do_uri_encode(Char) -> - case reserved(Char) of - true -> - [ $% | integer_to_hexlist(Char)]; - false -> - [Char] - end. - -do_uri_encode_binary(Char) -> - case reserved(Char) of - true -> - << $%, (integer_to_binary(Char, 16))/binary >>; - false -> - <> - end. - -reserved($;) -> true; -reserved($:) -> true; -reserved($@) -> true; -reserved($&) -> true; -reserved($=) -> true; -reserved($+) -> true; -reserved($,) -> true; -reserved($/) -> true; -reserved($?) -> true; -reserved($#) -> true; -reserved($[) -> true; -reserved($]) -> true; -reserved($<) -> true; -reserved($>) -> true; -reserved($\") -> true; -reserved(${) -> true; -reserved($}) -> true; -reserved($|) -> true; -reserved($\\) -> true; -reserved($') -> true; -reserved($^) -> true; -reserved($%) -> true; -reserved($\s) -> true; -reserved(_) -> false. - -integer_to_hexlist(Int) -> - integer_to_list(Int, 16). - -hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; -hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; -hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl deleted file mode 100644 index 7bcb7d056..000000000 --- a/test/emqx_http_lib_tests.erl +++ /dev/null @@ -1,94 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_http_lib_tests). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - -uri_encode_decode_test_() -> - Opts = [{numtests, 1000}, {to_file, user}], - {timeout, 10, - fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}. - -prop_run() -> - ?FORALL(Generated, prop_uri(), test_prop_uri(iolist_to_binary(Generated))). - -prop_uri() -> - proper_types:non_empty(proper_types:list(proper_types:union([prop_char(), prop_reserved()]))). - -prop_char() -> proper_types:integer(32, 126). - -prop_reserved() -> - proper_types:oneof([$;, $:, $@, $&, $=, $+, $,, $/, $?, - $#, $[, $], $<, $>, $\", ${, $}, $|, - $\\, $', $^, $%, $ ]). - -test_prop_uri(URI) -> - Encoded = emqx_http_lib:uri_encode(URI), - Decoded1 = emqx_http_lib:uri_decode(Encoded), - ?assertEqual(URI, Decoded1), - Decoded2 = uri_string:percent_decode(Encoded), - ?assertEqual(URI, Decoded2), - true. - -uri_parse_test_() -> - [ {"default port http", - fun() -> ?assertMatch({ok, #{port := 80, scheme := http, host := "localhost"}}, - emqx_http_lib:uri_parse("localhost")) - end - } - , {"default port https", - fun() -> ?assertMatch({ok, #{port := 443, scheme := https}}, - emqx_http_lib:uri_parse("https://localhost")) - end - } - , {"bad url", - fun() -> ?assertMatch({error, {invalid_uri, _}}, - emqx_http_lib:uri_parse("https://localhost:notnumber")) - end - } - , {"normalise", - fun() -> ?assertMatch({ok, #{scheme := https, host := {127, 0, 0, 1}}}, - emqx_http_lib:uri_parse("HTTPS://127.0.0.1")) - end - } - , {"coap default port", - fun() -> ?assertMatch({ok, #{scheme := coap, port := 5683}}, - emqx_http_lib:uri_parse("coap://127.0.0.1")) - end - } - , {"coaps default port", - fun() -> ?assertMatch({ok, #{scheme := coaps, port := 5684}}, - emqx_http_lib:uri_parse("coaps://127.0.0.1")) - end - } - , {"unsupported_scheme", - fun() -> ?assertEqual({error, {unsupported_scheme, <<"wss">>}}, - emqx_http_lib:uri_parse("wss://127.0.0.1")) - end - } - , {"ipv6 host", - fun() -> ?assertMatch({ok, #{scheme := http, host := T}} when size(T) =:= 8, - emqx_http_lib:uri_parse("http://[::1]:80")) - end - } - ]. - -normalise_headers_test() -> - ?assertEqual([{"content-type", "applicaiton/binary"}], - emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"}, - {"content-type", "applicaiton/json"}])). From 5795bcca6ac54fdcf4c6960158c2cca84875a3a0 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 4 Jun 2021 19:21:38 +0200 Subject: [PATCH 2/2] chore: fix app versions and check script --- .../emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mysql.app.src | 2 +- .../src/emqx_auth_pgsql.app.src | 2 +- .../src/emqx_auth_redis.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- .../src/emqx_management.app.src | 2 +- apps/emqx_recon/src/emqx_recon.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- scripts/apps-version-check.sh | 47 ++++++++++++++----- 11 files changed, 45 insertions(+), 22 deletions(-) diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 8635c4834..1b76c32c8 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index cc4e72ef3..ab0b4ff56 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 221061f03..8a0d116cc 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index f70612262..e97487e21 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 6a38af2ce..419131566 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 52ce96a54..9841f5bcb 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.0"}, %% strict semver + {vsn, "4.4.0"}, %% strict semver {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 43fb5ba53..fe68fef44 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_recon/src/emqx_recon.app.src b/apps/emqx_recon/src/emqx_recon.app.src index 061ed8e93..e25e2bbc5 100644 --- a/apps/emqx_recon/src/emqx_recon.app.src +++ b/apps/emqx_recon/src/emqx_recon.app.src @@ -1,6 +1,6 @@ {application, emqx_recon, [{description, "EMQ X Recon Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,recon]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index b03abdac1..2e66734ec 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 92bd59a7a..1604198dc 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 8de85cd8c..0134bf187 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -6,22 +6,45 @@ latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1 --remo bad_app_count=0 -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true + fi else - app_path="." + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed="$(git diff --name-only "$latest_release"...HEAD \ +} + +while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff --name-only "$latest_release"...HEAD \ -- "$app_path/src" \ -- "$app_path/priv" \ - -- "$app_path/c_src" | wc -l)" - if [ "$changed" -gt 0 ]; then - echo "$src_file needs a vsn bump" + -- "$app_path/c_src")" + if [ "$lines" != '' ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" bad_app_count=$(( bad_app_count + 1)) fi fi