diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 47994c9ad..9224a6b82 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -38,7 +38,8 @@ emqx_test(){ packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 + EMQX_MQTT__MAX_TOPIC_ALIAS=10 + [[ $(arch) == *arm* || $(arch) == aarch64 ]] && export EMQX_LISTENER__QUIC__EXTERNAL__ENDPOINT='' # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" @@ -48,7 +49,7 @@ emqx_test(){ exit 1 fi IDLE_TIME=0 - while ! curl http://localhost:8081/status >/dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -113,17 +114,31 @@ emqx_test(){ } running_test(){ - export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins + emqx_env_vars=$(dirname "$(readlink "$(command -v emqx)")")/../releases/emqx_vars - if ! emqx start; then + if [ -f "$emqx_env_vars" ]; + then + tee -a "$emqx_env_vars" </dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -138,14 +153,13 @@ running_test(){ if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \ || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then - if ! service emqx start; then cat /var/log/emqx/erlang.log.1 || true cat /var/log/emqx/emqx.log.1 || true exit 1 fi IDLE_TIME=0 - while ! curl http://localhost:8081/status >/dev/null 2>&1; do + while ! curl http://localhost:8081/api/v5/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx service error" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 99ea45d29..141645043 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -42,6 +42,7 @@ jobs: if: endsWith(github.repository, 'emqx') run: | make -C source deps-all + rm source/rebar.lock zip -ryq source.zip source/* source/.[^.]* - name: get_all_deps if: endsWith(github.repository, 'enterprise') @@ -63,6 +64,7 @@ jobs: if: endsWith(github.repository, 'emqx') strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} exclude: @@ -131,6 +133,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: @@ -183,7 +186,7 @@ jobs: ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do - if curl -fs 127.0.0.1:8081/status > /dev/null; then + if curl -fs 127.0.0.1:8081/api/v5/status > /dev/null; then ready='yes' break fi @@ -210,6 +213,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: @@ -336,6 +340,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 30768e023..162959040 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -112,7 +112,7 @@ jobs: ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do - if curl -fs 127.0.0.1:8081/status > /dev/null; then + if curl -fs 127.0.0.1:8081/api/v5/status > /dev/null; then ready='yes' break fi diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 5f85e6dc2..68463778d 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -16,11 +16,10 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.0"}}} - , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.6"}}} + , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.5"}}} ]}. {plugins, [rebar3_proper]}. @@ -31,7 +30,7 @@ [ meck , {bbmustache,"1.10.0"} , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers", {branch,"hocon"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.1"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.2"}}} ]}, {extra_src_dirs, [{"test",[recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index eae18f106..2352e4a81 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -1,11 +1,30 @@ -Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}, -AddBcrypt = fun(C) -> - {deps, Deps0} = lists:keyfind(deps, 1, C), - Deps = [Bcrypt | Deps0], - lists:keystore(deps, 1, C, {deps, Deps}) -end, +IsCentos6 = fun() -> + case file:read_file("/etc/centos-release") of + {ok, <<"CentOS release 6", _/binary >>} -> + true; + _ -> + false + end + end, -case os:type() of - {win32, _} -> CONFIG; - _ -> AddBcrypt(CONFIG) -end. +IsWin32 = fun() -> + win32 =:= element(1, os:type()) + end, + +IsQuicSupp = fun() -> + not (IsCentos6() orelse IsWin32() orelse + false =/= os:getenv("EMQX_BUILD_WITHOUT_QUIC") + ) + end, + +Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}, +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {branch, "main"}}}, + +ExtraDeps = fun(C) -> + {deps, Deps0} = lists:keyfind(deps, 1, C), + Deps = Deps0 ++ [Bcrypt || not IsWin32()] ++ + [ Quicer || IsQuicSupp()], + lists:keystore(deps, 1, C, {deps, Deps}) + end, + +ExtraDeps(CONFIG). diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index d9efbe82a..546b70f14 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -4,7 +4,7 @@ {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, - {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,quicer,jiffy]}, + {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,jiffy]}, {mod, {emqx_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 666a704f3..d2f5ba691 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -49,6 +49,8 @@ start(_Type, _Args) -> _ = load_ce_modules(), ekka:start(), ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity), + false == os:getenv("EMQX_NO_QUIC") + andalso application:ensure_all_started(quicer), {ok, Sup} = emqx_sup:start_link(), ok = start_autocluster(), % ok = emqx_plugins:init(), diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ad9542958..c24b790a3 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -40,8 +40,6 @@ , list_users/2 ]). --import(minirest, [return/1]). - -rest_api(#{name => create_chain, method => 'POST', path => "/authentication/chains", @@ -542,3 +540,7 @@ get_missed_params(Actual, Expected) -> end end, [], Expected), lists:reverse(Keys). + +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl new file mode 100644 index 000000000..692ff924e --- /dev/null +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -0,0 +1,288 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_http). + +-include("emqx_authn.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ structs/0 + , fields/1 + , validations/0 + ]). + +-type accept() :: 'application/json' | 'application/x-www-form-urlencoded'. +-type content_type() :: accept(). + +-reflect_type([ accept/0 + , content_type/0 + ]). + +-export([ create/3 + , update/4 + , authenticate/2 + , destroy/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +structs() -> [""]. + +fields("") -> + [ {config, #{type => hoconsc:union( + [ hoconsc:ref(?MODULE, get) + , hoconsc:ref(?MODULE, post) + ])}} + ]; + +fields(get) -> + [ {method, #{type => get, + default => get}} + ] ++ common_fields(); + +fields(post) -> + [ {method, #{type => post, + default => get}} + , {content_type, fun content_type/1} + ] ++ common_fields(). + +common_fields() -> + [ {url, fun url/1} + , {accept, fun accept/1} + , {headers, fun headers/1} + , {form_data, fun form_data/1} + , {request_timeout, fun request_timeout/1} + ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). + +validations() -> + [ {check_ssl_opts, fun emqx_connector_http:check_ssl_opts/1} ]. + +url(type) -> binary(); +url(nullable) -> false; +url(validate) -> [fun check_url/1]; +url(_) -> undefined. + +accept(type) -> accept(); +accept(default) -> 'application/json'; +accept(_) -> undefined. + +content_type(type) -> content_type(); +content_type(default) -> 'application/json'; +content_type(_) -> undefined. + +headers(type) -> list(); +headers(default) -> []; +headers(_) -> undefined. + +form_data(type) -> binary(); +form_data(nullable) -> false; +form_data(validate) -> [fun check_form_data/1]; +form_data(_) -> undefined. + +request_timeout(type) -> non_neg_integer(); +request_timeout(default) -> 5000; +request_timeout(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(ChainID, AuthenticatorName, + #{method := Method, + url := URL, + accept := Accept, + content_type := ContentType, + headers := Headers, + form_data := FormData, + request_timeout := RequestTimeout} = Config) -> + NHeaders = maps:merge(#{<<"accept">> => atom_to_binary(Accept, utf8), + <<"content-type">> => atom_to_binary(ContentType, utf8)}, Headers), + NFormData = preprocess_form_data(FormData), + #{path := Path, + query := Query} = URIMap = parse_url(URL), + BaseURL = generate_base_url(URIMap), + State = #{method => Method, + path => Path, + base_query => cow_qs:parse_qs(Query), + accept => Accept, + content_type => ContentType, + headers => NHeaders, + form_data => NFormData, + request_timeout => RequestTimeout}, + ResourceID = <>, + case emqx_resource:create_local(ResourceID, emqx_connector_http, Config#{base_url := BaseURL}) of + {ok, _} -> + {ok, State#{resource_id => ResourceID}}; + {error, already_created} -> + {ok, State#{resource_id => ResourceID}}; + {error, Reason} -> + {error, Reason} + end. + +update(_ChainID, _AuthenticatorName, Config, #{resource_id := ResourceID} = State) -> + case emqx_resource:update_local(ResourceID, emqx_connector_http, Config, []) of + {ok, _} -> {ok, State}; + {error, Reason} -> {error, Reason} + end. + +authenticate(ClientInfo, #{resource_id := ResourceID, + method := Method, + request_timeout := RequestTimeout} = State) -> + Request = generate_request(ClientInfo, State), + case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of + {ok, 204, _Headers} -> ok; + {ok, 200, Headers, Body} -> + ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), + case safely_parse_body(ContentType, Body) of + {ok, _NBody} -> + %% TODO: Return by user property + ok; + {error, Reason} -> + {stop, Reason} + end; + {error, _Reason} -> + ignore + end. + +destroy(#{resource_id := ResourceID}) -> + _ = emqx_resource:remove_local(ResourceID), + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +check_url(URL) -> + case emqx_http_lib:uri_parse(URL) of + {ok, _} -> true; + {error, _} -> false + end. + +check_form_data(FormData) -> + KVs = binary:split(FormData, [<<"&">>], [global]), + case false =:= lists:any(fun(T) -> T =:= <<>> end, KVs) of + true -> + NKVs = [list_to_tuple(binary:split(KV, [<<"=">>], [global])) || KV <- KVs], + false =:= + lists:any(fun({K, V}) -> + K =:= <<>> orelse V =:= <<>>; + (_) -> + true + end, NKVs); + false -> + false + end. + +preprocess_form_data(FormData) -> + KVs = binary:split(FormData, [<<"&">>], [global]), + [list_to_tuple(binary:split(KV, [<<"=">>], [global])) || KV <- KVs]. + +parse_url(URL) -> + {ok, URIMap} = emqx_http_lib:uri_parse(URL), + case maps:get(query, URIMap, undefined) of + undefined -> + URIMap#{query => ""}; + _ -> + URIMap + end. + +generate_base_url(#{scheme := Scheme, + host := Host, + port := Port}) -> + iolist_to_binary(io_lib:format("~p://~s:~p", [Scheme, Host, Port])). + +generate_request(ClientInfo, #{method := Method, + path := Path, + base_query := BaseQuery, + content_type := ContentType, + headers := Headers, + form_data := FormData0}) -> + FormData = replace_placeholders(FormData0, ClientInfo), + case Method of + get -> + NPath = append_query(Path, BaseQuery ++ FormData), + {NPath, Headers}; + post -> + NPath = append_query(Path, BaseQuery), + Body = serialize_body(ContentType, FormData), + {NPath, Headers, Body} + end. + +replace_placeholders(FormData0, ClientInfo) -> + FormData = lists:map(fun({K, V0}) -> + case replace_placeholder(V0, ClientInfo) of + undefined -> {K, undefined}; + V -> {K, bin(V)} + end + end, FormData0), + lists:filter(fun({_, V}) -> + V =/= undefined + end, FormData). + +replace_placeholder(<<"${mqtt-username}">>, ClientInfo) -> + maps:get(username, ClientInfo, undefined); +replace_placeholder(<<"${mqtt-clientid}">>, ClientInfo) -> + maps:get(clientid, ClientInfo, undefined); +replace_placeholder(<<"${ip-address}">>, ClientInfo) -> + maps:get(peerhost, ClientInfo, undefined); +replace_placeholder(<<"${cert-subject}">>, ClientInfo) -> + maps:get(dn, ClientInfo, undefined); +replace_placeholder(<<"${cert-common-name}">>, ClientInfo) -> + maps:get(cn, ClientInfo, undefined); +replace_placeholder(Constant, _) -> + Constant. + +append_query(Path, []) -> + Path; +append_query(Path, Query) -> + Path ++ "?" ++ binary_to_list(qs(Query)). + +qs(KVs) -> + qs(KVs, []). + +qs([], Acc) -> + <<$&, Qs/binary>> = iolist_to_binary(lists:reverse(Acc)), + Qs; +qs([{K, V} | More], Acc) -> + qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). + +serialize_body('application/json', FormData) -> + emqx_json:encode(FormData); +serialize_body('application/x-www-form-urlencoded', FormData) -> + qs(FormData). + +safely_parse_body(ContentType, Body) -> + try parse_body(ContentType, Body) of + Result -> Result + catch + _Class:_Reason -> + {error, invalid_body} + end. + +parse_body(<<"application/json">>, Body) -> + {ok, emqx_json:decode(Body)}; +parse_body(<<"application/x-www-form-urlencoded">>, Body) -> + {ok, cow_qs:parse_qs(Body)}; +parse_body(ContentType, _) -> + {error, {unsupported_content_type, ContentType}}. + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(L) when is_list(L) -> list_to_binary(L); +bin(X) -> X. \ No newline at end of file diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index f737d5168..8fae45ff4 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -294,12 +294,16 @@ do_verify_claims(Claims, [{Name, Value} | More]) -> {error, {claims, {Name, Value0}}} end. -check_verify_claims([]) -> +check_verify_claims(Conf) -> + Claims = hocon_schema:get_value("verify_claims", Conf), + do_check_verify_claims(Claims). + +do_check_verify_claims([]) -> false; -check_verify_claims([{Name, Expected} | More]) -> +do_check_verify_claims([{Name, Expected} | More]) -> check_claim_name(Name) andalso check_claim_expected(Expected) andalso - check_verify_claims(More). + do_check_verify_claims(More). check_claim_name(exp) -> false; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 3b5384d9c..cc4445eaf 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -58,10 +58,11 @@ query_timeout(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ -create(ChainID, ServiceName, #{query := Query0, - password_hash_algorithm := Algorithm} = Config) -> +create(ChainID, AuthenticatorName, + #{query := Query0, + password_hash_algorithm := Algorithm} = Config) -> {Query, PlaceHolders} = parse_query(Query0), - ResourceID = iolist_to_binary(io_lib:format("~s/~s",[ChainID, ServiceName])), + ResourceID = iolist_to_binary(io_lib:format("~s/~s",[ChainID, AuthenticatorName])), State = #{query => Query, placeholders => PlaceHolders, password_hash_algorithm => Algorithm}, @@ -74,7 +75,7 @@ create(ChainID, ServiceName, #{query := Query0, {error, Reason} end. -update(_ChainID, _ServiceName, Config, #{resource_id := ResourceID} = State) -> +update(_ChainID, _AuthenticatorName, Config, #{resource_id := ResourceID} = State) -> case emqx_resource:update_local(ResourceID, emqx_connector_mysql, Config, []) of {ok, _} -> {ok, State}; {error, Reason} -> {error, Reason} diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 725d884e1..78aa47d91 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -64,15 +64,10 @@ create_resource(#{type := DB, config := Config } = Rule) -> ResourceID = iolist_to_binary([io_lib:format("~s_~s",[?APP, DB]), "_", integer_to_list(erlang:system_time())]), - NConfig = case DB of - redis -> #{config => Config }; - mongo -> #{config => Config }; - _ -> Config - end, - case emqx_resource:check_and_create( + case emqx_resource:create( ResourceID, list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), - NConfig) + Config) of {ok, _} -> Rule#{resource_id => ResourceID}; diff --git a/apps/emqx_authz/src/emqx_authz_api.erl b/apps/emqx_authz/src/emqx_authz_api.erl index 08ff0a7d7..99ec2841c 100644 --- a/apps/emqx_authz/src/emqx_authz_api.erl +++ b/apps/emqx_authz/src/emqx_authz_api.erl @@ -53,21 +53,21 @@ ]). lookup_authz(_Bindings, _Params) -> - minirest:return({ok, emqx_authz:lookup()}). + return({ok, emqx_authz:lookup()}). update_authz(_Bindings, Params) -> Rules = get_rules(Params), - minirest:return(emqx_authz:update(Rules)). + return(emqx_authz:update(Rules)). append_authz(_Bindings, Params) -> Rules = get_rules(Params), NRules = lists:append(emqx_authz:lookup(), Rules), - minirest:return(emqx_authz:update(NRules)). + return(emqx_authz:update(NRules)). push_authz(_Bindings, Params) -> Rules = get_rules(Params), NRules = lists:append(Rules, emqx_authz:lookup()), - minirest:return(emqx_authz:update(NRules)). + return(emqx_authz:update(NRules)). %%------------------------------------------------------------------------------ %% Interval Funcs @@ -88,3 +88,7 @@ get_rules(Params) -> -endif. + +return(_) -> +%% TODO: V5 api + ok. \ No newline at end of file diff --git a/apps/emqx_authz/src/emqx_authz_mongo.erl b/apps/emqx_authz/src/emqx_authz_mongo.erl index 04af8f1ec..a32054997 100644 --- a/apps/emqx_authz/src/emqx_authz_mongo.erl +++ b/apps/emqx_authz/src/emqx_authz_mongo.erl @@ -60,8 +60,7 @@ match(Client, PubSub, Topic, <<"permission">> := Permission, <<"action">> := Action }) -> - Rule = #{<<"principal">> => all, - <<"permission">> => Permission, + Rule = #{<<"permission">> => Permission, <<"topics">> => Topics, <<"action">> => Action }, diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index 4c769085d..0ab1418f2 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -77,13 +77,9 @@ format_result(Columns, Row) -> match(Client, PubSub, Topic, #{<<"permission">> := Permission, <<"action">> := Action, - <<"clientid">> := ClientId, - <<"username">> := Username, - <<"ipaddress">> := IpAddress, <<"topic">> := TopicFilter }) -> - Rule = #{<<"principal">> => principal(IpAddress, Username, ClientId), - <<"topics">> => [TopicFilter], + Rule = #{<<"topics">> => [TopicFilter], <<"action">> => Action, <<"permission">> => Permission }, @@ -99,19 +95,6 @@ match(Client, PubSub, Topic, false -> nomatch end. -principal(CIDR, Username, ClientId) -> - Cols = [{<<"ipaddress">>, CIDR}, {<<"username">>, Username}, {<<"clientid">>, ClientId}], - case [#{C => V} || {C, V} <- Cols, not empty(V)] of - [] -> throw(undefined_who); - [Who] -> Who; - Conds -> #{<<"and">> => Conds} - end. - -empty(null) -> true; -empty("") -> true; -empty(<<>>) -> true; -empty(_) -> false. - replvar(Params, ClientInfo) -> replvar(Params, ClientInfo, []). diff --git a/apps/emqx_authz/src/emqx_authz_pgsql.erl b/apps/emqx_authz/src/emqx_authz_pgsql.erl index d74db36b2..c990a29d3 100644 --- a/apps/emqx_authz/src/emqx_authz_pgsql.erl +++ b/apps/emqx_authz/src/emqx_authz_pgsql.erl @@ -81,13 +81,9 @@ format_result(Columns, Row) -> match(Client, PubSub, Topic, #{<<"permission">> := Permission, <<"action">> := Action, - <<"clientid">> := ClientId, - <<"username">> := Username, - <<"ipaddress">> := IpAddress, <<"topic">> := TopicFilter }) -> - Rule = #{<<"principal">> => principal(IpAddress, Username, ClientId), - <<"topics">> => [TopicFilter], + Rule = #{<<"topics">> => [TopicFilter], <<"action">> => Action, <<"permission">> => Permission }, @@ -103,19 +99,6 @@ match(Client, PubSub, Topic, false -> nomatch end. -principal(CIDR, Username, ClientId) -> - Cols = [{<<"ipaddress">>, CIDR}, {<<"username">>, Username}, {<<"clientid">>, ClientId}], - case [#{C => V} || {C, V} <- Cols, not empty(V)] of - [] -> throw(undefined_who); - [Who] -> Who; - Conds -> #{<<"and">> => Conds} - end. - -empty(null) -> true; -empty("") -> true; -empty(<<>>) -> true; -empty(_) -> false. - replvar(Params, ClientInfo) -> replvar(Params, ClientInfo, []). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 0b6a1d107..f1a79db25 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -16,30 +16,20 @@ structs() -> ["emqx_authz"]. fields("emqx_authz") -> [ {rules, rules()} ]; -fields(mongo_connector) -> - [ {principal, principal()} - , {type, #{type => hoconsc:enum([mongo])}} - , {config, #{type => map()}} - , {collection, #{type => atom()}} +fields(mongo) -> + connector_fields(mongo) ++ + [ {collection, #{type => atom()}} , {find, #{type => map()}} ]; -fields(redis_connector) -> - [ {principal, principal()} - , {type, #{type => hoconsc:enum([redis])}} - , {config, #{type => hoconsc:union( - [ hoconsc:ref(emqx_connector_redis, cluster) - , hoconsc:ref(emqx_connector_redis, sentinel) - , hoconsc:ref(emqx_connector_redis, single) - ])} - } - , {cmd, query()} - ]; -fields(sql_connector) -> - [ {principal, principal() } - , {type, #{type => hoconsc:enum([mysql, pgsql])}} - , {config, #{type => map()}} - , {sql, query()} - ]; +fields(redis) -> + connector_fields(redis) ++ + [ {cmd, query()} ]; +fields(mysql) -> + connector_fields(mysql) ++ + [ {sql, query()} ]; +fields(pgsql) -> + connector_fields(pgsql) ++ + [ {sql, query()} ]; fields(simple_rule) -> [ {permission, #{type => permission()}} , {action, #{type => action()}} @@ -88,9 +78,10 @@ union_array(Item) when is_list(Item) -> rules() -> #{type => union_array( [ hoconsc:ref(?MODULE, simple_rule) - , hoconsc:ref(?MODULE, sql_connector) - , hoconsc:ref(?MODULE, redis_connector) - , hoconsc:ref(?MODULE, mongo_connector) + , hoconsc:ref(?MODULE, mysql) + , hoconsc:ref(?MODULE, pgsql) + , hoconsc:ref(?MODULE, redis) + , hoconsc:ref(?MODULE, mongo) ]) }. @@ -115,3 +106,9 @@ query() -> end end }. + +connector_fields(DB) -> + Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), + [ {principal, principal()} + , {type, #{type => DB}} + ] ++ Mod:fields(""). diff --git a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl index 24683cd5b..789de9fcc 100644 --- a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl @@ -35,7 +35,9 @@ -define(BASE_PATH, "api"). all() -> - emqx_ct:all(?MODULE). +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + []. groups() -> []. @@ -59,9 +61,8 @@ set_special_configs(emqx_authz) -> ok; set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], - default_application_id => <<"admin">>, - default_application_secret => <<"public">>}), + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), ok; set_special_configs(_App) -> diff --git a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl index daf4d1722..d2792e388 100644 --- a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl @@ -30,7 +30,7 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), Config. diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index edc35ca45..a9acf5e36 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -30,7 +30,7 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), Config. diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl index d5f89bcad..03bec2415 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl @@ -30,7 +30,7 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), Config. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 0d7ffa9d8..7530c3183 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -30,7 +30,7 @@ groups() -> init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, check_and_create, fun(_, _, _) -> {ok, meck_data} end ), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ), ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1), Config. diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 143816402..fb299b19b 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -1,4 +1,4 @@ -define(VALID, emqx_resource_validator). --define(REQUIRED(MSG), ?VALID:required(MSG)). +-define(NOT_EMPTY(MSG), ?VALID:not_empty(MSG)). -define(MAX(MAXV), ?VALID:max(number, MAXV)). -define(MIN(MINV), ?VALID:min(number, MINV)). diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index b12bd3edb..9bfbb9277 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -9,7 +9,7 @@ {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version - {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}}, + {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.8"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.7"}}}, %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl new file mode 100644 index 000000000..bef5c26f3 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -0,0 +1,215 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_http). + +-include("emqx_connector.hrl"). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + ]). + +-export([ structs/0 + , fields/1 + , validations/0]). + +-export([ check_ssl_opts/1 ]). + +-type connect_timeout() :: non_neg_integer() | infinity. +-type pool_type() :: random | hash. + +-reflect_type([ connect_timeout/0 + , pool_type/0 + ]). + +%%===================================================================== +%% Hocon schema +structs() -> [""]. + +fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> + [ {base_url, fun base_url/1} + , {connect_timeout, fun connect_timeout/1} + , {max_retries, fun max_retries/1} + , {retry_interval, fun retry_interval/1} + , {keepalive, fun keepalive/1} + , {pool_type, fun pool_type/1} + , {pool_size, fun pool_size/1} + , {ssl_opts, #{type => hoconsc:ref(?MODULE, ssl_opts), + nullable => true}} + ]; + +fields(ssl_opts) -> + [ {cacertfile, fun cacertfile/1} + , {keyfile, fun keyfile/1} + , {certfile, fun certfile/1} + , {verify, fun verify/1} + ]. + +validations() -> + [ {check_ssl_opts, fun check_ssl_opts/1} ]. + +base_url(type) -> binary(); +base_url(nullable) -> false; +base_url(validate) -> [fun check_base_url/1]; +base_url(_) -> undefined. + +connect_timeout(type) -> connect_timeout(); +connect_timeout(default) -> 5000; +connect_timeout(_) -> undefined. + +max_retries(type) -> non_neg_integer(); +max_retries(default) -> 5; +max_retries(_) -> undefined. + +retry_interval(type) -> non_neg_integer(); +retry_interval(default) -> 1000; +retry_interval(_) -> undefined. + +keepalive(type) -> non_neg_integer(); +keepalive(default) -> 5000; +keepalive(_) -> undefined. + +pool_type(type) -> pool_type(); +pool_type(default) -> random; +pool_type(_) -> undefined. + +pool_size(type) -> non_neg_integer(); +pool_size(default) -> 8; +pool_size(_) -> undefined. + +cacertfile(type) -> string(); +cacertfile(nullable) -> true; +cacertfile(_) -> undefined. + +keyfile(type) -> string(); +keyfile(nullable) -> true; +keyfile(_) -> undefined. + +certfile(type) -> string(); +certfile(nullable) -> false; +certfile(_) -> undefined. + +verify(type) -> boolean(); +verify(default) -> false; +verify(_) -> undefined. + +%% =================================================================== +on_start(InstId, #{url := URL, + connect_timeout := ConnectTimeout, + max_retries := MaxRetries, + retry_interval := RetryInterval, + keepalive := Keepalive, + pool_type := PoolType, + pool_size := PoolSize} = Config) -> + logger:info("starting http connector: ~p, config: ~p", [InstId, Config]), + {ok, #{scheme := Scheme, + host := Host, + port := Port, + path := BasePath}} = emqx_http_lib:uri_parse(URL), + {Transport, TransportOpts} = case Scheme of + http -> + {tcp, []}; + https -> + SSLOpts = emqx_plugin_libs_ssl:save_files_return_opts( + maps:get(ssl_opts, Config), "connectors", InstId), + {tls, SSLOpts} + end, + NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + PoolOpts = [ {host, Host} + , {port, Port} + , {connect_timeout, ConnectTimeout} + , {retry, MaxRetries} + , {retry_timeout, RetryInterval} + , {keepalive, Keepalive} + , {pool_type, PoolType} + , {pool_size, PoolSize} + , {transport, Transport} + , {transport, NTransportOpts}], + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), + {ok, #{pool_name => PoolName, + host => Host, + port => Port, + base_path => BasePath}}. + +on_stop(InstId, #{pool_name := PoolName}) -> + logger:info("stopping http connector: ~p", [InstId]), + ehttpc_sup:stop_pool(PoolName). + +on_query(InstId, {Method, Request}, AfterQuery, State) -> + on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State); +on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> + on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); +on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, + base_path := BasePath} = State) -> + logger:debug("http connector ~p received request: ~p, at state: ~p", [InstId, Request, State]), + NRequest = update_path(BasePath, Request), + case Result = ehttpc:request(case KeyOrNum of + undefined -> PoolName; + _ -> {PoolName, KeyOrNum} + end, Method, NRequest, Timeout) of + {error, Reason} -> + logger:debug("http connector ~p do reqeust failed, sql: ~p, reason: ~p", [InstId, NRequest, Reason]), + emqx_resource:query_failed(AfterQuery); + _ -> + emqx_resource:query_success(AfterQuery) + end, + Result. + +on_health_check(_InstId, #{server := {Host, Port}} = State) -> + case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), 3000) of + {ok, Sock} -> + gen_tcp:close(Sock), + {ok, State}; + {error, _Reason} -> + {error, test_query_failed, State} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +check_base_url(URL) -> + case emqx_http_lib:uri_parse(URL) of + {error, _} -> false; + {ok, #{query := _}} -> false; + _ -> true + end. + +check_ssl_opts(Conf) -> + URL = hocon_schema:get_value("url", Conf), + {ok, #{scheme := Scheme}} = emqx_http_lib:uri_parse(URL), + SSLOpts = hocon_schema:get_value("ssl_opts", Conf), + case {Scheme, SSLOpts} of + {http, undefined} -> true; + {http, _} -> false; + {https, undefined} -> false; + {https, _} -> true + end. + +update_path(BasePath, {Path, Headers}) -> + {filename:join(BasePath, Path), Headers}; +update_path(BasePath, {Path, Headers, Body}) -> + {filename:join(BasePath, Path), Headers, Body}. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index b8a3c0da0..daddb7e13 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -78,8 +78,8 @@ mongo_fields() -> [ {pool_size, fun emqx_connector_schema_lib:pool_size/1} , {username, fun emqx_connector_schema_lib:username/1} , {password, fun emqx_connector_schema_lib:password/1} - , {authentication_database, #{type => binary(), - nullable => true}} + , {auth_source, #{type => binary(), + nullable => true}} , {database, fun emqx_connector_schema_lib:database/1} ] ++ emqx_connector_schema_lib:ssl_fields(). @@ -88,24 +88,24 @@ on_jsonify(Config) -> Config. %% =================================================================== -on_start(InstId, #{config := #{server := Server, - mongo_type := single} = Config}) -> +on_start(InstId, Config = #{server := Server, + mongo_type := single}) -> logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), Opts = [{type, single}, {hosts, [Server]} ], do_start(InstId, Opts, Config); -on_start(InstId, #{config := #{servers := Servers, - mongo_type := rs, - replicaset_name := RsName} = Config}) -> +on_start(InstId, Config = #{servers := Servers, + mongo_type := rs, + replicaset_name := RsName}) -> logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), Opts = [{type, {rs, RsName}}, {hosts, Servers}], do_start(InstId, Opts, Config); -on_start(InstId, #{config := #{servers := Servers, - mongo_type := sharded} = Config}) -> +on_start(InstId, Config = #{servers := Servers, + mongo_type := sharded}) -> logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), Opts = [{type, sharded}, {hosts, Servers} @@ -218,7 +218,7 @@ init_topology_options([], Acc) -> init_worker_options([{database, V} | R], Acc) -> init_worker_options(R, [{database, V} | Acc]); -init_worker_options([{authentication_database, V} | R], Acc) -> +init_worker_options([{auth_source, V} | R], Acc) -> init_worker_options(R, [{auth_source, V} | Acc]); init_worker_options([{username, V} | R], Acc) -> init_worker_options(R, [{login, V} | Acc]); @@ -243,11 +243,11 @@ host_port(HostPort) -> end. server(type) -> server(); -server(validator) -> [?REQUIRED("the field 'server' is required")]; +server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(_) -> undefined. servers(type) -> hoconsc:array(server()); -servers(validator) -> [?REQUIRED("the field 'servers' is required")]; +servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; servers(_) -> undefined. duration(type) -> emqx_schema:duration_ms(); diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index a606bb82d..6a5d93ca2 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -37,6 +37,9 @@ structs() -> [""]. fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index ddcc2a7c7..e89ab7401 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -38,6 +38,9 @@ structs() -> [""]. fields("") -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]; + +fields(config) -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 0df12185d..1ea31ced8 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -78,11 +78,11 @@ on_jsonify(Config) -> Config. %% =================================================================== -on_start(InstId, #{config :=#{redis_type := Type, - database := Database, - pool_size := PoolSize, - auto_reconnect := AutoReconn, - ssl := SSL } = Config}) -> +on_start(InstId, #{redis_type := Type, + database := Database, + pool_size := PoolSize, + auto_reconnect := AutoReconn, + ssl := SSL } = Config) -> logger:info("starting redis connector: ~p, config: ~p", [InstId, Config]), Servers = case Type of single -> [{servers, [maps:get(server, Config)]}]; diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 743d37ae3..7dcf24be5 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -86,11 +86,13 @@ relational_db_fields() -> ]. server(type) -> emqx_schema:ip_port(); -server(validator) -> [?REQUIRED("the field 'server' is required")]; +server(nullable) -> false; +server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(_) -> undefined. database(type) -> binary(); -database(validator) -> [?REQUIRED("the field 'database' is required")]; +database(nullable) -> false; +database(validator) -> [?NOT_EMPTY("the value of the field 'database' cannot be empty")]; database(_) -> undefined. pool_size(type) -> integer(); @@ -127,7 +129,7 @@ verify(default) -> false; verify(_) -> undefined. servers(type) -> servers(); -servers(validator) -> [?REQUIRED("the field 'servers' is required")]; +servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; servers(_) -> undefined. to_ip_port(Str) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 0390339d3..8e81b979f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -19,7 +19,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --import(proplists, [get_value/3]). +%%-import(proplists, [get_value/3]). -export([ start_listeners/0 , stop_listeners/0 @@ -42,56 +42,61 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). %% Start HTTP Listener -start_listener({Proto, Port, Options}) when Proto == http -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); - -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). - -ranch_opts(Port, Options0) -> - NumAcceptors = get_value(num_acceptors, Options0, 4), - MaxConnections = get_value(max_connections, Options0, 512), - Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> - Acc; - ({inet6, true}, Acc) -> [inet6 | Acc]; - ({inet6, false}, Acc) -> Acc; - ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; - ({ipv6_v6only, false}, Acc) -> Acc; - ({K, V}, Acc)-> - [{K, V} | Acc] - end, [], Options0), - #{num_acceptors => NumAcceptors, - max_connections => MaxConnections, - socket_opts => [{port, Port} | Options]}. +start_listener(_) -> ok. +%% TODO: V5 API +%%start_listener({Proto, Port, Options}) when Proto == http -> +%% Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, +%% {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, +%% {"/api/v4/[...]", minirest, http_handlers()}], +%% minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); +%% +%%start_listener({Proto, Port, Options}) when Proto == https -> +%% Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, +%% {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, +%% {"/api/v4/[...]", minirest, http_handlers()}], +%% minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). +%% +%%ranch_opts(Port, Options0) -> +%% NumAcceptors = get_value(num_acceptors, Options0, 4), +%% MaxConnections = get_value(max_connections, Options0, 512), +%% Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> +%% Acc; +%% ({inet6, true}, Acc) -> [inet6 | Acc]; +%% ({inet6, false}, Acc) -> Acc; +%% ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; +%% ({ipv6_v6only, false}, Acc) -> Acc; +%% ({K, V}, Acc)-> +%% [{K, V} | Acc] +%% end, [], Options0), +%% #{num_acceptors => NumAcceptors, +%% max_connections => MaxConnections, +%% socket_opts => [{port, Port} | Options]}. stop_listeners() -> lists:foreach(fun(Listener) -> stop_listener(Listener) end, listeners()). -stop_listener({Proto, _Port, _}) -> - minirest:stop_http(listener_name(Proto)). +stop_listener(_) -> + ok. +%% TODO: V5 API +%%stop_listener({Proto, _Port, _}) -> +%% minirest:stop_http(listener_name(Proto)). listeners() -> application:get_env(?APP, listeners, []). -listener_name(Proto) -> - list_to_atom(atom_to_list(Proto) ++ ":dashboard"). +%%listener_name(Proto) -> +%% list_to_atom(atom_to_list(Proto) ++ ":dashboard"). %%-------------------------------------------------------------------- %% HTTP Handlers and Dispatcher %%-------------------------------------------------------------------- -http_handlers() -> - Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules], - filter => fun ?MODULE:filter/1}), - [{authorization, fun ?MODULE:is_authorized/1}]}]. +%%http_handlers() -> +%% Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), +%% [{"/api/v4/", +%% minirest:handler(#{apps => Plugins ++ [emqx_modules], +%% filter => fun ?MODULE:filter/1}), +%% [{authorization, fun ?MODULE:is_authorized/1}]}]. %%-------------------------------------------------------------------- %% Basic Authorization diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index e1c89efbb..653380ab6 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -18,8 +18,6 @@ -include("emqx_dashboard.hrl"). --import(minirest, [return/1]). - -rest_api(#{name => auth_user, method => 'POST', path => "/auth", @@ -107,3 +105,6 @@ delete(#{name := Username}, _Params) -> row(#mqtt_admin{username = Username, tags = Tags}) -> #{username => Username, tags => Tags}. +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 3ea8ab743..1ffb6786e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -40,7 +40,9 @@ -define(OVERVIEWS, ['alarms/activated', 'alarms/deactivated', banned, brokers, stats, metrics, listeners, clients, subscriptions, routes, plugins]). all() -> - emqx_ct:all(?MODULE). +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + []. init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), @@ -51,9 +53,8 @@ end_per_suite(_Config) -> ekka_mnesia:ensure_stopped(). set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], - default_application_id => <<"admin">>, - default_application_secret => <<"public">>}), + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), ok; set_special_configs(_) -> ok. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index b4749af0e..066d72096 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -5,11 +5,6 @@ %%====================================================================================== %% Hocon Schema Definitions --define(BRIDGE_FIELDS(T), - [{name, hoconsc:t(typerefl:binary())}, - {type, hoconsc:t(typerefl:atom(T))}, - {config, hoconsc:t(hoconsc:ref(list_to_atom("emqx_connector_"++atom_to_list(T)), ""))}]). - -define(TYPES, [mysql, pgsql, mongo, redis, ldap]). -define(BRIDGES, [hoconsc:ref(?MODULE, T) || T <- ?TYPES]). @@ -19,8 +14,13 @@ fields("emqx_data_bridge") -> [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), default => []}}]; -fields(mysql) -> ?BRIDGE_FIELDS(mysql); -fields(pgsql) -> ?BRIDGE_FIELDS(pgsql); -fields(mongo) -> ?BRIDGE_FIELDS(mongo); -fields(redis) -> ?BRIDGE_FIELDS(redis); -fields(ldap) -> ?BRIDGE_FIELDS(ldap). +fields(mysql) -> connector_fields(mysql); +fields(pgsql) -> connector_fields(pgsql); +fields(mongo) -> connector_fields(mongo); +fields(redis) -> connector_fields(redis); +fields(ldap) -> connector_fields(ldap). + +connector_fields(DB) -> + Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), + [{name, hoconsc:t(typerefl:binary())}, + {type, #{type => DB}}] ++ Mod:fields(""). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl index 6018aa7c7..80449238c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl @@ -16,8 +16,6 @@ -module(emqx_lwm2m_api). --import(minirest, [return/1]). - -rest_api(#{name => list, method => 'GET', path => "/lwm2m_channels/", @@ -160,3 +158,7 @@ path_list(Path) -> [ObjId, ObjInsId] -> [ObjId, ObjInsId]; [ObjId] -> [ObjId] end. + +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index c71a47628..52013c025 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -7,3 +7,6 @@ EMQ X Management API http://restful-api-design.readthedocs.io/en/latest/scope.html +default application see: +header: +authorization: Basic YWRtaW46cHVibGlj diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index a4a64b755..127a21e3b 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -1,12 +1,16 @@ emqx_management:{ - default_application_id: "admin" - default_application_secret: "public" + applications: [ + { + id: "admin", + secret: "public" + } + ] max_row_limit: 10000 listeners: [ { num_acceptors: 4 max_connections: 512 - protocol: "http" + protocol: http port: 8081 backlog: 512 send_timeout: 15s diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index b952332c5..40baec4e1 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -35,3 +35,29 @@ -define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). -define(MANAGEMENT_SHARD, emqx_management_shard). + +-define(GENERATE_API_METADATA(MetaData), + maps:fold( + fun(Method, MethodDef0, NextMetaData) -> + Default = #{ + tags => [?MODULE], + security => [#{application => []}]}, + MethodDef = + lists:foldl( + fun(Key, NMethodDef) -> + case maps:is_key(Key, NMethodDef) of + true -> + NMethodDef; + false -> + maps:put(Key, maps:get(Key, Default), NMethodDef) + end + end, MethodDef0, maps:keys(Default)), + maps:put(Method, MethodDef, NextMetaData) + end, + #{}, MetaData)). + +-define(GENERATE_API(Path, MetaData, Function), + {Path, ?GENERATE_API_METADATA(MetaData), Function}). + +-define(GENERATE_APIS(Apis), + [?GENERATE_API(Path, MetaData, Function) || {Path, MetaData, Function} <- Apis]). diff --git a/apps/emqx_management/src/emqx_management_schema.erl b/apps/emqx_management/src/emqx_management_schema.erl index ae6b6365d..f9543697f 100644 --- a/apps/emqx_management/src/emqx_management_schema.erl +++ b/apps/emqx_management/src/emqx_management_schema.erl @@ -20,19 +20,24 @@ -behaviour(hocon_schema). -export([ structs/0 - , fields/1]). + , fields/1]). structs() -> ["emqx_management"]. fields("emqx_management") -> - [ {default_application_id, fun default_application_id/1} - , {default_application_secret, fun default_application_secret/1} + [ {applications, hoconsc:array(hoconsc:ref(?MODULE, "application"))} , {max_row_limit, fun max_row_limit/1} , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), hoconsc:ref(?MODULE, "https")]))} ]; +fields("application") -> + [ {"id", emqx_schema:t(string(), undefined, "admin")} + , {"secret", emqx_schema:t(string(), undefined, "public")} + ]; + + fields("http") -> - [ {"protocol", emqx_schema:t(string(), undefined, "http")} + [ {"protocol", hoconsc:enum([http, https])} , {"port", emqx_schema:t(integer(), undefined, 8081)} , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)} , {"max_connections", emqx_schema:t(integer(), undefined, 512)} @@ -46,16 +51,6 @@ fields("http") -> fields("https") -> emqx_schema:ssl(#{enable => true}) ++ fields("http"). -default_application_id(type) -> string(); -default_application_id(default) -> "admin"; -default_application_id(nullable) -> true; -default_application_id(_) -> undefined. - -default_application_secret(type) -> string(); -default_application_secret(default) -> "public"; -default_application_secret(nullable) -> true; -default_application_secret(_) -> undefined. - max_row_limit(type) -> integer(); max_row_limit(default) -> 1000; max_row_limit(nullable) -> false; diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 0183785cc..bcd9ee1aa 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -106,10 +106,19 @@ , max_row_limit/0 ]). +-export([ return/0 + , return/1]). + -define(MAX_ROW_LIMIT, 10000). -define(APP, emqx_management). +%% TODO: remove these function after all api use minirest version 1.X +return() -> + ok. +return(_Response) -> + ok. + %%-------------------------------------------------------------------- %% Node Info %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_acl.erl b/apps/emqx_management/src/emqx_mgmt_api_acl.erl index 039b4035a..025a2263b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_acl.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_acl.erl @@ -36,12 +36,12 @@ clean_all(_Bindings, _Params) -> case emqx_mgmt:clean_acl_cache_all() of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, ?ERROR1, Reason}) end. clean_node(#{node := Node}, _Params) -> case emqx_mgmt:clean_acl_cache_all(Node) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, ?ERROR1, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index d8a0f25dc..9b9e4a4a6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -125,13 +125,13 @@ get_name(Params) -> binary_to_atom(proplists:get_value(<<"name">>, Params, undefined), utf8). do_deactivate(undefined, _) -> - minirest:return({error, missing_param}); + emqx_mgmt:return({error, missing_param}); do_deactivate(_, undefined) -> - minirest:return({error, missing_param}); + emqx_mgmt:return({error, missing_param}); do_deactivate(Node, Name) -> case emqx_mgmt:deactivate(Node, Name) of ok -> - minirest:return(); + emqx_mgmt:return(); {error, Reason} -> - minirest:return({error, Reason}) + emqx_mgmt:return({error, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl index cca0b41f0..fa6bfeeb3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_apps.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_apps.erl @@ -63,30 +63,30 @@ add_app(_Bindings, Params) -> Status = proplists:get_value(<<"status">>, Params), Expired = proplists:get_value(<<"expired">>, Params), case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of - {ok, AppSecret} -> minirest:return({ok, #{secret => AppSecret}}); - {error, Reason} -> minirest:return({error, Reason}) + {ok, AppSecret} -> emqx_mgmt:return({ok, #{secret => AppSecret}}); + {error, Reason} -> emqx_mgmt:return({error, Reason}) end. del_app(#{appid := AppId}, _Params) -> case emqx_mgmt_auth:del_app(AppId) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, Reason}) + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, Reason}) end. list_apps(_Bindings, _Params) -> - minirest:return({ok, [format(Apps)|| Apps <- emqx_mgmt_auth:list_apps()]}). + emqx_mgmt:return({ok, [format(Apps)|| Apps <- emqx_mgmt_auth:list_apps()]}). lookup_app(#{appid := AppId}, _Params) -> case emqx_mgmt_auth:lookup_app(AppId) of {AppId, AppSecret, Name, Desc, Status, Expired} -> - minirest:return({ok, #{app_id => AppId, + emqx_mgmt:return({ok, #{app_id => AppId, secret => AppSecret, name => Name, desc => Desc, status => Status, expired => Expired}}); undefined -> - minirest:return({ok, #{}}) + emqx_mgmt:return({ok, #{}}) end. update_app(#{appid := AppId}, Params) -> @@ -95,8 +95,8 @@ update_app(#{appid := AppId}, Params) -> Status = proplists:get_value(<<"status">>, Params), Expired = proplists:get_value(<<"expired">>, Params), case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of - ok -> minirest:return(); - {error, Reason} -> minirest:return({error, Reason}) + ok -> emqx_mgmt:return(); + {error, Reason} -> emqx_mgmt:return({error, Reason}) end. format({AppId, _AppSecret, Name, Desc, Status, Expired}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index b92875d9e..bdd43b35c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -44,7 +44,7 @@ ]). list(_Bindings, Params) -> - minirest:return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). + emqx_mgmt:return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). create(_Bindings, Params) -> case pipeline([fun ensure_required/1, @@ -52,9 +52,9 @@ create(_Bindings, Params) -> {ok, NParams} -> {ok, Banned} = pack_banned(NParams), ok = emqx_mgmt:create_banned(Banned), - minirest:return({ok, maps:from_list(Params)}); + emqx_mgmt:return({ok, maps:from_list(Params)}); {error, Code, Message} -> - minirest:return({error, Code, Message}) + emqx_mgmt:return({error, Code, Message}) end. delete(#{as := As, who := Who}, _) -> @@ -64,9 +64,9 @@ delete(#{as := As, who := Who}, _) -> fun validate_params/1], Params) of {ok, NParams} -> do_delete(proplists:get_value(<<"as">>, NParams), proplists:get_value(<<"who">>, NParams)), - minirest:return(); + emqx_mgmt:return(); {error, Code, Message} -> - minirest:return({error, Code, Message}) + emqx_mgmt:return({error, Code, Message}) end. pipeline([], Params) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl index bd901a3fe..836f097cb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl @@ -35,13 +35,13 @@ ]). list(_Bindings, _Params) -> - minirest:return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). + emqx_mgmt:return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). get(#{node := Node}, _Params) -> case emqx_mgmt:lookup_broker(Node) of {error, Reason} -> - minirest:return({error, ?ERROR2, Reason}); + emqx_mgmt:return({error, ?ERROR2, Reason}); Info -> - minirest:return({ok, Info}) + emqx_mgmt:return({ok, Info}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 2fe6a5ccb..74cd995a4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -16,308 +16,512 @@ -module(emqx_mgmt_api_clients). --include("emqx_mgmt.hrl"). +-behavior(minirest_api). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx.hrl"). --define(CLIENT_QS_SCHEMA, {emqx_channel_info, - [{<<"clientid">>, binary}, - {<<"username">>, binary}, - {<<"zone">>, atom}, - {<<"ip_address">>, ip}, - {<<"conn_state">>, atom}, - {<<"clean_start">>, atom}, - {<<"proto_name">>, binary}, - {<<"proto_ver">>, integer}, - {<<"_like_clientid">>, binary}, - {<<"_like_username">>, binary}, - {<<"_gte_created_at">>, timestamp}, - {<<"_lte_created_at">>, timestamp}, - {<<"_gte_connected_at">>, timestamp}, - {<<"_lte_connected_at">>, timestamp}]}). +-include_lib("emqx/include/logger.hrl"). --rest_api(#{name => list_clients, - method => 'GET', - path => "/clients/", - func => list, - descr => "A list of clients on current node"}). +-include("emqx_mgmt.hrl"). --rest_api(#{name => list_node_clients, - method => 'GET', - path => "nodes/:atom:node/clients/", - func => list, - descr => "A list of clients on specified node"}). +%% API +-export([api_spec/0]). --rest_api(#{name => lookup_client, - method => 'GET', - path => "/clients/:bin:clientid", - func => lookup, - descr => "Lookup a client in the cluster"}). - --rest_api(#{name => lookup_node_client, - method => 'GET', - path => "nodes/:atom:node/clients/:bin:clientid", - func => lookup, - descr => "Lookup a client on the node"}). - --rest_api(#{name => lookup_client_via_username, - method => 'GET', - path => "/clients/username/:bin:username", - func => lookup, - descr => "Lookup a client via username in the cluster" - }). - --rest_api(#{name => lookup_node_client_via_username, - method => 'GET', - path => "/nodes/:atom:node/clients/username/:bin:username", - func => lookup, - descr => "Lookup a client via username on the node " - }). - --rest_api(#{name => kickout_client, - method => 'DELETE', - path => "/clients/:bin:clientid", - func => kickout, - descr => "Kick out the client in the cluster"}). - --rest_api(#{name => clean_acl_cache, - method => 'DELETE', - path => "/clients/:bin:clientid/acl_cache", - func => clean_acl_cache, - descr => "Clear the ACL cache of a specified client in the cluster"}). - --rest_api(#{name => list_acl_cache, - method => 'GET', - path => "/clients/:bin:clientid/acl_cache", - func => list_acl_cache, - descr => "List the ACL cache of a specified client in the cluster"}). - --rest_api(#{name => set_ratelimit_policy, - method => 'POST', - path => "/clients/:bin:clientid/ratelimit", - func => set_ratelimit_policy, - descr => "Set the client ratelimit policy"}). - --rest_api(#{name => clean_ratelimit, - method => 'DELETE', - path => "/clients/:bin:clientid/ratelimit", - func => clean_ratelimit, - descr => "Clear the ratelimit policy"}). - --rest_api(#{name => set_quota_policy, - method => 'POST', - path => "/clients/:bin:clientid/quota", - func => set_quota_policy, - descr => "Set the client quota policy"}). - --rest_api(#{name => clean_quota, - method => 'DELETE', - path => "/clients/:bin:clientid/quota", - func => clean_quota, - descr => "Clear the quota policy"}). - --import(emqx_mgmt_util, [ ntoa/1 - , strftime/1 - ]). - --export([ list/2 - , lookup/2 - , kickout/2 - , clean_acl_cache/2 - , list_acl_cache/2 - , set_ratelimit_policy/2 - , set_quota_policy/2 - , clean_ratelimit/2 - , clean_quota/2 - ]). +-export([ clients/2 + , client/2 + , acl_cache/2 + , subscribe/2 + , subscribe_batch/2]). -export([ query/3 - , format_channel_info/1 - ]). + , format_channel_info/1]). + +%% for batch operation +-export([do_subscribe/3]). + +-define(CLIENT_QS_SCHEMA, {emqx_channel_info, + [ {<<"clientid">>, binary} + , {<<"username">>, binary} + , {<<"zone">>, atom} + , {<<"ip_address">>, ip} + , {<<"conn_state">>, atom} + , {<<"clean_start">>, atom} + , {<<"proto_name">>, binary} + , {<<"proto_ver">>, integer} + , {<<"_like_clientid">>, binary} + , {<<"_like_username">>, binary} + , {<<"_gte_created_at">>, timestamp} + , {<<"_lte_created_at">>, timestamp} + , {<<"_gte_connected_at">>, timestamp} + , {<<"_lte_connected_at">>, timestamp}]}). -define(query_fun, {?MODULE, query}). -define(format_fun, {?MODULE, format_channel_info}). -list(Bindings, Params) when map_size(Bindings) == 0 -> - fence(fun() -> - emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun) - end); +-define(CLIENT_ID_NOT_FOUND, + <<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">>). -list(#{node := Node}, Params) when Node =:= node() -> - fence(fun() -> - emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun) - end); +api_spec() -> + {apis(), schemas()}. -list(Bindings = #{node := Node}, Params) -> - case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> minirest:return({error, ?ERROR1, Reason}); - Res -> Res +apis() -> + [ clients_api() + , client_api() + , clients_acl_cache_api() + , subscribe_api()]. + +schemas() -> + ClientDef = #{ + <<"node">> => #{ + type => <<"string">>, + description => <<"Name of the node to which the client is connected">>}, + <<"clientid">> => #{ + type => <<"string">>, + description => <<"Client identifier">>}, + <<"username">> => #{ + type => <<"string">>, + description => <<"User name of client when connecting">>}, + <<"proto_name">> => #{ + type => <<"string">>, + description => <<"Client protocol name">>}, + <<"proto_ver">> => #{ + type => <<"integer">>, + description => <<"Protocol version used by the client">>}, + <<"ip_address">> => #{ + type => <<"string">>, + description => <<"Client's IP address">>}, + <<"is_bridge">> => #{ + type => <<"boolean">>, + description => <<"Indicates whether the client is connectedvia bridge">>}, + <<"connected_at">> => #{ + type => <<"string">>, + description => <<"Client connection time">>}, + <<"disconnected_at">> => #{ + type => <<"string">>, + description => <<"Client offline time, This field is only valid and returned when connected is false">>}, + <<"connected">> => #{ + type => <<"boolean">>, + description => <<"Whether the client is connected">>}, + <<"will_msg">> => #{ + type => <<"string">>, + description => <<"Client will message">>}, + <<"zone">> => #{ + type => <<"string">>, + description => <<"Indicate the configuration group used by the client">>}, + <<"keepalive">> => #{ + type => <<"integer">>, + description => <<"keepalive time, with the unit of second">>}, + <<"clean_start">> => #{ + type => <<"boolean">>, + description => <<"Indicate whether the client is using a brand new session">>}, + <<"expiry_interval">> => #{ + type => <<"integer">>, + description => <<"Session expiration interval, with the unit of second">>}, + <<"created_at">> => #{ + type => <<"string">>, + description => <<"Session creation time">>}, + <<"subscriptions_cnt">> => #{ + type => <<"integer">>, + description => <<"Number of subscriptions established by this client.">>}, + <<"subscriptions_max">> => #{ + type => <<"integer">>, + description => <<"v4 api name [max_subscriptions] Maximum number of subscriptions allowed by this client">>}, + <<"inflight_cnt">> => #{ + type => <<"integer">>, + description => <<"Current length of inflight">>}, + <<"inflight_max">> => #{ + type => <<"integer">>, + description => <<"v4 api name [max_inflight]. Maximum length of inflight">>}, + <<"mqueue_len">> => #{ + type => <<"integer">>, + description => <<"Current length of message queue">>}, + <<"mqueue_max">> => #{ + type => <<"integer">>, + description => <<"v4 api name [max_mqueue]. Maximum length of message queue">>}, + <<"mqueue_dropped">> => #{ + type => <<"integer">>, + description => <<"Number of messages dropped by the message queue due to exceeding the length">>}, + <<"awaiting_rel_cnt">> => #{ + type => <<"integer">>, + description => <<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>}, + <<"awaiting_rel_max">> => #{ + type => <<"integer">>, + description => <<"v4 api name [max_awaiting_rel]. Maximum allowed number of awaiting PUBREC packet">>}, + <<"recv_oct">> => #{ + type => <<"integer">>, + description => <<"Number of bytes received by EMQ X Broker (the same below)">>}, + <<"recv_cnt">> => #{ + type => <<"integer">>, + description => <<"Number of TCP packets received">>}, + <<"recv_pkt">> => #{ + type => <<"integer">>, + description => <<"Number of MQTT packets received">>}, + <<"recv_msg">> => #{ + type => <<"integer">>, + description => <<"Number of PUBLISH packets received">>}, + <<"send_oct">> => #{ + type => <<"integer">>, + description => <<"Number of bytes sent">>}, + <<"send_cnt">> => #{ + type => <<"integer">>, + description => <<"Number of TCP packets sent">>}, + <<"send_pkt">> => #{ + type => <<"integer">>, + description => <<"Number of MQTT packets sent">>}, + <<"send_msg">> => #{ + type => <<"integer">>, + description => <<"Number of PUBLISH packets sent">>}, + <<"mailbox_len">> => #{ + type => <<"integer">>, + description => <<"Process mailbox size">>}, + <<"heap_size">> => #{ + type => <<"integer">>, + description => <<"Process heap size with the unit of byte">> + }, + <<"reductions">> => #{ + type => <<"integer">>, + description => <<"Erlang reduction">>}}, + ACLCacheDefinitionProperties = #{ + <<"topic">> => #{ + type => <<"string">>, + description => <<"Topic name">>}, + <<"access">> => #{ + type => <<"string">>, + enum => [<<"subscribe">>, <<"publish">>], + description => <<"Access type">>}, + <<"result">> => #{ + type => <<"string">>, + enum => [<<"allow">>, <<"deny">>], + default => <<"allow">>, + description => <<"Allow or deny">>}, + <<"updated_time">> => #{ + type => <<"integer">>, + description => <<"Update time">>}}, + [{<<"client">>, ClientDef}, {<<"acl_cache">>, ACLCacheDefinitionProperties}]. + +clients_api() -> + Metadata = #{ + get => #{ + description => "List clients", + responses => #{ + <<"200">> => #{ + description => <<"List clients 200 OK">>, + schema => #{ + type => array, + items => minirest:ref(<<"client">>)}}}}}, + {"/clients", Metadata, clients}. + +client_api() -> + Metadata = #{ + get => #{ + description => "Get clients info by client ID", + parameters => [#{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456}], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"Client id not found">>), + <<"200">> => #{ + description => <<"Get clients 200 OK">>, + schema => minirest:ref(<<"client">>)}}}, + delete => #{ + description => "Kick out client by client ID", + parameters => [#{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456}], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"Client id not found">>), + <<"200">> => #{description => <<"Kick out clients OK">>}}}}, + {"/clients/:clientid", Metadata, client}. + +clients_acl_cache_api() -> + Metadata = #{ + get => #{ + description => "Get client acl cache", + parameters => [#{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456}], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"Client id not found">>), + <<"200">> => #{ + description => <<"List 200 OK">>, + schema => minirest:ref(<<"acl_cache">>)}}}, + delete => #{ + description => "Clean client acl cache", + parameters => [#{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456}], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"client id not found">>), + <<"200">> => #{ + description => <<"Clean acl cache 200 OK">>}}}}, + {"/clients/:clientid/acl_cache", Metadata, acl_cache}. + +subscribe_api() -> + Metadata = #{ + post => #{ + description => "subscribe", + parameters => [ + #{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456 + }, + #{ + name => topic_data, + in => body, + schema => #{ + type => object, + properties => #{ + <<"topic">> => #{ + type => <<"string">>, + example => <<"topic_1">>, + description => <<"Topic">>}, + <<"qos">> => #{ + type => <<"integer">>, + enum => [0, 1, 2], + example => 0, + description => <<"QOS">>}}} + } + ], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"Client id not found">>), + <<"200">> => #{description => <<"subscribe ok">>}}}, + delete => #{ + description => "unsubscribe", + parameters => [ + #{ + name => clientid, + in => path, + type => string, + required => true, + default => 123456 + }, + #{ + name => topic, + in => query, + required => true, + default => <<"topic_1">> + } + ], + responses => #{ + <<"404">> => emqx_mgmt_util:not_found_schema(<<"Client id not found">>), + <<"200">> => #{description => <<"unsubscribe ok">>}}}}, + {"/clients/:clientid/subscribe", Metadata, subscribe}. + +%%%============================================================================================== +%% parameters trans +clients(get, _Request) -> + list(#{}). + +client(get, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + lookup(#{clientid => ClientID}); + +client(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + kickout(#{clientid => ClientID}). + +acl_cache(get, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + get_acl_cache(#{clientid => ClientID}); + +acl_cache(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + clean_acl_cache(#{clientid => ClientID}). + +subscribe(post, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + TopicInfo = emqx_json:decode(Body, [return_maps]), + Topic = maps:get(<<"topic">>, TopicInfo), + Qos = maps:get(<<"qos">>, TopicInfo, 0), + subscribe(#{clientid => ClientID, topic => Topic, qos => Qos}); + +subscribe(delete, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + #{topic := Topic} = cowboy_req:match_qs([topic], Request), + unsubscribe(#{clientid => ClientID, topic => Topic}). + +%% TODO: batch +subscribe_batch(post, Request) -> + ClientID = cowboy_req:binding(clientid, Request), + {ok, Body, _} = cowboy_req:read_body(Request), + TopicInfos = emqx_json:decode(Body, [return_maps]), + Topics = + [begin + Topic = maps:get(<<"topic">>, TopicInfo), + Qos = maps:get(<<"qos">>, TopicInfo, 0), + #{topic => Topic, qos => Qos} + end || TopicInfo <- TopicInfos], + subscribe_batch(#{clientid => ClientID, topics => Topics}). + +%%%============================================================================================== +%% api apply + +list(Params) -> + Data = emqx_mgmt_api:cluster_query(maps:to_list(Params), ?CLIENT_QS_SCHEMA, ?query_fun), + Body = emqx_json:encode(Data), + {200, Body}. + +lookup(#{clientid := ClientID}) -> + case emqx_mgmt:lookup_client({clientid, ClientID}, ?format_fun) of + [] -> + {404, ?CLIENT_ID_NOT_FOUND}; + ClientInfo -> + Response = emqx_json:encode(hd(ClientInfo)), + {200, Response} end. -%% @private -fence(Func) -> - try - minirest:return({ok, Func()}) - catch - throw : {bad_value_type, {_Key, Type, Value}} -> - Reason = iolist_to_binary( - io_lib:format("Can't convert ~p to ~p type", - [Value, Type]) - ), - minirest:return({error, ?ERROR8, Reason}) +kickout(#{clientid := ClientID}) -> + emqx_mgmt:kickout_client(ClientID), + {200}. + +get_acl_cache(#{clientid := ClientID})-> + case emqx_mgmt:list_acl_cache(ClientID) of + {error, not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + {500, #{code => <<"UNKNOW_ERROR">>, reason => io_lib:format("~p", [Reason])}}; + Caches -> + Response = emqx_json:encode([format_acl_cache(Cache) || Cache <- Caches]), + {200, Response} end. -lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); - -lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); - -lookup(#{node := Node, username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); - -lookup(#{username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). - -kickout(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +clean_acl_cache(#{clientid := ClientID}) -> + case emqx_mgmt:clean_acl_cache(ClientID) of + ok -> + {200}; + {error, not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + {500, #{code => <<"UNKNOW_ERROR">>, reason => io_lib:format("~p", [Reason])}} end. -clean_acl_cache(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:clean_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) +subscribe(#{clientid := ClientID, topic := Topic, qos := Qos}) -> + case do_subscribe(ClientID, Topic, Qos) of + {error, channel_not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Body = emqx_json:encode(#{code => <<"UNKNOW_ERROR">>, reason => io_lib:format("~p", [Reason])}), + {500, Body}; + ok -> + {200} end. -list_acl_cache(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:list_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}); - Caches -> minirest:return({ok, [format_acl_cache(Cache) || Cache <- Caches]}) +unsubscribe(#{clientid := ClientID, topic := Topic}) -> + case do_unsubscribe(ClientID, Topic) of + {error, channel_not_found} -> + {404, ?CLIENT_ID_NOT_FOUND}; + {error, Reason} -> + Body = emqx_json:encode(#{code => <<"UNKNOW_ERROR">>, reason => io_lib:format("~p", [Reason])}), + {500, Body}; + {unsubscribe, [{Topic, #{}}]} -> + {200} end. -set_ratelimit_policy(#{clientid := ClientId}, Params) -> - P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, - {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> minirest:return(); - Policy -> - case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end - end. +subscribe_batch(#{clientid := ClientID, topics := Topics}) -> + ArgList = [[ClientID, Topic, Qos]|| #{topic := Topic, qos := Qos} <- Topics], + emqx_mgmt_util:batch_operation(?MODULE, do_subscribe, ArgList). -clean_ratelimit(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end. +%%%============================================================================================== +%% internal function +format_channel_info({_, ClientInfo, ClientStats}) -> + Fun = + fun + (_Key, Value, Current) when is_map(Value) -> + maps:merge(Current, Value); + (Key, Value, Current) -> + maps:put(Key, Value, Current) + end, + StatsMap = maps:without([memory, next_pkt_id, total_heap_size], + maps:from_list(ClientStats)), + ClientInfoMap0 = maps:fold(Fun, #{}, ClientInfo), + IpAddress = peer_to_binary(maps:get(peername, ClientInfoMap0)), + Connected = maps:get(conn_state, ClientInfoMap0) =:= connected, + ClientInfoMap1 = maps:merge(StatsMap, ClientInfoMap0), + ClientInfoMap2 = maps:put(node, node(), ClientInfoMap1), + ClientInfoMap3 = maps:put(ip_address, IpAddress, ClientInfoMap2), + ClientInfoMap = maps:put(connected, Connected, ClientInfoMap3), + RemoveList = [ + auth_result + , peername + , sockname + , peerhost + , conn_state + , send_pend + , conn_props + , peercert + , sockstate + , receive_maximum + , protocol + , is_superuser + , sockport + , anonymous + , mountpoint + , socktype + , active_n + , await_rel_timeout + , conn_mod + , sockname + , retry_interval + , upgrade_qos + ], + maps:without(RemoveList, ClientInfoMap). -set_quota_policy(#{clientid := ClientId}, Params) -> - P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> minirest:return(); - Policy -> - case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end - end. - -clean_quota(#{clientid := ClientId}, _Params) -> - case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> minirest:return(); - {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); - {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) - end. - -%% @private -%% S = 100,1s -%% | 100KB, 1m -parse_ratelimit_str(S) when is_binary(S) -> - parse_ratelimit_str(binary_to_list(S)); -parse_ratelimit_str(S) -> - [L, D] = string:tokens(S, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason1} -> error(Reason1) - end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason} -> error(Reason) - end, - {Limit, Duration}. - -%%-------------------------------------------------------------------- -%% Format - -format_channel_info({_Key, Info, Stats0}) -> - Stats = maps:from_list(Stats0), - ClientInfo = maps:get(clientinfo, Info, #{}), - ConnInfo = maps:get(conninfo, Info, #{}), - Session = case maps:get(session, Info, #{}) of - undefined -> #{}; - _Sess -> _Sess - end, - SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), - Connected = case maps:get(conn_state, Info, connected) of - connected -> true; - _ -> false - end, - NStats = Stats#{max_subscriptions => maps:get(subscriptions_max, Stats, 0), - max_inflight => maps:get(inflight_max, Stats, 0), - max_awaiting_rel => maps:get(awaiting_rel_max, Stats, 0), - max_mqueue => maps:get(mqueue_max, Stats, 0), - inflight => maps:get(inflight_cnt, Stats, 0), - awaiting_rel => maps:get(awaiting_rel_cnt, Stats, 0)}, - format( - lists:foldl(fun(Items, Acc) -> - maps:merge(Items, Acc) - end, #{connected => Connected}, - [maps:with([ subscriptions_cnt, max_subscriptions, - inflight, max_inflight, awaiting_rel, - max_awaiting_rel, mqueue_len, mqueue_dropped, - max_mqueue, heap_size, reductions, mailbox_len, - recv_cnt, recv_msg, recv_oct, recv_pkt, send_cnt, - send_msg, send_oct, send_pkt], NStats), - maps:with([clientid, username, mountpoint, is_bridge, zone], ClientInfo), - maps:with([clean_start, keepalive, expiry_interval, proto_name, - proto_ver, peername, connected_at, disconnected_at], ConnInfo), - #{created_at => SessCreated}])). - -format(Data) when is_map(Data)-> - {IpAddr, Port} = maps:get(peername, Data), - ConnectedAt = maps:get(connected_at, Data), - CreatedAt = maps:get(created_at, Data), - Data1 = maps:without([peername], Data), - maps:merge(Data1#{node => node(), - ip_address => iolist_to_binary(ntoa(IpAddr)), - port => Port, - connected_at => iolist_to_binary(strftime(ConnectedAt div 1000)), - created_at => iolist_to_binary(strftime(CreatedAt div 1000))}, - case maps:get(disconnected_at, Data, undefined) of - undefined -> #{}; - DisconnectedAt -> #{disconnected_at => iolist_to_binary(strftime(DisconnectedAt div 1000))} - end). +peer_to_binary({Addr, Port}) -> + AddrBinary = list_to_binary(inet:ntoa(Addr)), + PortBinary = integer_to_binary(Port), + <>; +peer_to_binary(Addr) -> + list_to_binary(inet:ntoa(Addr)). format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> - #{access => PubSub, - topic => Topic, - result => AclResult, - updated_time => Timestamp}. + #{ + access => PubSub, + topic => Topic, + result => AclResult, + updated_time => Timestamp + }. -%%-------------------------------------------------------------------- +do_subscribe(ClientID, Topic0, Qos) -> + {Topic, Opts} = emqx_topic:parse(Topic0), + TopicTable = [{Topic, Opts#{qos => Qos}}], + emqx_mgmt:subscribe(ClientID, TopicTable), + case emqx_mgmt:subscribe(ClientID, TopicTable) of + {error, Reason} -> + {error, Reason}; + {subscribe, Subscriptions} -> + case proplists:is_defined(Topic, Subscriptions) of + true -> + ok; + false -> + {error, unknow_error} + end + end. + +do_unsubscribe(ClientID, Topic) -> + case emqx_mgmt:unsubscribe(ClientID, Topic) of + {error, Reason} -> + {error, Reason}; + Res -> + Res + end. +%%%============================================================================================== %% Query Functions -%%-------------------------------------------------------------------- query({Qs, []}, Start, Limit) -> Ms = qs2ms(Qs), @@ -328,37 +532,8 @@ query({Qs, Fuzzy}, Start, Limit) -> MatchFun = match_fun(Ms, Fuzzy), emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1). -%%-------------------------------------------------------------------- -%% Match funcs - -match_fun(Ms, Fuzzy) -> - MsC = ets:match_spec_compile(Ms), - REFuzzy = lists:map(fun({K, like, S}) -> - {ok, RE} = re:compile(S), - {K, like, RE} - end, Fuzzy), - fun(Rows) -> - case ets:match_spec_run(Rows, MsC) of - [] -> []; - Ls -> - lists:filter(fun(E) -> - run_fuzzy_match(E, REFuzzy) - end, Ls) - end - end. - -run_fuzzy_match(_, []) -> - true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> - Val = case maps:get(Key, ClientInfo, "") of - undefined -> ""; - V -> V - end, - re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). - -%%-------------------------------------------------------------------- +%%%============================================================================================== %% QueryString to Match Spec - -spec qs2ms(list()) -> ets:match_spec(). qs2ms(Qs) -> {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), @@ -380,7 +555,7 @@ put_conds({_, Op, V}, Holder, Conds) -> [{Op, Holder, V} | Conds]; put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) -> [{Op2, Holder, V2}, - {Op1, Holder, V1} | Conds]. + {Op1, Holder, V1} | Conds]. ms(clientid, X) -> #{clientinfo => #{clientid => X}}; @@ -403,51 +578,29 @@ ms(connected_at, X) -> ms(created_at, X) -> #{session => #{created_at => X}}. -%%-------------------------------------------------------------------- -%% EUnits -%%-------------------------------------------------------------------- +%%%============================================================================================== +%% Match funcs +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + REFuzzy = lists:map(fun({K, like, S}) -> + {ok, RE} = re:compile(S), + {K, like, RE} + end, Fuzzy), + fun(Rows) -> + case ets:match_spec_run(Rows, MsC) of + [] -> []; + Ls -> + lists:filter(fun(E) -> + run_fuzzy_match(E, REFuzzy) + end, Ls) + end + end. --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -params2qs_test() -> - QsSchema = element(2, ?CLIENT_QS_SCHEMA), - Params = [{<<"clientid">>, <<"abc">>}, - {<<"username">>, <<"def">>}, - {<<"zone">>, <<"external">>}, - {<<"ip_address">>, <<"127.0.0.1">>}, - {<<"conn_state">>, <<"connected">>}, - {<<"clean_start">>, true}, - {<<"proto_name">>, <<"MQTT">>}, - {<<"proto_ver">>, 4}, - {<<"_gte_created_at">>, 1}, - {<<"_lte_created_at">>, 5}, - {<<"_gte_connected_at">>, 1}, - {<<"_lte_connected_at">>, 5}, - {<<"_like_clientid">>, <<"a">>}, - {<<"_like_username">>, <<"e">>} - ], - ExpectedMtchHead = - #{clientinfo => #{clientid => <<"abc">>, - username => <<"def">>, - zone => external, - peerhost => {127,0,0,1} - }, - conn_state => connected, - conninfo => #{clean_start => true, - proto_name => <<"MQTT">>, - proto_ver => 4, - connected_at => '$3'}, - session => #{created_at => '$2'}}, - ExpectedCondi = [{'>=','$2', 1}, - {'=<','$2', 5}, - {'>=','$3', 1}, - {'=<','$3', 5}], - {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema), - [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1), - ?assertEqual(ExpectedMtchHead, MtchHead), - ?assertEqual(ExpectedCondi, Condi), - - [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). - --endif. +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> + Val = case maps:get(Key, ClientInfo, "") of + undefined -> ""; + V -> V + end, + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 1b0c90033..a2bc7309d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -44,25 +44,25 @@ %% List listeners on a node. list(#{node := Node}, _Params) -> - minirest:return({ok, format(emqx_mgmt:list_listeners(Node))}); + emqx_mgmt:return({ok, format(emqx_mgmt:list_listeners(Node))}); %% List listeners in the cluster. list(_Binding, _Params) -> - minirest:return({ok, [#{node => Node, listeners => format(Listeners)} + emqx_mgmt:return({ok, [#{node => Node, listeners => format(Listeners)} || {Node, Listeners} <- emqx_mgmt:list_listeners()]}). %% Restart listeners on a node. restart(#{node := Node, identifier := Identifier}, _Params) -> case emqx_mgmt:restart_listener(Node, Identifier) of - ok -> minirest:return({ok, "Listener restarted."}); - {error, Error} -> minirest:return({error, Error}) + ok -> emqx_mgmt:return({ok, "Listener restarted."}); + {error, Error} -> emqx_mgmt:return({error, Error}) end; %% Restart listeners on all nodes in the cluster. restart(#{identifier := Identifier}, _Params) -> Results = [{Node, emqx_mgmt:restart_listener(Node, Identifier)} || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun({_, Result}) -> Result =/= ok end, Results) of - [] -> minirest:return(ok); - Errors -> minirest:return({error, {restart, Errors}}) + [] -> emqx_mgmt:return(ok); + Errors -> emqx_mgmt:return({error, {restart, Errors}}) end. format(Listeners) when is_list(Listeners) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index b59aa0ac5..a4bf652a2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -31,12 +31,12 @@ -export([list/2]). list(Bindings, _Params) when map_size(Bindings) == 0 -> - minirest:return({ok, [#{node => Node, metrics => maps:from_list(Metrics)} + emqx_mgmt:return({ok, [#{node => Node, metrics => maps:from_list(Metrics)} || {Node, Metrics} <- emqx_mgmt:get_metrics()]}); list(#{node := Node}, _Params) -> case emqx_mgmt:get_metrics(Node) of - {error, Reason} -> minirest:return({error, Reason}); - Metrics -> minirest:return({ok, maps:from_list(Metrics)}) + {error, Reason} -> emqx_mgmt:return({error, Reason}); + Metrics -> emqx_mgmt:return({ok, maps:from_list(Metrics)}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index c24e46de9..3cb38c3a7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -13,49 +13,158 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_mgmt_api_nodes). --rest_api(#{name => list_nodes, - method => 'GET', - path => "/nodes/", - func => list, - descr => "A list of nodes in the cluster"}). +-behavior(minirest_api). --rest_api(#{name => get_node, - method => 'GET', - path => "/nodes/:atom:node", - func => get, - descr => "Lookup a node in the cluster"}). +-export([api_spec/0]). --export([ list/2 - , get/2 - ]). +-export([ nodes/2 + , node/2]). -list(_Bindings, _Params) -> - minirest:return({ok, [format(Node, Info) || {Node, Info} <- emqx_mgmt:list_nodes()]}). +-include_lib("emqx/include/emqx.hrl"). -get(#{node := Node}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_node(Node)}). +api_spec() -> + {apis(), schemas()}. -format(Node, {error, Reason}) -> #{node => Node, error => Reason}; +apis() -> + [ nodes_api() + , node_api()]. +schemas() -> + [node_schema()]. + +node_schema() -> + DefinitionName = <<"node">>, + DefinitionProperties = #{ + <<"node">> => #{ + type => <<"string">>, + description => <<"Node name">>}, + <<"connections">> => #{ + type => <<"integer">>, + description => <<"Number of clients currently connected to this node">>}, + <<"load1">> => #{ + type => <<"string">>, + description => <<"CPU average load in 1 minute">>}, + <<"load5">> => #{ + type => <<"string">>, + description => <<"CPU average load in 5 minute">>}, + <<"load15">> => #{ + type => <<"string">>, + description => <<"CPU average load in 15 minute">>}, + <<"max_fds">> => #{ + type => <<"integer">>, + description => <<"Maximum file descriptor limit for the operating system">>}, + <<"memory_total">> => #{ + type => <<"string">>, + description => <<"VM allocated system memory">>}, + <<"memory_used">> => #{ + type => <<"string">>, + description => <<"VM occupied system memory">>}, + <<"node_status">> => #{ + type => <<"string">>, + description => <<"Node status">>}, + <<"otp_release">> => #{ + type => <<"string">>, + description => <<"Erlang/OTP version used by EMQ X Broker">>}, + <<"process_available">> => #{ + type => <<"integer">>, + description => <<"Number of available processes">>}, + <<"process_used">> => #{ + type => <<"integer">>, + description => <<"Number of used processes">>}, + <<"uptime">> => #{ + type => <<"string">>, + description => <<"EMQ X Broker runtime">>}, + <<"version">> => #{ + type => <<"string">>, + description => <<"EMQ X Broker version">>}, + <<"sys_path">> => #{ + type => <<"string">>, + description => <<"EMQ X system file location">>}, + <<"log_path">> => #{ + type => <<"string">>, + description => <<"EMQ X log file location">>}, + <<"config_path">> => #{ + type => <<"string">>, + description => <<"EMQ X config file location">>} + }, + {DefinitionName, DefinitionProperties}. + +nodes_api() -> + Metadata = #{ + get => #{ + description => "List EMQ X nodes", + responses => #{ + <<"200">> => #{description => <<"List EMQ X Nodes">>, + schema => #{ + type => array, + items => cowboy_swagger:schema(<<"node">>)}}}}}, + {"/nodes", Metadata, nodes}. + +node_api() -> + Metadata = #{ + get => #{ + description => "Get node info", + parameters => [#{ + name => node_name, + in => path, + description => "node name", + type => string, + required => true, + default => node()}], + responses => #{ + <<"400">> => + emqx_mgmt_util:not_found_schema(<<"Node error">>, [<<"SOURCE_ERROR">>]), + <<"200">> => #{ + description => <<"Get EMQ X Nodes info by name">>, + schema => cowboy_swagger:schema(<<"node">>)}}}}, + {"/nodes/:node_name", Metadata, node}. + +%%%============================================================================================== +%% parameters trans +nodes(get, _Request) -> + list(#{}). + +node(get, Request) -> + NodeName = cowboy_req:binding(node_name, Request), + Node = binary_to_atom(NodeName, utf8), + get_node(#{node => Node}). + +%%%============================================================================================== +%% api apply +list(#{}) -> + NodesInfo = [format(Node, NodeInfo) || {Node, NodeInfo} <- emqx_mgmt:list_nodes()], + Response = emqx_json:encode(NodesInfo), + {200, Response}. + +get_node(#{node := Node}) -> + case emqx_mgmt:lookup_node(Node) of + #{node_status := 'ERROR'} -> + {400, emqx_json:encode(#{code => 'SOURCE_ERROR', reason => <<"rpc_failed">>})}; + NodeInfo -> + Response = emqx_json:encode(format(Node, NodeInfo)), + {200, Response} + end. + +%%============================================================================================================ +%% internal function format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> {ok, SysPathBinary} = file:get_cwd(), - SysPath = list_to_binary(SysPathBinary), - ConfigPath = <>, - LogPath = case log_path() of - undefined -> - <<"not found">>; - Path0 -> - Path = list_to_binary(Path0), - <> - end, - Info#{ memory_total := emqx_mgmt_util:kmg(Total) - , memory_used := emqx_mgmt_util:kmg(Used) - , sys_path => SysPath - , config_path => ConfigPath - , log_path => LogPath}. + SysPath = list_to_binary(SysPathBinary), + ConfigPath = <>, + LogPath = case log_path() of + undefined -> + <<"not found">>; + Path0 -> + Path = list_to_binary(Path0), + <> + end, + Info#{ memory_total := emqx_mgmt_util:kmg(Total) + , memory_used := emqx_mgmt_util:kmg(Used) + , sys_path => SysPath + , config_path => ConfigPath + , log_path => LogPath}. log_path() -> Configs = logger:get_handler_config(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 369ad6782..fda7151d7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -69,36 +69,36 @@ ]). list(#{node := Node}, _Params) -> - minirest:return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); + emqx_mgmt:return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); list(_Bindings, _Params) -> - minirest:return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). + emqx_mgmt:return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). load(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:load_plugin(Node, Plugin)). + emqx_mgmt:return(emqx_mgmt:load_plugin(Node, Plugin)). unload(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:unload_plugin(Node, Plugin)); + emqx_mgmt:return(emqx_mgmt:unload_plugin(Node, Plugin)); unload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:unload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - minirest:return(ok); + emqx_mgmt:return(ok); Errors -> - minirest:return(lists:last(Errors)) + emqx_mgmt:return(lists:last(Errors)) end. reload(#{node := Node, plugin := Plugin}, _Params) -> - minirest:return(emqx_mgmt:reload_plugin(Node, Plugin)); + emqx_mgmt:return(emqx_mgmt:reload_plugin(Node, Plugin)); reload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:reload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - minirest:return(ok); + emqx_mgmt:return(ok); Errors -> - minirest:return(lists:last(Errors)) + emqx_mgmt:return(lists:last(Errors)) end. format({Node, Plugins}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index e5a3e9d77..28e67c9f1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -67,7 +67,7 @@ subscribe(_Bindings, Params) -> logger:debug("API subscribe Params:~p", [Params]), {ClientId, Topic, QoS} = parse_subscribe_params(Params), - minirest:return(do_subscribe(ClientId, Topic, QoS)). + emqx_mgmt:return(do_subscribe(ClientId, Topic, QoS)). publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), @@ -75,33 +75,33 @@ publish(_Bindings, Params) -> case do_publish(ClientId, Topic, Qos, Retain, Payload) of {ok, MsgIds} -> case proplists:get_value(<<"return">>, Params, undefined) of - undefined -> minirest:return(ok); + undefined -> emqx_mgmt:return(ok); _Val -> case proplists:get_value(<<"topics">>, Params, undefined) of - undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); - _ -> minirest:return({ok, #{msgids => MsgIds}}) + undefined -> emqx_mgmt:return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> emqx_mgmt:return({ok, #{msgids => MsgIds}}) end end; Result -> - minirest:return(Result) + emqx_mgmt:return(Result) end. unsubscribe(_Bindings, Params) -> logger:debug("API unsubscribe Params:~p", [Params]), {ClientId, Topic} = parse_unsubscribe_params(Params), - minirest:return(do_unsubscribe(ClientId, Topic)). + emqx_mgmt:return(do_unsubscribe(ClientId, Topic)). subscribe_batch(_Bindings, Params) -> logger:debug("API subscribe batch Params:~p", [Params]), - minirest:return({ok, loop_subscribe(Params)}). + emqx_mgmt:return({ok, loop_subscribe(Params)}). publish_batch(_Bindings, Params) -> logger:debug("API publish batch Params:~p", [Params]), - minirest:return({ok, loop_publish(Params)}). + emqx_mgmt:return({ok, loop_publish(Params)}). unsubscribe_batch(_Bindings, Params) -> logger:debug("API unsubscribe batch Params:~p", [Params]), - minirest:return({ok, loop_unsubscribe(Params)}). + emqx_mgmt:return({ok, loop_unsubscribe(Params)}). loop_subscribe(Params) -> loop_subscribe(Params, []). diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index 380c9f0f6..00cb7bb90 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -35,11 +35,11 @@ ]). list(Bindings, Params) when map_size(Bindings) == 0 -> - minirest:return({ok, emqx_mgmt_api:paginate(emqx_route, Params, fun format/1)}). + emqx_mgmt:return({ok, emqx_mgmt_api:paginate(emqx_route, Params, fun format/1)}). lookup(#{topic := Topic}, _Params) -> Topic1 = emqx_mgmt_util:urldecode(Topic), - minirest:return({ok, [format(R) || R <- emqx_mgmt:lookup_routes(Topic1)]}). + emqx_mgmt:return({ok, [format(R) || R <- emqx_mgmt:lookup_routes(Topic1)]}). format(#route{topic = Topic, dest = {_, Node}}) -> #{topic => Topic, node => Node}; format(#route{topic = Topic, dest = Node}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 95d54b775..e57c3dc0e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -34,12 +34,12 @@ %% List stats of all nodes list(Bindings, _Params) when map_size(Bindings) == 0 -> - minirest:return({ok, [#{node => Node, stats => maps:from_list(Stats)} + emqx_mgmt:return({ok, [#{node => Node, stats => maps:from_list(Stats)} || {Node, Stats} <- emqx_mgmt:get_stats()]}). %% List stats of a node lookup(#{node := Node}, _Params) -> case emqx_mgmt:get_stats(Node) of - {error, Reason} -> minirest:return({error, Reason}); - Stats -> minirest:return({ok, maps:from_list(Stats)}) + {error, Reason} -> emqx_mgmt:return({error, Reason}); + Stats -> emqx_mgmt:return({ok, maps:from_list(Stats)}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl new file mode 100644 index 000000000..f7f013f20 --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_status). +%% API +-behavior(minirest_api). + +-export([api_spec/0]). + +-export([running_status/2]). + +api_spec() -> + {[status_api()], []}. + +status_api() -> + Path = "/status", + Metadata = #{ + get => #{ + security => [], + responses => #{ + <<"200">> => #{description => <<"running">>}}}}, + {Path, Metadata, running_status}. + +running_status(get, _Request) -> + {InternalStatus, _ProvidedStatus} = init:get_status(), + AppStatus = + case lists:keysearch(emqx, 1, application:which_applications()) of + false -> not_running; + {value, _Val} -> running + end, + Status = io_lib:format("Node ~s is ~s~nemqx is ~s", [node(), InternalStatus, AppStatus]), + Body = list_to_binary(Status), + {200, #{<<"content-type">> => <<"text/plain">>}, Body}. + + diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 4165ca51a..3f563427b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -63,9 +63,9 @@ list(Bindings, Params) when map_size(Bindings) == 0 -> case proplists:get_value(<<"topic">>, Params) of undefined -> - minirest:return({ok, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}); + emqx_mgmt:return({ok, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}); Topic -> - minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(emqx_mgmt_util:urldecode(Topic), ?format_fun)}) + emqx_mgmt:return({ok, emqx_mgmt:list_subscriptions_via_topic(emqx_mgmt_util:urldecode(Topic), ?format_fun)}) end; list(#{node := Node} = Bindings, Params) -> @@ -73,22 +73,22 @@ list(#{node := Node} = Bindings, Params) -> undefined -> case Node =:= node() of true -> - minirest:return({ok, emqx_mgmt_api:node_query(Node, Params, ?SUBS_QS_SCHEMA, ?query_fun)}); + emqx_mgmt:return({ok, emqx_mgmt_api:node_query(Node, Params, ?SUBS_QS_SCHEMA, ?query_fun)}); false -> case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> minirest:return({error, Reason}); + {badrpc, Reason} -> emqx_mgmt:return({error, Reason}); Res -> Res end end; Topic -> - minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(Node, emqx_mgmt_util:urldecode(Topic), ?format_fun)}) + emqx_mgmt:return({ok, emqx_mgmt:list_subscriptions_via_topic(Node, emqx_mgmt_util:urldecode(Topic), ?format_fun)}) end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))}); + emqx_mgmt:return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))}); lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}). + emqx_mgmt:return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}). format(Items) when is_list(Items) -> [format(Item) || Item <- Items]; diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index fbba0b2a4..d42b921b4 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -66,18 +66,20 @@ mnesia(copy) -> %%-------------------------------------------------------------------- %% Manage Apps %%-------------------------------------------------------------------- --spec(add_default_app() -> ok | {ok, appsecret()} | {error, term()}). +-spec(add_default_app() -> list()). add_default_app() -> - AppId = emqx_config:get([?APP, default_application_id], undefined), - AppSecret = emqx_config:get([?APP, default_application_secret], undefined), - case {AppId, AppSecret} of - {undefined, _} -> ok; - {_, undefined} -> ok; - {_, _} -> - AppId1 = to_binary(AppId), - AppSecret1 = to_binary(AppSecret), - add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) - end. + Apps = emqx_config:get([?APP, applications], []), + [ begin + case {AppId, AppSecret} of + {undefined, _} -> ok; + {_, undefined} -> ok; + {_, _} -> + AppId1 = to_binary(AppId), + AppSecret1 = to_binary(AppSecret), + add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) + end + end + || #{id := AppId, secret := AppSecret} <- Apps]. -spec(add_app(appid(), binary()) -> {ok, appsecret()} | {error, term()}). add_app(AppId, Name) when is_binary(AppId) -> diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index ecf204128..178e4b04d 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -13,27 +13,21 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_mgmt_http). -export([ start_listeners/0 - , handle_request/2 , stop_listeners/0 , start_listener/1 - , stop_listener/1 - ]). + , stop_listener/1]). --export([init/2]). +%% Authorization +-export([authorize_appid/1]). -include_lib("emqx/include/emqx.hrl"). -define(APP, emqx_management). --define(EXCEPT_PLUGIN, [emqx_dashboard]). --ifdef(TEST). --define(EXCEPT, []). --else. --define(EXCEPT, [add_app, del_app, list_apps, lookup_app, update_app]). --endif. + +-define(BASE_PATH, "/api/v5"). %%-------------------------------------------------------------------- %% Start/Stop Listeners @@ -45,85 +39,72 @@ start_listeners() -> stop_listeners() -> lists:foreach(fun stop_listener/1, listeners()). -start_listener({Proto, Port, Options}) when Proto == http -> - Dispatch = [{"/status", emqx_mgmt_http, []}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); +start_listener({Proto, Port, Options}) -> + {ok, _} = application:ensure_all_started(minirest), + Authorization = {?MODULE, authorize_appid}, + RanchOptions = ranch_opts(Port, Options), + GlobalSpec = #{ + swagger => "2.0", + info => #{title => "EMQ X API", version => "5.0.0"}, + basePath => ?BASE_PATH, + securityDefinitions => #{ + application => #{ + type => apiKey, + name => "authorization", + in => header}}}, + Minirest = #{ + protocol => Proto, + base_path => ?BASE_PATH, + apps => apps(), + authorization => Authorization, + security => [#{application => []}], + swagger_global_spec => GlobalSpec}, + MinirestOptions = maps:merge(Minirest, RanchOptions), + minirest:start(listener_name(Proto), MinirestOptions). -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/status", emqx_mgmt_http, []}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). +apps() -> + Apps = [App || {App, _, _} <- application:loaded_applications(), + case re:run(atom_to_list(App), "^emqx") of + {match,[{0,4}]} -> true; + _ -> false + end], + Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), + Apps ++ Plugins. ranch_opts(Port, Options0) -> - NumAcceptors = proplists:get_value(num_acceptors, Options0, 4), - MaxConnections = proplists:get_value(max_connections, Options0, 512), - Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> - Acc; - ({inet6, true}, Acc) -> [inet6 | Acc]; - ({inet6, false}, Acc) -> Acc; - ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; - ({ipv6_v6only, false}, Acc) -> Acc; - ({K, V}, Acc)-> - [{K, V} | Acc] - end, [], Options0), - - Res = #{num_acceptors => NumAcceptors, - max_connections => MaxConnections, - socket_opts => [{port, Port} | Options]}, - Res. + Options = lists:foldl( + fun + ({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> Acc; + ({inet6, true}, Acc) -> [inet6 | Acc]; + ({inet6, false}, Acc) -> Acc; + ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; + ({ipv6_v6only, false}, Acc) -> Acc; + ({K, V}, Acc)-> + [{K, V} | Acc] + end, [], Options0), + maps:from_list([{port, Port} | Options]). stop_listener({Proto, Port, _}) -> io:format("Stop http:management listener on ~s successfully.~n",[format(Port)]), - minirest:stop_http(listener_name(Proto)). + minirest:stop(listener_name(Proto)). listeners() -> - [{list_to_atom(Protocol), Port, maps:to_list(maps:without([protocol, port], Map))} + [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} || Map = #{protocol := Protocol,port := Port} <- emqx_config:get([emqx_management, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":management"). -http_handlers() -> - Apps = [ App || {App, _, _} <- application:loaded_applications(), - case re:run(atom_to_list(App), "^emqx") of - {match,[{0,4}]} -> true; - _ -> false - end], - Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ Apps -- ?EXCEPT_PLUGIN, - except => ?EXCEPT, - filter => fun(_) -> true end}), - [{authorization, fun authorize_appid/1}]}]. - -%%-------------------------------------------------------------------- -%% Handle 'status' request -%%-------------------------------------------------------------------- -init(Req, Opts) -> - Req1 = handle_request(cowboy_req:path(Req), Req), - {ok, Req1, Opts}. - -handle_request(Path, Req) -> - handle_request(cowboy_req:method(Req), Path, Req). - -handle_request(<<"GET">>, <<"/status">>, Req) -> - {InternalStatus, _ProvidedStatus} = init:get_status(), - AppStatus = case lists:keysearch(emqx, 1, application:which_applications()) of - false -> not_running; - {value, _Val} -> running - end, - Status = io_lib:format("Node ~s is ~s~nemqx is ~s", - [node(), InternalStatus, AppStatus]), - cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Status, Req); - -handle_request(_Method, _Path, Req) -> - cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req). - authorize_appid(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of - {basic, AppId, AppSecret} -> emqx_mgmt_auth:is_authorized(AppId, AppSecret); - _ -> false + {basic, AppId, AppSecret} -> + case emqx_mgmt_auth:is_authorized(AppId, AppSecret) of + true -> ok; + false -> {401} + end; + _ -> + {401} end. format(Port) when is_integer(Port) -> diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 132bbc83f..4197973e7 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -21,6 +21,9 @@ , kmg/1 , ntoa/1 , merge_maps/2 + , not_found_schema/1 + , not_found_schema/2 + , batch_operation/3 ]). -export([urldecode/1]). @@ -77,3 +80,35 @@ merge_maps(Default, New) -> urldecode(S) -> emqx_http_lib:uri_decode(S). +not_found_schema(Description) -> + not_found_schema(Description, ["RESOURCE_NOT_FOUND"]). + +not_found_schema(Description, Enum) -> + #{ + description => Description, + schema => #{ + type => object, + properties => #{ + code => #{ + type => string, + enum => Enum}, + reason => #{ + type => string}}} + }. + +batch_operation(Module, Function, ArgsList) -> + Failed = batch_operation(Module, Function, ArgsList, []), + Len = erlang:length(Failed), + Success = erlang:length(ArgsList) - Len, + Fun = fun({Args, Reason}, Detail) -> [#{data => Args, reason => io_lib:format("~p", [Reason])} | Detail] end, + #{success => Success, failed => Len, detail => lists:foldl(Fun, [], Failed)}. + +batch_operation(_Module, _Function, [], Failed) -> + lists:reverse(Failed); +batch_operation(Module, Function, [Args | ArgsList], Failed) -> + case erlang:apply(Module, Function, Args) of + ok -> + batch_operation(Module, Function, ArgsList, Failed); + {error ,Reason} -> + batch_operation(Module, Function, ArgsList, [{Args, Reason} | Failed]) + end. diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl deleted file mode 100644 index 415bfca2e..000000000 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ /dev/null @@ -1,340 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). - --define(LOG_LEVELS, ["debug", "error", "info"]). --define(LOG_HANDLER_ID, [file, default]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - ekka_mnesia:start(), - emqx_mgmt_auth:mnesia(boot), - emqx_ct_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]). - -set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}]}), - ok; -set_special_configs(_App) -> - ok. - -t_app(_Config) -> - {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), - ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), - ?assertEqual(AppSecret, emqx_mgmt_auth:get_appsecret(<<"app_id">>)), - ?assertEqual({<<"app_id">>, AppSecret, - <<"app_name">>, <<"Application user">>, - true, undefined}, - lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())), - emqx_mgmt_auth:del_app(<<"app_id">>), - application:set_env(emqx_management, application, []), - %% Specify the application secret - {ok, AppSecret2} = emqx_mgmt_auth:add_app( - <<"app_id">>, <<"app_name">>, <<"secret">>, - <<"app_desc">>, true, undefined), - ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret2)), - ?assertEqual(AppSecret2, emqx_mgmt_auth:get_appsecret(<<"app_id">>)), - ?assertEqual({<<"app_id">>, AppSecret2, <<"app_name">>, <<"app_desc">>, true, undefined}, - lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())), - emqx_mgmt_auth:del_app(<<"app_id">>), - ok. - -t_log_cmd(_) -> - mock_print(), - lists:foreach(fun(Level) -> - emqx_mgmt_cli:log(["primary-level", Level]), - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) - end, ?LOG_LEVELS), - lists:foreach(fun(Level) -> - emqx_mgmt_cli:log(["set-level", Level]), - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) - end, ?LOG_LEVELS), - [lists:foreach(fun(Level) -> - ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["handlers", "set-level", - atom_to_list(Id), Level])) - end, ?LOG_LEVELS) - || #{id := Id} <- emqx_logger:get_log_handlers()], - meck:unload(). - -t_mgmt_cmd(_) -> - % ct:pal("start testing the mgmt command"), - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["lookup", "emqx_appid"]), "Not Found.")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["insert", "emqx_appid", "emqx_name"]), "AppSecret:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["insert", "emqx_appid", "emqx_name"]), "Error:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["lookup", "emqx_appid"]), "app_id:")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["update", "emqx_appid", "ts"]), "update successfully")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( - ["delete", "emqx_appid"]), "ok")), - ok = emqx_mgmt_cli:mgmt(["list"]), - meck:unload(). - -t_status_cmd(_) -> - % ct:pal("start testing status command"), - mock_print(), - %% init internal status seem to be always 'starting' when running ct tests - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), - meck:unload(). - -t_broker_cmd(_) -> - % ct:pal("start testing the broker command"), - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([]), "sysdescr")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["stats"]), "subscriptions.shared")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["metrics"]), "bytes.sent")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")), - meck:unload(). - -t_clients_cmd(_) -> - % ct:pal("start testing the client command"), - mock_print(), - process_flag(trap_exit, true), - {ok, T} = emqtt:start_link([{clientid, <<"client12">>}, - {username, <<"testuser1">>}, - {password, <<"pass1">>} - ]), - {ok, _} = emqtt:connect(T), - timer:sleep(300), - emqx_mgmt_cli:clients(["list"]), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client12"]), "client12")), - ?assertEqual((emqx_mgmt_cli:clients(["kick", "client12"])), "ok~n"), - timer:sleep(500), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client12"]), "Not Found")), - receive - {'EXIT', T, _} -> - ok - % ct:pal("Connection closed: ~p~n", [Reason]) - after - 500 -> - erlang:error("Client is not kick") - end, - WS = rfc6455_client:new("ws://127.0.0.1:8083" ++ "/mqtt", self()), - {ok, _} = rfc6455_client:open(WS), - Packet = raw_send_serialize(?CONNECT_PACKET(#mqtt_packet_connect{ - clientid = <<"client13">>})), - ok = rfc6455_client:send_binary(WS, Packet), - Connack = ?CONNACK_PACKET(?CONNACK_ACCEPT), - {binary, Bin} = rfc6455_client:recv(WS), - {ok, Connack, <<>>, _} = raw_recv_pase(Bin), - timer:sleep(300), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "client13")), - meck:unload(). - % emqx_mgmt_cli:clients(["kick", "client13"]), - % timer:sleep(500), - % ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "Not Found")). - -raw_recv_pase(Packet) -> - emqx_frame:parse(Packet). - -raw_send_serialize(Packet) -> - emqx_frame:serialize(Packet). - -t_vm_cmd(_) -> - % ct:pal("start testing the vm command"), - mock_print(), - [[?assertMatch({match, _}, re:run(Result, Name)) - || Result <- emqx_mgmt_cli:vm([Name])] - || Name <- ["load", "memory", "process", "io", "ports"]], - [?assertMatch({match, _}, re:run(Result, "load")) - || Result <- emqx_mgmt_cli:vm(["load"])], - [?assertMatch({match, _}, re:run(Result, "memory")) - || Result <- emqx_mgmt_cli:vm(["memory"])], - [?assertMatch({match, _}, re:run(Result, "process")) - || Result <- emqx_mgmt_cli:vm(["process"])], - [?assertMatch({match, _}, re:run(Result, "io")) - || Result <- emqx_mgmt_cli:vm(["io"])], - [?assertMatch({match, _}, re:run(Result, "ports")) - || Result <- emqx_mgmt_cli:vm(["ports"])], - unmock_print(). - -t_trace_cmd(_) -> - % ct:pal("start testing the trace command"), - mock_print(), - logger:set_primary_config(level, debug), - {ok, T} = emqtt:start_link([{clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - emqtt:connect(T), - emqtt:subscribe(T, <<"a/b/c">>), - Trace1 = emqx_mgmt_cli:trace(["start", "client", "client", - "log/clientid_trace.log"]), - ?assertMatch({match, _}, re:run(Trace1, "successfully")), - Trace2 = emqx_mgmt_cli:trace(["stop", "client", "client"]), - ?assertMatch({match, _}, re:run(Trace2, "successfully")), - Trace3 = emqx_mgmt_cli:trace(["start", "client", "client", - "log/clientid_trace.log", - "error"]), - ?assertMatch({match, _}, re:run(Trace3, "successfully")), - Trace4 = emqx_mgmt_cli:trace(["stop", "client", "client"]), - ?assertMatch({match, _}, re:run(Trace4, "successfully")), - Trace5 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", - "log/clientid_trace.log"]), - ?assertMatch({match, _}, re:run(Trace5, "successfully")), - Trace6 = emqx_mgmt_cli:trace(["stop", "topic", "a/b/c"]), - ?assertMatch({match, _}, re:run(Trace6, "successfully")), - Trace7 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", - "log/clientid_trace.log", "error"]), - ?assertMatch({match, _}, re:run(Trace7, "successfully")), - logger:set_primary_config(level, error), - unmock_print(). - -t_router_cmd(_) -> - % ct:pal("start testing the router command"), - mock_print(), - {ok, T} = emqtt:start_link([{clientid, <<"client1">>}, - {username, <<"testuser1">>}, - {password, <<"pass1">>} - ]), - emqtt:connect(T), - emqtt:subscribe(T, <<"a/b/c">>), - {ok, T1} = emqtt:start_link([{clientid, <<"client2">>}, - {username, <<"testuser2">>}, - {password, <<"pass2">>} - ]), - - emqtt:connect(T1), - emqtt:subscribe(T1, <<"a/b/c/d">>), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["list"]), "a/b/c | a/b/c")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")), - unmock_print(). - -t_subscriptions_cmd(_) -> - % ct:pal("Start testing the subscriptions command"), - mock_print(), - {ok, T3} = emqtt:start_link([{clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - {ok, _} = emqtt:connect(T3), - {ok, _, _} = emqtt:subscribe(T3, <<"b/b/c">>), - timer:sleep(300), - [?assertMatch({match, _} , re:run(Result, "b/b/c")) - || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], - ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok~n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"), - unmock_print(). - -t_listeners_cmd_old(_) -> - ok = emqx_listeners:ensure_all_started(), - mock_print(), - ?assertEqual(emqx_mgmt_cli:listeners([]), ok), - ?assertEqual( - "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", - emqx_mgmt_cli:listeners(["stop", "wss", "8084"]) - ), - unmock_print(). - -t_listeners_cmd_new(_) -> - ok = emqx_listeners:ensure_all_started(), - mock_print(), - ?assertEqual(emqx_mgmt_cli:listeners([]), ok), - ?assertEqual( - "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", - emqx_mgmt_cli:listeners(["stop", "mqtt:wss:external"]) - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "mqtt:tcp:external"]), - "Restarted mqtt:tcp:external listener successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "mqtt:ssl:external"]), - "Restarted mqtt:ssl:external listener successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), - "Failed to restart bad:listener:identifier listener: {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). - -t_plugins_cmd(_) -> - mock_print(), - meck:new(emqx_plugins, [non_strict, passthrough]), - meck:expect(emqx_plugins, load, fun(_) -> ok end), - meck:expect(emqx_plugins, unload, fun(_) -> ok end), - meck:expect(emqx_plugins, reload, fun(_) -> ok end), - ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), - ?assertEqual( - emqx_mgmt_cli:plugins(["unload", "emqx_retainer"]), - "Plugin emqx_retainer unloaded successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:plugins(["load", "emqx_retainer"]), - "Plugin emqx_retainer loaded successfully.\n" - ), - ?assertEqual( - emqx_mgmt_cli:plugins(["unload", "emqx_management"]), - "Plugin emqx_management can not be unloaded.~n" - ), - unmock_print(). - -t_cli(_) -> - mock_print(), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), - [?assertMatch({match, _}, re:run(Value, "broker")) - || Value <- emqx_mgmt_cli:broker([""])], - [?assertMatch({match, _}, re:run(Value, "cluster")) - || Value <- emqx_mgmt_cli:cluster([""])], - [?assertMatch({match, _}, re:run(Value, "clients")) - || Value <- emqx_mgmt_cli:clients([""])], - [?assertMatch({match, _}, re:run(Value, "routes")) - || Value <- emqx_mgmt_cli:routes([""])], - [?assertMatch({match, _}, re:run(Value, "subscriptions")) - || Value <- emqx_mgmt_cli:subscriptions([""])], - [?assertMatch({match, _}, re:run(Value, "plugins")) - || Value <- emqx_mgmt_cli:plugins([""])], - [?assertMatch({match, _}, re:run(Value, "listeners")) - || Value <- emqx_mgmt_cli:listeners([""])], - [?assertMatch({match, _}, re:run(Value, "vm")) - || Value <- emqx_mgmt_cli:vm([""])], - [?assertMatch({match, _}, re:run(Value, "mnesia")) - || Value <- emqx_mgmt_cli:mnesia([""])], - [?assertMatch({match, _}, re:run(Value, "trace")) - || Value <- emqx_mgmt_cli:trace([""])], - [?assertMatch({match, _}, re:run(Value, "mgmt")) - || Value <- emqx_mgmt_cli:mgmt([""])], - unmock_print(). - -mock_print() -> - catch meck:unload(emqx_ctl), - meck:new(emqx_ctl, [non_strict, passthrough]), - meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end), - meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), - meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), - meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). - -unmock_print() -> - meck:unload(emqx_ctl). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl deleted file mode 100644 index 66d9328a6..000000000 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ /dev/null @@ -1,593 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_api_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_management/include/emqx_mgmt.hrl"). - --define(CONTENT_TYPE, "application/x-www-form-urlencoded"). - --define(HOST, "http://127.0.0.1:8081/"). - --define(API_VERSION, "v4"). - --define(BASE_PATH, "api"). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_management]), - Config. - -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, Config) -> - Config. - -set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], - default_application_id => <<"admin">>, - default_application_secret => <<"public">>}), - ok; -set_special_configs(_App) -> - ok. - -get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). - -lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> - true; -lookup_alarm(Name, [_Alarm | More]) -> - lookup_alarm(Name, More); -lookup_alarm(_Name, []) -> - false. - -is_existing(Name, [#{name := Name} | _More]) -> - true; -is_existing(Name, [_Alarm | More]) -> - is_existing(Name, More); -is_existing(_Name, []) -> - false. - -t_alarms(_) -> - emqx_alarm:activate(alarm1), - emqx_alarm:activate(alarm2), - - ?assert(is_existing(alarm1, emqx_alarm:get_alarms(activated))), - ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), - - {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - - emqx_alarm:deactivate(alarm1), - - {ok, Return2} = request_api(get, api_path(["alarms"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - - {ok, Return3} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - - emqx_alarm:deactivate(alarm2), - - {ok, Return4} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - - {ok, _} = request_api(delete, api_path(["alarms/deactivated"]), auth_header_()), - - {ok, Return5} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). - -t_apps(_) -> - AppId = <<"123456">>, - meck:new(emqx_mgmt_auth, [passthrough, no_history]), - meck:expect(emqx_mgmt_auth, add_app, 6, fun(_, _, _, _, _, _) -> {error, undefined} end), - {ok, Error1} = request_api(post, api_path(["apps"]), [], - auth_header_(), #{<<"app_id">> => AppId, - <<"name">> => <<"test">>, - <<"status">> => true}), - ?assertMatch(<<"undefined">>, get(<<"message">>, Error1)), - - meck:expect(emqx_mgmt_auth, del_app, 1, fun(_) -> {error, undefined} end), - {ok, Error2} = request_api(delete, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertMatch(<<"undefined">>, get(<<"message">>, Error2)), - meck:unload(emqx_mgmt_auth), - - {ok, NoApp} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertEqual(0, maps:size(get(<<"data">>, NoApp))), - {ok, NotFound} = request_api(put, api_path(["apps", binary_to_list(AppId)]), [], - auth_header_(), #{<<"name">> => <<"test 2">>, - <<"status">> => true}), - ?assertEqual(<<"not_found">>, get(<<"message">>, NotFound)), - - {ok, _} = request_api(post, api_path(["apps"]), [], - auth_header_(), #{<<"app_id">> => AppId, - <<"name">> => <<"test">>, - <<"status">> => true}), - {ok, _} = request_api(get, api_path(["apps"]), auth_header_()), - {ok, _} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - {ok, _} = request_api(put, api_path(["apps", binary_to_list(AppId)]), [], - auth_header_(), #{<<"name">> => <<"test 2">>, - <<"status">> => true}), - {ok, AppInfo} = request_api(get, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - ?assertEqual(<<"test 2">>, maps:get(<<"name">>, get(<<"data">>, AppInfo))), - {ok, _} = request_api(delete, api_path(["apps", binary_to_list(AppId)]), auth_header_()), - {ok, Result} = request_api(get, api_path(["apps"]), auth_header_()), - [App] = get(<<"data">>, Result), - ?assertEqual(<<"admin">>, maps:get(<<"app_id">>, App)). - -t_banned(_) -> - Who = <<"myclient">>, - {ok, _} = request_api(post, api_path(["banned"]), [], - auth_header_(), #{<<"who">> => Who, - <<"as">> => <<"clientid">>, - <<"reason">> => <<"test">>, - <<"by">> => <<"dashboard">>, - <<"at">> => erlang:system_time(second), - <<"until">> => erlang:system_time(second) + 10}), - - {ok, Result} = request_api(get, api_path(["banned"]), auth_header_()), - [Banned] = get(<<"data">>, Result), - ?assertEqual(Who, maps:get(<<"who">>, Banned)), - - {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), auth_header_()), - {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), - ?assertEqual([], get(<<"data">>, Result2)). - -t_brokers(_) -> - {ok, _} = request_api(get, api_path(["brokers"]), auth_header_()), - {ok, _} = request_api(get, api_path(["brokers", atom_to_list(node())]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, lookup_broker, 1, fun(_) -> {error, undefined} end), - {ok, Error} = request_api(get, api_path(["brokers", atom_to_list(node())]), auth_header_()), - ?assertEqual(<<"undefined">>, get(<<"message">>, Error)), - meck:unload(emqx_mgmt). - -t_clients(_) -> - process_flag(trap_exit, true), - Username1 = <<"user1">>, - Username2 = <<"user2">>, - ClientId1 = <<"client1">>, - ClientId2 = <<"client2">>, - {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), - {ok, _} = emqtt:connect(C1), - {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), - {ok, _} = emqtt:connect(C2), - - timer:sleep(300), - - {ok, Clients1} = request_api(get, api_path(["clients", binary_to_list(ClientId1)]) - , auth_header_()), - ?assertEqual(<<"client1">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients1)))), - - {ok, Clients2} = request_api(get, api_path(["nodes", atom_to_list(node()), - "clients", binary_to_list(ClientId2)]) - , auth_header_()), - ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients2)))), - - {ok, Clients3} = request_api(get, api_path(["clients", - "username", binary_to_list(Username1)]), - auth_header_()), - ?assertEqual(<<"client1">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients3)))), - - {ok, Clients4} = request_api(get, api_path(["nodes", atom_to_list(node()), - "clients", - "username", binary_to_list(Username2)]) - , auth_header_()), - ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients4)))), - - {ok, Clients5} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), - ?assertEqual(2, maps:get(<<"count">>, get(<<"meta">>, Clients5))), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), - - {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet1)), - - meck:expect(emqx_mgmt, clean_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet2)), - - meck:expect(emqx_mgmt, list_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet3} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), - - meck:unload(emqx_mgmt), - - {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), - - timer:sleep(300), - - {ok, NotFound0} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound0)), - - {ok, Clients6} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), - ?assertEqual(1, maps:get(<<"count">>, get(<<"meta">>, Clients6))), - - {ok, NotFound1} = request_api(get, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - - {ok, NotFound2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - - {ok, EmptyAclCache} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(0, length(get(<<"data">>, EmptyAclCache))), - - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), - ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)). - -receive_exit(0) -> - ok; -receive_exit(Count) -> - receive - {'EXIT', Client, {shutdown, tcp_closed}} -> - ct:log("receive exit signal, Client: ~p", [Client]), - receive_exit(Count - 1); - {'EXIT', Client, _Reason} -> - ct:log("receive exit signal, Client: ~p", [Client]), - receive_exit(Count - 1) - after 1000 -> - ct:log("timeout") - end. - -t_listeners(_) -> - {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, list_listeners, 0, fun() -> [{node(), {error, undefined}}] end), - {ok, Return} = request_api(get, api_path(["listeners"]), auth_header_()), - [Error] = get(<<"data">>, Return), - ?assertEqual(<<"undefined">>, - maps:get(<<"error">>, maps:get(<<"listeners">>, Error))), - meck:unload(emqx_mgmt). - -t_metrics(_) -> - {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, get_metrics, 1, fun(_) -> {error, undefined} end), - {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), - meck:unload(emqx_mgmt). - -t_nodes(_) -> - {ok, _} = request_api(get, api_path(["nodes"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node())]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, list_nodes, 0, fun() -> [{node(), {error, undefined}}] end), - {ok, Return} = request_api(get, api_path(["nodes"]), auth_header_()), - [Error] = get(<<"data">>, Return), - ?assertEqual(<<"undefined">>, maps:get(<<"error">>, Error)), - meck:unload(emqx_mgmt). - -% t_plugins(_) -> -% application:ensure_all_started(emqx_retainer), -% {ok, Plugins1} = request_api(get, api_path(["plugins"]), auth_header_()), -% [Plugins11] = filter(get(<<"data">>, Plugins1), <<"node">>, atom_to_binary(node(), utf8)), -% [Plugin1] = filter(maps:get(<<"plugins">>, Plugins11), <<"name">>, <<"emqx_retainer">>), -% ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin1)), -% ?assertEqual(true, maps:get(<<"active">>, Plugin1)), -% -% {ok, _} = request_api(put, -% api_path(["plugins", -% atom_to_list(emqx_retainer), -% "unload"]), -% auth_header_()), -% {ok, Error1} = request_api(put, -% api_path(["plugins", -% atom_to_list(emqx_retainer), -% "unload"]), -% auth_header_()), -% ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), -% {ok, Plugins2} = request_api(get, -% api_path(["nodes", atom_to_list(node()), "plugins"]), -% auth_header_()), -% [Plugin2] = filter(get(<<"data">>, Plugins2), <<"name">>, <<"emqx_retainer">>), -% ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin2)), -% ?assertEqual(false, maps:get(<<"active">>, Plugin2)), -% -% {ok, _} = request_api(put, -% api_path(["nodes", -% atom_to_list(node()), -% "plugins", -% atom_to_list(emqx_retainer), -% "load"]), -% auth_header_()), -% {ok, Plugins3} = request_api(get, -% api_path(["nodes", atom_to_list(node()), "plugins"]), -% auth_header_()), -% [Plugin3] = filter(get(<<"data">>, Plugins3), <<"name">>, <<"emqx_retainer">>), -% ?assertEqual(<<"emqx_retainer">>, maps:get(<<"name">>, Plugin3)), -% ?assertEqual(true, maps:get(<<"active">>, Plugin3)), -% -% {ok, _} = request_api(put, -% api_path(["nodes", -% atom_to_list(node()), -% "plugins", -% atom_to_list(emqx_retainer), -% "unload"]), -% auth_header_()), -% {ok, Error2} = request_api(put, -% api_path(["nodes", -% atom_to_list(node()), -% "plugins", -% atom_to_list(emqx_retainer), -% "unload"]), -% auth_header_()), -% ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), -% application:stop(emqx_retainer). - -t_acl_cache(_) -> - ClientId = <<"client1">>, - Topic = <<"mytopic">>, - {ok, C1} = emqtt:start_link(#{clientid => ClientId}), - {ok, _} = emqtt:connect(C1), - {ok, _, _} = emqtt:subscribe(C1, Topic, 2), - %% get acl cache, should not be empty - {ok, Result} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches} = jiffy:decode(list_to_binary(Result), [return_maps]), - ?assert(length(Caches) > 0), - ?assertMatch(#{<<"access">> := <<"subscribe">>, - <<"topic">> := Topic, - <<"result">> := <<"allow">>, - <<"updated_time">> := _}, hd(Caches)), - %% clear acl cache - {ok, Result2} = request_api(delete, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - ?assertMatch(#{<<"code">> := 0}, jiffy:decode(list_to_binary(Result2), [return_maps])), - %% get acl cache again, after the acl cache is cleared - {ok, Result3} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches3} = jiffy:decode(list_to_binary(Result3), [return_maps]), - ?assertEqual(0, length(Caches3)), - ok = emqtt:disconnect(C1). - -t_pubsub(_) -> - Qos1Received = emqx_metrics:val('messages.qos1.received'), - Qos2Received = emqx_metrics:val('messages.qos2.received'), - Received = emqx_metrics:val('messages.received'), - - ClientId = <<"client1">>, - Options = #{clientid => ClientId, - proto_ver => 5}, - Topic = <<"mytopic">>, - {ok, C1} = emqtt:start_link(Options), - {ok, _} = emqtt:connect(C1), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, subscribe, 2, fun(_, _) -> {error, undefined} end), - {ok, NotFound1} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic, - <<"qos">> => 2}), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - meck:unload(emqx_mgmt), - - {ok, BadTopic1} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topics">> => <<"">>, - <<"qos">> => 2}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic1)), - - {ok, BadTopic2} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topics">> => <<"">>, - <<"qos">> => 1, - <<"payload">> => <<"hello">>}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic2)), - - {ok, BadTopic3} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"">>}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic3)), - - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, unsubscribe, 2, fun(_, _) -> {error, undefined} end), - {ok, NotFound2} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic}), - ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - meck:unload(emqx_mgmt), - - {ok, Code} = request_api(post, api_path(["mqtt/subscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic, - <<"qos">> => 2}), - ?assertEqual(?SUCCESS, get(<<"code">>, Code)), - {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"mytopic">>, - <<"qos">> => 1, - <<"payload">> => <<"hello">>}), - ?assert(receive - {publish, #{payload := <<"hello">>}} -> - true - after 100 -> - false - end), - %% json payload - {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => <<"mytopic">>, - <<"qos">> => 1, - <<"payload">> => #{body => "hello world"}}), - Payload = emqx_json:encode(#{body => "hello world"}), - ?assert(receive - {publish, #{payload := Payload}} -> - true - after 100 -> - false - end), - - {ok, Code} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), - #{<<"clientid">> => ClientId, - <<"topic">> => Topic}), - - %% tests subscribe_batch - Topic_list = [<<"mytopic1">>, <<"mytopic2">>], - [ {ok, _, [2]} = emqtt:subscribe(C1, Topics, 2) || Topics <- Topic_list], - - Body1 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], - {ok, Data1} = request_api(post, api_path(["mqtt/subscribe_batch"]), [], auth_header_(), Body1), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data1), [return_maps]))), - - %% tests publish_batch - Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} || Topics <- Topic_list ], - {ok, Data2} = request_api(post, api_path(["mqtt/publish_batch"]), [], auth_header_(), Body2), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data2), [return_maps]))), - [ ?assert(receive - {publish, #{topic := Topics}} -> - true - after 100 -> - false - end) || Topics <- Topic_list ], - - %% tests unsubscribe_batch - Body3 = [#{<<"clientid">> => ClientId, <<"topic">> => Topics} || Topics <- Topic_list], - {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), - loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), - - ok = emqtt:disconnect(C1), - - ?assertEqual(2, emqx_metrics:val('messages.qos1.received') - Qos1Received), - ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(4, emqx_metrics:val('messages.received') - Received). - -loop([]) -> []; - -loop(Data) -> - [H | T] = Data, - ct:pal("H: ~p~n", [H]), - ?assertEqual(0, maps:get(<<"code">>, H)), - loop(T). - -t_routes_and_subscriptions(_) -> - ClientId = <<"myclient">>, - Topic = <<"mytopic">>, - {ok, NonRoute} = request_api(get, api_path(["routes"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonRoute)), - {ok, NonSubscription} = request_api(get, api_path(["subscriptions"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription)), - {ok, NonSubscription1} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription1)), - {ok, NonSubscription2} = request_api(get, - api_path(["subscriptions", binary_to_list(ClientId)]), - auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription2)), - {ok, NonSubscription3} = request_api(get, api_path(["nodes", - atom_to_list(node()), - "subscriptions", - binary_to_list(ClientId)]) - , auth_header_()), - ?assertEqual([], get(<<"data">>, NonSubscription3)), - {ok, C1} = emqtt:start_link(#{clean_start => true, - clientid => ClientId, - proto_ver => ?MQTT_PROTO_V5}), - {ok, _} = emqtt:connect(C1), - {ok, _, [2]} = emqtt:subscribe(C1, Topic, qos2), - {ok, Result} = request_api(get, api_path(["routes"]), auth_header_()), - [Route] = get(<<"data">>, Result), - ?assertEqual(Topic, maps:get(<<"topic">>, Route)), - - {ok, Result2} = request_api(get, api_path(["routes", binary_to_list(Topic)]), auth_header_()), - [Route] = get(<<"data">>, Result2), - - {ok, Result3} = request_api(get, api_path(["subscriptions"]), auth_header_()), - [Subscription] = get(<<"data">>, Result3), - ?assertEqual(Topic, maps:get(<<"topic">>, Subscription)), - ?assertEqual(ClientId, maps:get(<<"clientid">>, Subscription)), - - {ok, Result3} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - - {ok, Result4} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), - [Subscription] = get(<<"data">>, Result4), - {ok, Result4} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) - , auth_header_()), - - ok = emqtt:disconnect(C1). - -t_stats(_) -> - {ok, _} = request_api(get, api_path(["stats"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), - meck:new(emqx_mgmt, [passthrough, no_history]), - meck:expect(emqx_mgmt, get_stats, 1, fun(_) -> {error, undefined} end), - {ok, Return} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), - ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), - meck:unload(emqx_mgmt). - -request_api(Method, Url, Auth) -> - request_api(Method, Url, [], Auth, []). - -request_api(Method, Url, QueryParams, Auth) -> - request_api(Method, Url, QueryParams, Auth, []). - -request_api(Method, Url, QueryParams, Auth, []) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth]}); -request_api(Method, Url, QueryParams, Auth, Body) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). - -do_request_api(Method, Request)-> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{"HTTP/1.1", Code, _}, _, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User,":",Pass])), - {"Authorization","Basic " ++ Encoded}. - -api_path(Parts)-> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - -filter(List, Key, Value) -> - lists:filter(fun(Item) -> - maps:get(Key, Item) == Value - end, List). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl new file mode 100644 index 000000000..1925b52e5 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -0,0 +1,85 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_test_util). +-compile(export_all). +-compile(nowarn_export_all). + +-define(SERVER, "http://127.0.0.1:8081"). +-define(BASE_PATH, "/api/v5"). + +default_init() -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + ok. + + +default_end() -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + + +request_api(Method, Url) -> + request_api(Method, Url, [], auth_header_(), []). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?SERVER ++ filename:join([?BASE_PATH | Parts]). diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl new file mode 100644 index 000000000..688989211 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -0,0 +1,96 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_clients_api_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(APP, emqx_management). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:default_init(), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:default_end(). + +t_clients(_) -> + process_flag(trap_exit, true), + + Username1 = <<"user1">>, + ClientId1 = <<"client1">>, + + Username2 = <<"user2">>, + ClientId2 = <<"client2">>, + + Topic = <<"topic_1">>, + Qos = 0, + + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + + {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), + {ok, _} = emqtt:connect(C1), + {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), + {ok, _} = emqtt:connect(C2), + + timer:sleep(300), + + %% get /clients + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsMeta = maps:get(<<"meta">>, ClientsResponse), + ClientsPage = maps:get(<<"page">>, ClientsMeta), + ClientsLimit = maps:get(<<"limit">>, ClientsMeta), + ClientsCount = maps:get(<<"count">>, ClientsMeta), + ?assertEqual(ClientsPage, 1), + ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()), + ?assertEqual(ClientsCount, 2), + + %% get /clients/:clientid + Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), + {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), + Client1Response = emqx_json:decode(Client1, [return_maps]), + ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), + ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), + + %% delete /clients/:clientid kickout + Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path), + AfterKickoutResponse = emqx_mgmt_api_test_util:request_api(get, Client2Path), + ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse), + + %% get /clients/:clientid/acl_cache should has no acl cache + Client1AclCachePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), + {ok, Client1AclCache} = emqx_mgmt_api_test_util:request_api(get, Client1AclCachePath), + ?assertEqual("[]", Client1AclCache), + + %% post /clients/:clientid/subscribe + SubscribeBody = #{topic => Topic, qos => Qos}, + SubscribePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "subscribe"]), + {ok, _} = emqx_mgmt_api_test_util:request_api(post, SubscribePath, "", AuthHeader, SubscribeBody), + [{{_, AfterSubTopic}, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1), + ?assertEqual(AfterSubTopic, Topic), + ?assertEqual(AfterSubQos, Qos), + + %% delete /clients/:clientid/subscribe + UnSubscribeQuery = "topic=" ++ binary_to_list(Topic), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, SubscribePath, UnSubscribeQuery, AuthHeader), + ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)). diff --git a/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl new file mode 100644 index 000000000..52d2bf626 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_nodes_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(APP, emqx_management). + +-define(SERVER, "http://127.0.0.1:8081"). +-define(BASE_PATH, "/api/v5"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ekka_mnesia:start(), + emqx_mgmt_auth:mnesia(boot), + emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([emqx_management]). + +set_special_configs(emqx_management) -> + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), + ok; +set_special_configs(_App) -> + ok. + +t_nodes_api(_) -> + NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), + {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), + NodesResponse = emqx_json:decode(Nodes, [return_maps]), + LocalNodeInfo = hd(NodesResponse), + Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), + ?assertEqual(Node, node()), + + NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), + {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), + NodeNameResponse = binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + ?assertEqual(node(), NodeNameResponse). diff --git a/apps/emqx_management/test/etc/emqx_management.conf b/apps/emqx_management/test/etc/emqx_management.conf deleted file mode 100644 index e54164cbd..000000000 --- a/apps/emqx_management/test/etc/emqx_management.conf +++ /dev/null @@ -1,39 +0,0 @@ -emqx_management:{ - default_application_id: "admin" - default_application_secret: "public" - max_row_limit: 10000 - listeners: [ - { - num_acceptors: 4 - max_connections: 512 - protocol: "http" - port: 8080 - backlog: 512 - send_timeout: 15s - send_timeout_close: on - inet6: false - ipv6_v6only: false - } -## , -## { -## protocol: https -## port: 8081 -## acceptors: 2 -## backlog: 512 -## send_timeout: 15s -## send_timeout_close: on -## inet6: false -## ipv6_v6only: false -## certfile = "etc/certs/cert.pem" -## keyfile = "etc/certs/key.pem" -## cacertfile = "etc/certs/cacert.pem" -## verify = verify_peer -## tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" -## ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" -## fail_if_no_peer_cert = true -## inet6 = false -## ipv6_v6only = false -## } - ] -} - diff --git a/apps/emqx_management/test/etc/emqx_reloader.conf b/apps/emqx_management/test/etc/emqx_reloader.conf deleted file mode 100644 index 0919c8411..000000000 --- a/apps/emqx_management/test/etc/emqx_reloader.conf +++ /dev/null @@ -1,24 +0,0 @@ -##-------------------------------------------------------------------- -## Reloader Plugin -##-------------------------------------------------------------------- - -## Interval of hot code reloading. -## -## Value: Duration -## - h: hour -## - m: minute -## - s: second -## -## Examples: -## - 2h: 2 hours -## - 30m: 30 minutes -## - 20s: 20 seconds -## -## Defaut: 60s -reloader.interval = 60s - -## Logfile of reloader. -## -## Value: File -reloader.logfile = reloader.log - diff --git a/apps/emqx_management/test/rfc6455_client.erl b/apps/emqx_management/test/rfc6455_client.erl deleted file mode 100644 index 987b72407..000000000 --- a/apps/emqx_management/test/rfc6455_client.erl +++ /dev/null @@ -1,252 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. -%% -%% The Original Code is RabbitMQ Management Console. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2012-2016 Pivotal Software, Inc. All rights reserved. -%% - --module(rfc6455_client). - --export([new/2, open/1, recv/1, send/2, send_binary/2, close/1, close/2]). - --record(state, {host, port, addr, path, ppid, socket, data, phase}). - -%% -------------------------------------------------------------------------- - -new(WsUrl, PPid) -> - crypto:start(), - "ws://" ++ Rest = WsUrl, - [Addr, Path] = split("/", Rest, 1), - [Host, MaybePort] = split(":", Addr, 1, empty), - Port = case MaybePort of - empty -> 80; - V -> {I, ""} = string:to_integer(V), I - end, - State = #state{host = Host, - port = Port, - addr = Addr, - path = "/" ++ Path, - ppid = PPid}, - spawn(fun() -> - start_conn(State) - end). - -open(WS) -> - receive - {rfc6455, open, WS, Opts} -> - {ok, Opts}; - {rfc6455, close, WS, R} -> - {close, R} - end. - -recv(WS) -> - receive - {rfc6455, recv, WS, Payload} -> - {ok, Payload}; - {rfc6455, recv_binary, WS, Payload} -> - {binary, Payload}; - {rfc6455, close, WS, R} -> - {close, R} - end. - -send(WS, IoData) -> - WS ! {send, IoData}, - ok. - -send_binary(WS, IoData) -> - WS ! {send_binary, IoData}, - ok. - -close(WS) -> - close(WS, {1000, ""}). - -close(WS, WsReason) -> - WS ! {close, WsReason}, - receive - {rfc6455, close, WS, R} -> - {close, R} - end. - - -%% -------------------------------------------------------------------------- - -start_conn(State) -> - {ok, Socket} = gen_tcp:connect(State#state.host, State#state.port, - [binary, - {packet, 0}]), - Key = base64:encode_to_string(crypto:strong_rand_bytes(16)), - gen_tcp:send(Socket, - "GET " ++ State#state.path ++ " HTTP/1.1\r\n" ++ - "Host: " ++ State#state.addr ++ "\r\n" ++ - "Upgrade: websocket\r\n" ++ - "Connection: Upgrade\r\n" ++ - "Sec-WebSocket-Key: " ++ Key ++ "\r\n" ++ - "Origin: null\r\n" ++ - "Sec-WebSocket-Protocol: mqtt\r\n" ++ - "Sec-WebSocket-Version: 13\r\n\r\n"), - - loop(State#state{socket = Socket, - data = <<>>, - phase = opening}). - -do_recv(State = #state{phase = opening, ppid = PPid, data = Data}) -> - case split("\r\n\r\n", binary_to_list(Data), 1, empty) of - [_Http, empty] -> State; - [Http, Data1] -> - %% TODO: don't ignore http response data, verify key - PPid ! {rfc6455, open, self(), [{http_response, Http}]}, - State#state{phase = open, - data = Data1} - end; -do_recv(State = #state{phase = Phase, data = Data, socket = Socket, ppid = PPid}) - when Phase =:= open orelse Phase =:= closing -> - R = case Data of - <> - when L < 126 -> - {F, O, Payload, Rest}; - - <> -> - {F, O, Payload, Rest}; - - <> -> - {F, O, Payload, Rest}; - - <<_:1, _:3, _:4, 1:1, _/binary>> -> - %% According o rfc6455 5.1 the server must not mask any frames. - die(Socket, PPid, {1006, "Protocol error"}, normal); - _ -> - moredata - end, - case R of - moredata -> - State; - _ -> do_recv2(State, R) - end. - -do_recv2(State = #state{phase = Phase, socket = Socket, ppid = PPid}, R) -> - case R of - {1, 1, Payload, Rest} -> - PPid ! {rfc6455, recv, self(), Payload}, - State#state{data = Rest}; - {1, 2, Payload, Rest} -> - PPid ! {rfc6455, recv_binary, self(), Payload}, - State#state{data = Rest}; - {1, 8, Payload, _Rest} -> - WsReason = case Payload of - <> -> {WC, WR}; - <<>> -> {1005, "No status received"} - end, - case Phase of - open -> %% echo - do_close(State, WsReason), - gen_tcp:close(Socket); - closing -> - ok - end, - die(Socket, PPid, WsReason, normal); - {_, _, _, _Rest2} -> - io:format("Unknown frame type~n"), - die(Socket, PPid, {1006, "Unknown frame type"}, normal) - end. - -encode_frame(F, O, Payload) -> - Mask = crypto:strong_rand_bytes(4), - MaskedPayload = apply_mask(Mask, iolist_to_binary(Payload)), - - L = byte_size(MaskedPayload), - IoData = case L of - _ when L < 126 -> - [<>, Mask, MaskedPayload]; - _ when L < 65536 -> - [<>, Mask, MaskedPayload]; - _ -> - [<>, Mask, MaskedPayload] - end, - iolist_to_binary(IoData). - -do_send(State = #state{socket = Socket}, Payload) -> - gen_tcp:send(Socket, encode_frame(1, 1, Payload)), - State. - -do_send_binary(State = #state{socket = Socket}, Payload) -> - gen_tcp:send(Socket, encode_frame(1, 2, Payload)), - State. - -do_close(State = #state{socket = Socket}, {Code, Reason}) -> - Payload = iolist_to_binary([<>, Reason]), - gen_tcp:send(Socket, encode_frame(1, 8, Payload)), - State#state{phase = closing}. - - -loop(State = #state{socket = Socket, ppid = PPid, data = Data, - phase = Phase}) -> - receive - {tcp, Socket, Bin} -> - State1 = State#state{data = iolist_to_binary([Data, Bin])}, - loop(do_recv(State1)); - {send, Payload} when Phase == open -> - loop(do_send(State, Payload)); - {send_binary, Payload} when Phase == open -> - loop(do_send_binary(State, Payload)); - {tcp_closed, Socket} -> - die(Socket, PPid, {1006, "Connection closed abnormally"}, normal); - {close, WsReason} when Phase == open -> - loop(do_close(State, WsReason)) - end. - - -die(Socket, PPid, WsReason, Reason) -> - gen_tcp:shutdown(Socket, read_write), - PPid ! {rfc6455, close, self(), WsReason}, - exit(Reason). - - -%% -------------------------------------------------------------------------- - -split(SubStr, Str, Limit) -> - split(SubStr, Str, Limit, ""). - -split(SubStr, Str, Limit, Default) -> - Acc = split(SubStr, Str, Limit, [], Default), - lists:reverse(Acc). -split(_SubStr, Str, 0, Acc, _Default) -> [Str | Acc]; -split(SubStr, Str, Limit, Acc, Default) -> - {L, R} = case string:str(Str, SubStr) of - 0 -> {Str, Default}; - I -> {string:substr(Str, 1, I-1), - string:substr(Str, I+length(SubStr))} - end, - split(SubStr, R, Limit-1, [L | Acc], Default). - - -apply_mask(Mask, Data) when is_number(Mask) -> - apply_mask(<>, Data); - -apply_mask(<<0:32>>, Data) -> - Data; -apply_mask(Mask, Data) -> - iolist_to_binary(lists:reverse(apply_mask2(Mask, Data, []))). - -apply_mask2(M = <>, <>, Acc) -> - T = Data bxor Mask, - apply_mask2(M, Rest, [<> | Acc]); -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(<>, <>, Acc) -> - T = Data bxor Mask, - [<> | Acc]; -apply_mask2(_, <<>>, Acc) -> - Acc. diff --git a/apps/emqx_management/test/test_utils.erl b/apps/emqx_management/test/test_utils.erl deleted file mode 100644 index 337a9499b..000000000 --- a/apps/emqx_management/test/test_utils.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% @author: -%% @description: --module(test_utils). -%% ==================================================================== -%% API functions -%% ==================================================================== --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_rule_engine/include/rule_engine.hrl"). - --compile([export_all, nowarn_export_all]). - -%% ==================================================================== -%% Internal functions -%% ==================================================================== -resource_is_alive(Id) -> - {ok, #resource_params{status = #{is_alive := Alive}} = Params} = emqx_rule_registry:find_resource_params(Id), - ct:pal("Id: ~p, Alive: ~p, Resource ===> :~p~n", [Id, Alive, Params]), - ?assertEqual(true, Alive), - Alive. diff --git a/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl b/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl index d78b3f18a..2f8fbd017 100644 --- a/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_mod_api_topic_metrics.erl @@ -16,8 +16,6 @@ -module(emqx_mod_api_topic_metrics). --import(minirest, [return/1]). - -rest_api(#{name => list_all_topic_metrics, method => 'GET', path => "/topic-metrics", @@ -203,3 +201,7 @@ rpc_call(Node, Fun, Args) -> {badrpc, Reason} -> {error, Reason}; Res -> Res end. + +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_modules/src/emqx_modules_api.erl b/apps/emqx_modules/src/emqx_modules_api.erl index 3a4b05fd0..99a3b89f9 100644 --- a/apps/emqx_modules/src/emqx_modules_api.erl +++ b/apps/emqx_modules/src/emqx_modules_api.erl @@ -16,8 +16,6 @@ -module(emqx_modules_api). --import(minirest, [return/1]). - -rest_api(#{name => list_all_modules, method => 'GET', path => "/modules/", @@ -167,3 +165,7 @@ name(emqx_mod_presence) -> presence; name(emqx_mod_recon) -> recon; name(emqx_mod_rewrite) -> rewrite; name(emqx_mod_topic_metrics) -> topic_metrics. + +return(_) -> +%% TODO: V5 API + ok. diff --git a/apps/emqx_modules/test/emqx_modules_SUITE.erl b/apps/emqx_modules/test/emqx_modules_SUITE.erl index 897c73d50..0ee097258 100644 --- a/apps/emqx_modules/test/emqx_modules_SUITE.erl +++ b/apps/emqx_modules/test/emqx_modules_SUITE.erl @@ -37,9 +37,8 @@ init_per_suite(Config) -> Config. set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], - default_application_id => <<"admin">>, - default_application_secret => <<"public">>}), + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), ok; set_special_configs(_) -> ok. @@ -59,59 +58,60 @@ t_list(_) -> ?assertMatch([_ | _ ], emqx_modules:list()), emqx_modules:unload(presence). -t_modules_api(_) -> - emqx_modules:load(presence, #{qos => 1}), - timer:sleep(50), - {ok, Modules1} = request_api(get, api_path(["modules"]), auth_header_()), - [Modules11] = filter(get(<<"data">>, Modules1), <<"node">>, atom_to_binary(node(), utf8)), - [Module1] = filter(maps:get(<<"modules">>, Modules11), <<"name">>, <<"presence">>), - ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module1)), - {ok, _} = request_api(put, - api_path(["modules", - atom_to_list(presence), - "unload"]), - auth_header_()), - {ok, Error1} = request_api(put, - api_path(["modules", - atom_to_list(presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), - {ok, Modules2} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module2] = filter(get(<<"data">>, Modules2), <<"name">>, <<"presence">>), - ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module2)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(presence), - "load"]), - auth_header_()), - {ok, Modules3} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module3] = filter(get(<<"data">>, Modules3), <<"name">>, <<"presence">>), - ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module3)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(presence), - "unload"]), - auth_header_()), - {ok, Error2} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), - emqx_modules:unload(presence). +%% TODO: V5 API +%%t_modules_api(_) -> +%% emqx_modules:load(presence, #{qos => 1}), +%% timer:sleep(50), +%% {ok, Modules1} = request_api(get, api_path(["modules"]), auth_header_()), +%% [Modules11] = filter(get(<<"data">>, Modules1), <<"node">>, atom_to_binary(node(), utf8)), +%% [Module1] = filter(maps:get(<<"modules">>, Modules11), <<"name">>, <<"presence">>), +%% ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module1)), +%% {ok, _} = request_api(put, +%% api_path(["modules", +%% atom_to_list(presence), +%% "unload"]), +%% auth_header_()), +%% {ok, Error1} = request_api(put, +%% api_path(["modules", +%% atom_to_list(presence), +%% "unload"]), +%% auth_header_()), +%% ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), +%% {ok, Modules2} = request_api(get, +%% api_path(["nodes", atom_to_list(node()), "modules"]), +%% auth_header_()), +%% [Module2] = filter(get(<<"data">>, Modules2), <<"name">>, <<"presence">>), +%% ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module2)), +%% +%% {ok, _} = request_api(put, +%% api_path(["nodes", +%% atom_to_list(node()), +%% "modules", +%% atom_to_list(presence), +%% "load"]), +%% auth_header_()), +%% {ok, Modules3} = request_api(get, +%% api_path(["nodes", atom_to_list(node()), "modules"]), +%% auth_header_()), +%% [Module3] = filter(get(<<"data">>, Modules3), <<"name">>, <<"presence">>), +%% ?assertEqual(<<"presence">>, maps:get(<<"name">>, Module3)), +%% +%% {ok, _} = request_api(put, +%% api_path(["nodes", +%% atom_to_list(node()), +%% "modules", +%% atom_to_list(presence), +%% "unload"]), +%% auth_header_()), +%% {ok, Error2} = request_api(put, +%% api_path(["nodes", +%% atom_to_list(node()), +%% "modules", +%% atom_to_list(presence), +%% "unload"]), +%% auth_header_()), +%% ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), +%% emqx_modules:unload(presence). t_modules_cmd(_) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 29acc72f6..04ebd78d3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -25,8 +25,6 @@ -include_lib("prometheus/include/prometheus.hrl"). -include_lib("prometheus/include/prometheus_model.hrl"). --import(minirest, [return/1]). - -rest_api(#{name => stats, method => 'GET', path => "/emqx_prometheus", @@ -610,3 +608,7 @@ emqx_cluster_data() -> #{running_nodes := Running, stopped_nodes := Stopped} = ekka_mnesia:cluster_info(), [{nodes_running, length(Running)}, {nodes_stopped, length(Stopped)}]. + +%% TODO: V5 API +return(_) -> + ok. diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index e9517f160..ee8cb6067 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -20,7 +20,7 @@ , max/2 , equals/2 , enum/1 - , required/1 + , not_empty/1 ]). max(Type, Max) -> @@ -38,8 +38,8 @@ enum(Items) -> err_limit({enum, {is_member_of, Items}, {got, Value}})) end. -required(ErrMsg) -> - fun(undefined) -> {error, ErrMsg}; +not_empty(ErrMsg) -> + fun(<<>>) -> {error, ErrMsg}; (_) -> ok end. diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 237a3b19c..1b5b8adcc 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -36,7 +36,7 @@ lookup_config(_Bindings, _Params) -> Config = emqx_config:get([emqx_retainer]), - minirest:return({ok, Config}). + return({ok, Config}). update_config(_Bindings, Params) -> try @@ -47,9 +47,9 @@ update_config(_Bindings, Params) -> #{emqx_retainer := Conf} = hocon_schema:richmap_to_map(RichConf), Action = proplists:get_value(<<"action">>, Params, undefined), do_update_config(Action, Conf), - minirest:return() + return() catch _:_:Reason -> - minirest:return({error, Reason}) + return({error, Reason}) end. %%------------------------------------------------------------------------------ @@ -59,3 +59,9 @@ do_update_config(undefined, Config) -> emqx_retainer:update_config(Config); do_update_config(<<"test">>, _) -> ok. + +%% TODO: V5 API +return() -> + ok. +return(_) -> + ok. diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 57196c7bd..1f5a32542 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -35,7 +35,9 @@ -define(BASE_PATH, "api"). all() -> - emqx_ct:all(?MODULE). +%% TODO: V5 API +%% emqx_ct:all(?MODULE). + []. groups() -> []. @@ -56,9 +58,8 @@ init_per_testcase(_, Config) -> set_special_configs(emqx_retainer) -> init_emqx_retainer_conf(0); set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], - default_application_id => <<"admin">>, - default_application_secret => <<"public">>}), + emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + applications =>[#{id => "admin", secret => "public"}]}), ok; set_special_configs(_) -> ok. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 4fa3b8aa3..24b4d2c13 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -21,8 +21,6 @@ -logger_header("[RuleEngineAPI]"). --import(minirest, [return/1]). - -rest_api(#{name => create_rule, method => 'POST', path => "/rules/", @@ -552,3 +550,6 @@ get_rule_metrics(Id) -> get_action_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) || Node <- ekka_mnesia:running_nodes()]. + +%% TODO: V5 API +return(_) -> ok. \ No newline at end of file diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index da3e963f0..a056d0c26 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -54,14 +54,15 @@ groups() -> [t_inspect_action ,t_republish_action ]}, - {api, [], - [t_crud_rule_api, - t_list_actions_api, - t_show_action_api, - t_crud_resources_api, - t_list_resource_types_api, - t_show_resource_type_api - ]}, +%% TODO: V5 API +%% {api, [], +%% [t_crud_rule_api, +%% t_list_actions_api, +%% t_show_action_api, +%% t_crud_resources_api, +%% t_list_resource_types_api, +%% t_show_resource_type_api +%% ]}, {cli, [], [t_rules_cli, t_actions_cli, diff --git a/apps/emqx_telemetry/src/emqx_telemetry_api.erl b/apps/emqx_telemetry/src/emqx_telemetry_api.erl index 8bb97086e..798d114eb 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry_api.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry_api.erl @@ -44,8 +44,6 @@ , get_telemetry_data/0 ]). --import(minirest, [return/1]). - %%-------------------------------------------------------------------- %% CLI %%-------------------------------------------------------------------- @@ -129,3 +127,6 @@ rpc_call(Node, Module, Fun, Args) -> {badrpc, Reason} -> {error, Reason}; Result -> Result end. + +%% TODO: V5 API +return(_) -> ok. diff --git a/bin/emqx b/bin/emqx index 9f4aa9bd4..50e6dda02 100755 --- a/bin/emqx +++ b/bin/emqx @@ -3,6 +3,11 @@ # ex: ts=4 sw=4 et set -e +set -o pipefail + +if [ -n "$DEBUG" ]; then + set -x +fi ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 @@ -197,6 +202,7 @@ call_hocon() { export RUNNER_ETC_DIR export REL_VSN "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" + return $? } # Run an escript in the node's environment @@ -252,13 +258,13 @@ generate_config() { ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') ## use the key to look up in vm.args file for the value - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" || true | awk '{print $NF}') ## compare generated (to override) value to original (to be overriden) value if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then ## if they are different if [ -n "$TMP_ARG_VALUE" ]; then ## if the old value is present, replace it with generated value - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" + sh -c "$SED_REPLACE 's|^$ARG_KEY.*$|$ARG_LINE|' $TMP_ARG_FILE" else ## otherwise append generated value to the end echo "$ARG_LINE" >> "$TMP_ARG_FILE" @@ -366,7 +372,7 @@ if [ -z "$COOKIE" ]; then fi # Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460 -PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') +PROTO_DIST="$(call_hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get cluster.proto_dist | tr -d \")" if [ -z "$PROTO_DIST" ]; then PROTO_DIST_ARG="" else diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 535ccb8e1..446b26f07 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -63,6 +63,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. |nil| | `service.loadBalancerIP` | loadBalancerIP for Service | nil | | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | +| `service.externalIPs` | ExternalIPs for the service | [] | | `service.annotations` | Service annotations | {}(evaluated as a template)| | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 4cad21569..6ebbf5121 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -162,7 +162,7 @@ spec: {{ end }} readinessProbe: httpGet: - path: /status + path: /api/v5/status port: {{ .Values.emqxConfig.EMQX_MANAGEMENT__LISTENER__HTTP | default 8081 }} initialDelaySeconds: 5 periodSeconds: 5 diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index b3c0d6017..6e31a97c3 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -21,6 +21,9 @@ spec: {{- if .Values.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} {{- end }} + {{- if .Values.service.externalIPs }} + externalIPs: {{- toYaml .Values.service.externalIPs | nindent 4 }} + {{- end }} {{- end }} ports: - name: mqtt diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 3e145aafe..36e9be47a 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -145,6 +145,9 @@ service: ## - 10.10.10.0/24 ## loadBalancerSourceRanges: [] + ## Set the ExternalIPs + ## + externalIPs: [] ## Provide any additional annotations which may be required. Evaluated as a template ## annotations: {} diff --git a/rebar.config b/rebar.config index 37380db13..3e7b9be01 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.6"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.7"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} @@ -51,19 +51,18 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} % TODO: delete when all apps moved to hocon - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.6"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.1.1"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} - , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.1"}}} + , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.2"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.6"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.5"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index de42601af..258cd0bf3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -15,12 +15,14 @@ do(Dir, CONFIG) -> bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}. +quicer() -> + %% @todo use tag + {quicer, {git, "https://github.com/emqx/quic.git", {branch, "main"}}}. + deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), - MoreDeps = case provide_bcrypt_dep() of - true -> [bcrypt()]; - false -> [] - end, + MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++ + [quicer() || is_quicer_supported()], {HasElixir, ExtraDeps} = extra_deps(), {HasElixir, lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ ExtraDeps})}. @@ -78,6 +80,24 @@ is_cover_enabled() -> is_enterprise() -> filelib:is_regular("EMQX_ENTERPRISE"). +is_quicer_supported() -> + not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse + is_win32() orelse is_centos_6() + ). + +is_centos_6() -> + %% reason: + %% glibc is too old + case file:read_file("/etc/centos-release") of + {ok, <<"CentOS release 6", _/binary >>} -> + true; + _ -> + false + end. + +is_win32() -> + win32 =:= element(1, os:type()). + project_app_dirs() -> ["apps/*"] ++ case is_enterprise() of @@ -242,7 +262,6 @@ relx_apps(ReleaseType) -> , compiler , runtime_tools , cuttlefish - , quicer , emqx , {mnesia, load} , {ekka, load} @@ -263,6 +282,7 @@ relx_apps(ReleaseType) -> , emqx_retainer , emqx_statsd ] + ++ [quicer || is_quicer_supported()] ++ [emqx_telemetry || not is_enterprise()] ++ [emqx_license || is_enterprise()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript index d875a2c40..3cc8fdc53 100755 --- a/scripts/check-deps-integrity.escript +++ b/scripts/check-deps-integrity.escript @@ -48,7 +48,7 @@ do_collect_deps([{Name, Ref} | Deps], File, Acc) -> count_bad_deps([]) -> 0; count_bad_deps([{Name, Refs0} | Rest]) -> Refs = lists:keysort(1, Refs0), - case is_unique_ref(Refs) of + case is_unique_ref(Refs) andalso not_branch_ref(Refs) of true -> count_bad_deps(Rest); false -> @@ -61,3 +61,7 @@ is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> is_unique_ref([{Ref, File2} | Rest]); is_unique_ref(_) -> false. + +not_branch_ref([]) -> true; +not_branch_ref([{{git, _Repo, {branch, _Branch}}, _File} | _Rest]) -> false; +not_branch_ref([_Ref | Rest]) -> not_branch_ref(Rest).