yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8578 from emqx/master 

Merge master into dev/ee5.0
This commit is contained in:
DDDHuang 2022-07-27 11:34:22 +08:00 committed by GitHub
parent e4eba157c3 ccc734c6ee
commit c9dedd7132
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
74 changed files with 786 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/build_slim_packages.yaml vendored
View File

@ -76,7 +76,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
path: _packages/${{ matrix.profile}}/*.tar.gz
path: _packages/${{ matrix.profile}}/*
- uses: actions/upload-artifact@v2
with:
name: "${{ matrix.profile }}_schema_dump"
@ -120,7 +120,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: windows
path: _packages/${{ matrix.profile}}/*.tar.gz
path: _packages/${{ matrix.profile}}/*
mac:
strategy:
@ -195,7 +195,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: macos
path: _packages/**/*.tar.gz
path: _packages/**/*
spellcheck:
needs: linux

6
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -157,6 +157,10 @@ jobs:
if: matrix.discovery == 'k8s'
run: |
helm install emqx \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
--set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
--set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="emqx-headless" \
--set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
--set image.repository=$TARGET \
--set image.pullPolicy=Never \
--set emqxAclConfig="" \
@ -173,8 +177,8 @@ jobs:
run: |
helm install emqx \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
--set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
--set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
--set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
--set image.repository=$TARGET \
--set image.pullPolicy=Never \
--set emqxAclConfig="" \

9
CHANGES-5.0.md
View File

@ -17,13 +17,20 @@
* Fix `chars_limit` is not working when `formatter` is `json`. [#8518](http://github.com/emqx/emqx/pull/8518)
* Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530)
* Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528)
* Fix AuthN `cert_subject` and `cert_common_name` placeholder rendering failure. [#8531](https://github.com/emqx/emqx/pull/8531)
* Support listen on an IPv6 address, e.g: [::1]:1883 or ::1:1883. [#8547](https://github.com/emqx/emqx/pull/8547)
* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472)
**‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become
`{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`,
which will carry the paging meta information.
## Enhancements
* Improve the dashboard listener startup log, the listener name is no longer spliced with port information,
and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480)
* Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485)
* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439)
* Optimize performance of builtin database operations in processes with long message queue [#8439](https://github.com/emqx/emqx/pull/8439)
* Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554)
# 5.0.3

2
Makefile
View File

@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.0.3
export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
ifeq ($(OS),Windows_NT)

4
README-CN.md
View File

@ -4,10 +4,10 @@
[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com)
[![Community](https://img.shields.io/badge/Community-EMQX-yellow)](https://askemq.com)
[![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg)

2
README-JP.md
View File

@ -4,7 +4,7 @@
[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
[![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)

4
README-RU.md
View File

@ -4,10 +4,10 @@
[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions)
[![Community](https://img.shields.io/badge/Community-EMQX-yellow?logo=github)](https://github.com/emqx/emqx/discussions)
[![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
[![The best IoT MQTT open source team looks forward to your joining](https://assets.emqx.com/images/github_readme_en_bg.png)](https://www.emqx.com/en/careers)

2
README.md
View File

@ -4,7 +4,7 @@
[![Build Status](https://img.shields.io/travis/emqx/emqx?label=Build)](https://travis-ci.org/emqx/emqx)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
[![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)

13
apps/emqx/include/emqx_authentication.hrl
View File

@ -17,6 +17,19 @@
-ifndef(EMQX_AUTHENTICATION_HRL).
-define(EMQX_AUTHENTICATION_HRL, true).
-include_lib("emqx/include/logger.hrl").
-define(AUTHN_TRACE_TAG, "AUTHN").
-define(TRACE_AUTHN_PROVIDER(Msg), ?TRACE_AUTHN_PROVIDER(Msg, #{})).
-define(TRACE_AUTHN_PROVIDER(Msg, Meta), ?TRACE_AUTHN_PROVIDER(debug, Msg, Meta)).
-define(TRACE_AUTHN_PROVIDER(Level, Msg, Meta),
?TRACE_AUTHN(Level, Msg, (Meta)#{provider => ?MODULE})
).
-define(TRACE_AUTHN(Msg, Meta), ?TRACE_AUTHN(debug, Msg, Meta)).
-define(TRACE_AUTHN(Level, Msg, Meta), ?TRACE(Level, ?AUTHN_TRACE_TAG, Msg, Meta)).
%% config root name all auth providers have to agree on.
-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication").
-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication).

18
apps/emqx/include/logger.hrl
View File

@ -42,17 +42,21 @@
-define(TRACE_FILTER, emqx_trace_filter).
-define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)).
%% Only evaluate when necessary
%% Always debug the trace events.
-define(TRACE(Tag, Msg, Meta), begin
case persistent_term:get(?TRACE_FILTER, undefined) of
undefined -> ok;
-define(TRACE(Level, Tag, Msg, Meta), begin
case persistent_term:get(?TRACE_FILTER, []) of
[] -> ok;
List -> emqx_trace:log(List, Msg, Meta#{trace_tag => Tag})
%% We can't bind filter list to a variablebecause we pollute the calling scope with it.
%% We also don't want to wrap the macro body in a fun
%% beacause this adds overhead to the happy path.
%% So evaluate `persistent_term:get` twice.
_ -> emqx_trace:log(persistent_term:get(?TRACE_FILTER, []), Msg, (Meta)#{trace_tag => Tag})
end,
?SLOG(
debug,
(emqx_trace_formatter:format_meta(Meta))#{msg => Msg, tag => Tag},
Level,
(emqx_trace_formatter:format_meta_map(Meta))#{msg => Msg, tag => Tag},
#{is_trace => false}
)
end).

2
apps/emqx/rebar.config
View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

31
apps/emqx/src/emqx_authentication.erl
View File

@ -101,6 +101,14 @@
-define(CHAINS_TAB, emqx_authn_chains).
-define(TRACE_RESULT(Label, Result, Reason), begin
?TRACE_AUTHN(Label, #{
result => (Result),
reason => (Reason)
}),
Result
end).
-type chain_name() :: atom().
-type authenticator_id() :: binary().
-type position() :: front | rear | {before, authenticator_id()} | {'after', authenticator_id()}.
@ -216,14 +224,14 @@ when
authenticate(#{enable_authn := false}, _AuthResult) ->
inc_authenticate_metric('authentication.success.anonymous'),
ignore;
?TRACE_RESULT("authentication_result", ignore, enable_authn_false);
authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) ->
case get_authenticators(Listener, global_chain(Protocol)) of
{ok, ChainName, Authenticators} ->
case get_enabled(Authenticators) of
[] ->
inc_authenticate_metric('authentication.success.anonymous'),
ignore;
?TRACE_RESULT("authentication_result", ignore, empty_chain);
NAuthenticators ->
Result = do_authenticate(ChainName, NAuthenticators, Credential),
@ -235,11 +243,11 @@ authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthRe
_ ->
ok
end,
Result
?TRACE_RESULT("authentication_result", Result, chain_result)
end;
none ->
inc_authenticate_metric('authentication.success.anonymous'),
ignore
?TRACE_RESULT("authentication_result", ignore, no_chain)
end.
get_authenticators(Listener, Global) ->
@ -626,11 +634,11 @@ handle_create_authenticator(Chain, Config, Providers) ->
do_authenticate(_ChainName, [], _) ->
{stop, {error, not_authorized}};
do_authenticate(
ChainName, [#authenticator{id = ID, provider = Provider, state = State} | More], Credential
ChainName, [#authenticator{id = ID} = Authenticator | More], Credential
) ->
MetricsID = metrics_id(ChainName, ID),
emqx_metrics_worker:inc(authn_metrics, MetricsID, total),
try Provider:authenticate(Credential, State) of
try authenticate_with_provider(Authenticator, Credential) of
ignore ->
ok = emqx_metrics_worker:inc(authn_metrics, MetricsID, nomatch),
do_authenticate(ChainName, More, Credential);
@ -651,8 +659,7 @@ do_authenticate(
{stop, Result}
catch
Class:Reason:Stacktrace ->
?SLOG(warning, #{
msg => "unexpected_error_in_authentication",
?TRACE_AUTHN(warning, "authenticator_error", #{
exception => Class,
reason => Reason,
stacktrace => Stacktrace,
@ -662,6 +669,14 @@ do_authenticate(
do_authenticate(ChainName, More, Credential)
end.
authenticate_with_provider(#authenticator{id = ID, provider = Provider, state = State}, Credential) ->
AuthnResult = Provider:authenticate(Credential, State),
?TRACE_AUTHN("authenticator_result", #{
authenticator => ID,
result => AuthnResult
}),
AuthnResult.
reply(Reply, State) ->
{reply, Reply, State}.

28
apps/emqx/src/emqx_schema.erl
View File

@ -2113,9 +2113,13 @@ to_comma_separated_atoms(Str) ->
to_bar_separated_list(Str) ->
{ok, string:tokens(Str, "| ")}.
%% @doc support the following format:
%% - 127.0.0.1:1883
%% - ::1:1883
%% - [::1]:1883
to_ip_port(Str) ->
case string:tokens(Str, ": ") of
[Ip, Port] ->
case split_ip_port(Str) of
{Ip, Port} ->
PortVal = list_to_integer(Port),
case inet:parse_address(Ip) of
{ok, R} ->
@ -2133,6 +2137,26 @@ to_ip_port(Str) ->
{error, Str}
end.
split_ip_port(Str0) ->
Str = re:replace(Str0, " ", "", [{return, list}, global]),
case lists:split(string:rchr(Str, $:), Str) of
%% no port
{[], Str} ->
error;
{IpPlusColon, PortString} ->
IpStr0 = lists:droplast(IpPlusColon),
case IpStr0 of
%% dropp head/tail brackets
[$[ | S] ->
case lists:last(S) of
$] -> {lists:droplast(S), PortString};
_ -> error
end;
_ ->
{IpStr0, PortString}
end
end.
to_erl_cipher_suite(Str) ->
case ssl:str_to_suite(Str) of
{error, Reason} -> error({invalid_cipher, Reason});

15
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -92,15 +92,16 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) ->
unsubscribe(Topic, SubOpts) ->
?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
log(List, Msg, Meta0) ->
Meta =
case logger:get_process_metadata() of
undefined -> Meta0;
ProcMeta -> maps:merge(ProcMeta, Meta0)
end,
Log = #{level => debug, meta => Meta, msg => Msg},
log(List, Msg, Meta) ->
Log = #{level => debug, meta => enrich_meta(Meta), msg => Msg},
log_filter(List, Log).
enrich_meta(Meta) ->
case logger:get_process_metadata() of
undefined -> Meta;
ProcMeta -> maps:merge(ProcMeta, Meta)
end.
log_filter([], _Log) ->
ok;
log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->

53
apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
View File

@ -16,7 +16,7 @@
-module(emqx_trace_formatter).
-export([format/2]).
-export([format_meta/1]).
-export([format_meta_map/1]).
%%%-----------------------------------------------------------------
%%% API
@ -31,32 +31,39 @@ format(
ClientId = to_iolist(maps:get(clientid, Meta, "")),
Peername = maps:get(peername, Meta, ""),
MetaBin = format_meta(Meta, PEncode),
[Time, " [", Tag, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
[Time, " [", Tag, "] ", ClientId, "@", Peername, " msg: ", Msg, ", ", MetaBin, "\n"];
format(Event, Config) ->
emqx_logger_textfmt:format(Event, Config).
format_meta(Meta) ->
format_meta_map(Meta) ->
Encode = emqx_trace_handler:payload_encode(),
do_format_meta(Meta, Encode).
format_meta_map(Meta, Encode).
format_meta(Meta0, Encode) ->
Meta1 = #{packet := Packet0, payload := Payload0} = do_format_meta(Meta0, Encode),
Packet = enrich(", packet: ", Packet0),
Payload = enrich(", payload: ", Payload0),
Meta2 = maps:without([msg, clientid, peername, packet, payload, trace_tag], Meta1),
case Meta2 =:= #{} of
true -> [Packet, Payload];
false -> [Packet, ", ", map_to_iolist(Meta2), Payload]
format_meta_map(Meta, Encode) ->
format_meta_map(Meta, Encode, [{packet, fun format_packet/2}, {payload, fun format_payload/2}]).
format_meta_map(Meta, _Encode, []) ->
Meta;
format_meta_map(Meta, Encode, [{Name, FormatFun} | Rest]) ->
case Meta of
#{Name := Value} ->
NewMeta = Meta#{Name => FormatFun(Value, Encode)},
format_meta_map(NewMeta, Encode, Rest);
#{} ->
format_meta_map(Meta, Encode, Rest)
end.
enrich(_, "") -> "";
enrich(Key, IoData) -> [Key, IoData].
format_meta(Meta0, Encode) ->
Meta1 = maps:without([msg, clientid, peername, trace_tag], Meta0),
Meta2 = format_meta_map(Meta1, Encode),
kvs_to_iolist(lists:sort(fun compare_meta_kvs/2, maps:to_list(Meta2))).
do_format_meta(Meta, Encode) ->
Meta#{
packet => format_packet(maps:get(packet, Meta, undefined), Encode),
payload => format_payload(maps:get(payload, Meta, undefined), Encode)
}.
%% packet always goes first; payload always goes last
compare_meta_kvs(KV1, KV2) -> weight(KV1) =< weight(KV2).
weight({packet, _}) -> {0, packet};
weight({payload, _}) -> {2, payload};
weight({K, _}) -> {1, K}.
format_packet(undefined, _) -> "";
format_packet(Packet, Encode) -> emqx_packet:format(Packet, Encode).
@ -69,14 +76,14 @@ format_payload(_, hidden) -> "******".
to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"];
to_iolist(SubMap) when is_map(SubMap) -> ["[", kvs_to_iolist(maps:to_list(SubMap)), "]"];
to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char).
map_to_iolist(Map) ->
kvs_to_iolist(KVs) ->
lists:join(
",",
", ",
lists:map(
fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
maps:to_list(Map)
KVs
)
).

4
apps/emqx_authn/include/emqx_authn.hrl
View File

@ -36,6 +36,6 @@
-type authenticator_id() :: binary().
-endif.
-define(RESOURCE_GROUP, <<"emqx_authn">>).
-endif.

10
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -33,14 +33,8 @@
% Swagger
-define(API_TAGS_GLOBAL, [
?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(global)">>
]).
-define(API_TAGS_SINGLE, [
?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(single listener)">>
]).
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
-export([
api_spec/0,

11
apps/emqx_authn/src/emqx_authn_user_import_api.erl
View File

@ -29,15 +29,8 @@
-define(NOT_FOUND, 'NOT_FOUND').
% Swagger
-define(API_TAGS_GLOBAL, [
?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(global)">>
]).
-define(API_TAGS_SINGLE, [
?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(single listener)">>
]).
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
-export([
api_spec/0,

27
apps/emqx_authn/src/emqx_authn_utils.erl
View File

@ -33,7 +33,8 @@
bin/1,
ensure_apps_started/1,
cleanup_resources/0,
make_resource_id/1
make_resource_id/1,
without_password/1
]).
-define(AUTHN_PLACEHOLDERS, [
@ -117,21 +118,21 @@ parse_sql(Template, ReplaceWith) ->
render_deep(Template, Credential) ->
emqx_placeholder:proc_tmpl_deep(
Template,
Credential,
mapping_credential(Credential),
#{return => full_binary, var_trans => fun handle_var/2}
).
render_str(Template, Credential) ->
emqx_placeholder:proc_tmpl(
Template,
Credential,
mapping_credential(Credential),
#{return => full_binary, var_trans => fun handle_var/2}
).
render_sql_params(ParamList, Credential) ->
emqx_placeholder:proc_tmpl(
ParamList,
Credential,
mapping_credential(Credential),
#{return => rawlist, var_trans => fun handle_sql_var/2}
).
@ -199,10 +200,23 @@ make_resource_id(Name) ->
NameBin = bin(Name),
emqx_resource:generate_id(NameBin).
without_password(Credential) ->
without_password(Credential, [password, <<"password">>]).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
without_password(Credential, []) ->
Credential;
without_password(Credential, [Name | Rest]) ->
case maps:is_key(Name, Credential) of
true ->
without_password(Credential#{Name => <<"[password]">>}, Rest);
false ->
without_password(Credential, Rest)
end.
handle_var({var, Name}, undefined) ->
error({cannot_get_variable, Name});
handle_var({var, <<"peerhost">>}, PeerHost) ->
@ -216,3 +230,8 @@ handle_sql_var({var, <<"peerhost">>}, PeerHost) ->
emqx_placeholder:bin(inet:ntoa(PeerHost));
handle_sql_var(_, Value) ->
emqx_placeholder:sql_data(Value).
mapping_credential(C = #{cn := CN, dn := DN}) ->
C#{cert_common_name => CN, cert_subject => DN};
mapping_credential(C) ->
C.

10
apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
View File

@ -331,7 +331,10 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S
{continue, ServerFirstMessage, Cache};
ignore ->
ignore;
{error, _Reason} ->
{error, Reason} ->
?TRACE_AUTHN_PROVIDER("check_client_first_message_error", #{
reason => Reason
}),
{error, not_authorized}
end.
@ -344,7 +347,10 @@ check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algori
of
{ok, ServerFinalMessage} ->
{ok, #{is_superuser => IsSuperuser}, ServerFinalMessage};
{error, _Reason} ->
{error, Reason} ->
?TRACE_AUTHN_PROVIDER("check_client_final_message_error", #{
reason => Reason
}),
{error, not_authorized}
end.

68
apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
View File

@ -188,23 +188,22 @@ authenticate(
} = State
) ->
Request = generate_request(Credential, State),
case emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}) of
Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
?TRACE_AUTHN_PROVIDER("http_response", #{
request => request_for_log(Credential, State),
response => response_for_log(Response),
resource => ResourceId
}),
case Response of
{ok, 204, _Headers} ->
{ok, #{is_superuser => false}};
{ok, 200, Headers, Body} ->
handle_response(Headers, Body);
{ok, _StatusCode, _Headers} = Response ->
log_response(ResourceId, Response),
ignore;
{ok, _StatusCode, _Headers, _Body} = Response ->
log_response(ResourceId, Response),
ignore;
{error, Reason} ->
?SLOG(error, #{
msg => "http_server_query_failed",
resource => ResourceId,
reason => Reason
}),
{error, _Reason} ->
ignore
end.
@ -296,7 +295,8 @@ parse_config(
cow_qs:parse_qs(to_bin(Query))
),
body_template => emqx_authn_utils:parse_deep(maps:get(body, Config, #{})),
request_timeout => RequestTimeout
request_timeout => RequestTimeout,
url => RawUrl
},
{Config#{base_url => BaseUrl, pool_type => random}, State}.
@ -379,11 +379,6 @@ parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) ->
parse_body(ContentType, _) ->
{error, {unsupported_content_type, ContentType}}.
may_append_body(Output, {ok, _, _, Body}) ->
Output#{body => Body};
may_append_body(Output, {ok, _, _}) ->
Output.
uri_encode(T) ->
emqx_http_lib:uri_encode(to_list(T)).
@ -391,26 +386,33 @@ encode_path(Path) ->
Parts = string:split(Path, "/", all),
lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]).
log_response(ResourceId, Other) ->
Output = may_append_body(#{resource => ResourceId}, Other),
case erlang:element(2, Other) of
Code5xx when Code5xx >= 500 andalso Code5xx < 600 ->
?SLOG(error, Output#{
msg => "http_server_error",
code => Code5xx
});
Code4xx when Code4xx >= 400 andalso Code4xx < 500 ->
?SLOG(warning, Output#{
msg => "refused_by_http_server",
code => Code4xx
});
OtherCode ->
?SLOG(error, Output#{
msg => "undesired_response_code",
code => OtherCode
})
request_for_log(Credential, #{url := Url} = State) ->
SafeCredential = emqx_authn_utils:without_password(Credential),
case generate_request(SafeCredential, State) of
{PathQuery, Headers} ->
#{
method => post,
base_url => Url,
path_query => PathQuery,
headers => Headers
};
{PathQuery, Headers, Body} ->
#{
method => post,
base_url => Url,
path_query => PathQuery,
headers => Headers,
mody => Body
}
end.
response_for_log({ok, StatusCode, Headers}) ->
#{status => StatusCode, headers => Headers};
response_for_log({ok, StatusCode, Headers, Body}) ->
#{status => StatusCode, headers => Headers, body => Body};
response_for_log({error, Error}) ->
#{error => Error}.
to_list(A) when is_atom(A) ->
atom_to_list(A);
to_list(B) when is_binary(B) ->

34
apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
View File

@ -227,8 +227,7 @@ authenticate(
) ->
case emqx_resource:query(ResourceId, get_jwks) of
{error, Reason} ->
?SLOG(error, #{
msg => "get_jwks_failed",
?TRACE_AUTHN_PROVIDER(error, "get_jwks_failed", #{
resource => ResourceId,
reason => Reason
}),
@ -350,10 +349,17 @@ verify(undefined, _, _, _) ->
ignore;
verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
case do_verify(JWT, JWKs, VerifyClaims) of
{ok, Extra} -> {ok, acl(Extra, AclClaimName)};
{error, {missing_claim, _}} -> {error, bad_username_or_password};
{error, invalid_signature} -> ignore;
{error, {claims, _}} -> {error, bad_username_or_password}
{ok, Extra} ->
{ok, acl(Extra, AclClaimName)};
{error, {missing_claim, Claim}} ->
?TRACE_AUTHN_PROVIDER("missing_jwt_claim", #{jwt => JWT, claim => Claim}),
{error, bad_username_or_password};
{error, invalid_signature} ->
?TRACE_AUTHN_PROVIDER("invalid_jwt_signature", #{jwks => JWKs, jwt => JWT}),
ignore;
{error, {claims, Claims}} ->
?TRACE_AUTHN_PROVIDER("invalid_jwt_claims", #{jwt => JWT, claims => Claims}),
{error, bad_username_or_password}
end.
acl(Claims, AclClaimName) ->
@ -371,11 +377,11 @@ acl(Claims, AclClaimName) ->
end,
maps:merge(emqx_authn_utils:is_superuser(Claims), Acl).
do_verify(_JWS, [], _VerifyClaims) ->
do_verify(_JWT, [], _VerifyClaims) ->
{error, invalid_signature};
do_verify(JWS, [JWK | More], VerifyClaims) ->
try jose_jws:verify(JWK, JWS) of
{true, Payload, _JWS} ->
do_verify(JWT, [JWK | More], VerifyClaims) ->
try jose_jws:verify(JWK, JWT) of
{true, Payload, _JWT} ->
Claims0 = emqx_json:decode(Payload, [return_maps]),
Claims = try_convert_to_int(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]),
case verify_claims(Claims, VerifyClaims) of
@ -385,11 +391,11 @@ do_verify(JWS, [JWK | More], VerifyClaims) ->
{error, Reason}
end;
{false, _, _} ->
do_verify(JWS, More, VerifyClaims)
do_verify(JWT, More, VerifyClaims)
catch
_:_Reason ->
?TRACE("JWT", "authn_jwt_invalid_signature", #{jwk => JWK, jws => JWS}),
{error, invalid_signature}
_:Reason ->
?TRACE_AUTHN_PROVIDER("jwt_verify_error", #{jwk => JWK, jwt => JWT, reason => Reason}),
do_verify(JWT, More, VerifyClaims)
end.
verify_claims(Claims, VerifyClaims0) ->

8
apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
View File

@ -17,6 +17,7 @@
-module(emqx_authn_mnesia).
-include("emqx_authn.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include_lib("hocon/include/hoconsc.hrl").
@ -158,6 +159,7 @@ authenticate(
UserID = get_user_identity(Credential, Type),
case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of
[] ->
?TRACE_AUTHN_PROVIDER("user_not_found"),
ignore;
[#user_info{password_hash = PasswordHash, salt = Salt, is_superuser = IsSuperuser}] ->
case
@ -165,8 +167,10 @@ authenticate(
Algorithm, Salt, PasswordHash, Password
)
of
true -> {ok, #{is_superuser => IsSuperuser}};
false -> {error, bad_username_or_password}
true ->
{ok, #{is_superuser => IsSuperuser}};
false ->
{error, bad_username_or_password}
end
end.

7
apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
View File

@ -167,8 +167,7 @@ authenticate(
undefined ->
ignore;
{error, Reason} ->
?SLOG(error, #{
msg => "mongodb_query_failed",
?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
resource => ResourceId,
collection => Collection,
filter => Filter,
@ -180,11 +179,11 @@ authenticate(
ok ->
{ok, is_superuser(Doc, State)};
{error, {cannot_find_password_hash_field, PasswordHashField}} ->
?SLOG(error, #{
msg => "cannot_find_password_hash_field",
?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{
resource => ResourceId,
collection => Collection,
filter => Filter,
document => Doc,
password_hash_field => PasswordHashField
}),
ignore;

3
apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
View File

@ -130,8 +130,7 @@ authenticate(
{error, Reason}
end;
{error, Reason} ->
?SLOG(error, #{
msg => "mysql_query_failed",
?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{
resource => ResourceId,
tmpl_token => TmplToken,
params => Params,

3
apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
View File

@ -133,8 +133,7 @@ authenticate(
{error, Reason}
end;
{error, Reason} ->
?SLOG(error, #{
msg => "postgresql_query_failed",
?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{
resource => ResourceId,
params => Params,
reason => Reason

8
apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
View File

@ -128,13 +128,14 @@ authenticate(#{auth_method := _}, _) ->
authenticate(
#{password := Password} = Credential,
#{
cmd := {Command, KeyTemplate, Fields},
cmd := {CommandName, KeyTemplate, Fields},
resource_id := ResourceId,
password_hash_algorithm := Algorithm
}
) ->
NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of
Command = [CommandName, NKey | Fields],
case emqx_resource:query(ResourceId, {cmd, Command}) of
{ok, []} ->
ignore;
{ok, Values} ->
@ -150,8 +151,7 @@ authenticate(
{error, Reason}
end;
{error, Reason} ->
?SLOG(error, #{
msg => "redis_query_failed",
?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{
resource => ResourceId,
cmd => Command,
keys => NKey,

12
apps/emqx_authn/test/emqx_authn_http_SUITE.erl
View File

@ -34,7 +34,9 @@
password => <<"plain">>,
peerhost => {127, 0, 0, 1},
listener => 'tcp:default',
protocol => mqtt
protocol => mqtt,
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>
}).
-define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)).
@ -517,7 +519,9 @@ samples() ->
<<"username">> := <<"plain">>,
<<"password">> := <<"plain">>,
<<"clientid">> := <<"clienta">>,
<<"peerhost">> := <<"127.0.0.1">>
<<"peerhost">> := <<"127.0.0.1">>,
<<"cert_subject">> := <<"cert_subject_data">>,
<<"cert_common_name">> := <<"cert_common_name_data">>
} = jiffy:decode(RawBody, [return_maps]),
Req = cowboy_req:reply(
200,
@ -534,7 +538,9 @@ samples() ->
<<"clientid">> => ?PH_CLIENTID,
<<"username">> => ?PH_USERNAME,
<<"password">> => ?PH_PASSWORD,
<<"peerhost">> => ?PH_PEERHOST
<<"peerhost">> => ?PH_PEERHOST,
<<"cert_subject">> => ?PH_CERT_SUBJECT,
<<"cert_common_name">> => ?PH_CERT_CN_NAME
}
},
result => {ok, #{is_superuser => false, user_property => #{}}}

27
apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
View File

@ -345,6 +345,33 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
#{
data => #{
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>,
password_hash =>
<<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>,
salt => <<"salt">>,
is_superuser => 1
},
credentials => #{
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>,
password => <<"sha256">>
},
config_params => #{
<<"filter">> => #{
<<"cert_subject">> => <<"${cert_subject}">>,
<<"cert_common_name">> => <<"${cert_common_name}">>
},
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"prefix">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => <<"bcrypt">>,

46
apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
View File

@ -318,6 +318,36 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => "sha256",
password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>,
salt => "salt",
is_superuser_int => 1
},
credentials => #{
clientid => <<"sha256">>,
password => <<"sha256">>,
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>
},
config_params => #{
<<"query">> =>
<<
"SELECT password_hash, salt, is_superuser_int as is_superuser\n"
" FROM users where cert_subject = ${cert_subject} AND \n"
" cert_common_name = ${cert_common_name} LIMIT 1"
>>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"prefix">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => <<"bcrypt">>,
@ -433,14 +463,24 @@ init_seeds() ->
" username VARCHAR(255),\n"
" password_hash VARCHAR(255),\n"
" salt VARCHAR(255),\n"
" cert_subject VARCHAR(255),\n"
" cert_common_name VARCHAR(255),\n"
" is_superuser_str VARCHAR(255),\n"
" is_superuser_int TINYINT)"
),
Fields = [username, password_hash, salt, is_superuser_str, is_superuser_int],
Fields = [
username,
password_hash,
salt,
cert_subject,
cert_common_name,
is_superuser_str,
is_superuser_int
],
InsertQuery =
"INSERT INTO users(username, password_hash, salt, "
" is_superuser_str, is_superuser_int) VALUES(?, ?, ?, ?, ?)",
"INSERT INTO users(username, password_hash, salt, cert_subject, cert_common_name,"
" is_superuser_str, is_superuser_int) VALUES(?, ?, ?, ?, ?, ?, ?)",
lists:foreach(
fun(#{data := Values}) ->

47
apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
View File

@ -380,6 +380,36 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => "sha256",
password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>,
salt => "salt",
is_superuser_int => 1
},
credentials => #{
clientid => <<"sha256">>,
password => <<"sha256">>,
cert_subject => <<"cert_subject_data">>,
cert_common_name => <<"cert_common_name_data">>
},
config_params => #{
<<"query">> =>
<<
"SELECT password_hash, salt, is_superuser_int as is_superuser\n"
" FROM users where cert_subject = ${cert_subject} AND \n"
" cert_common_name = ${cert_common_name} LIMIT 1"
>>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"prefix">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => <<"bcrypt">>,
@ -474,6 +504,8 @@ init_seeds() ->
" username varchar(255),\n"
" password_hash varchar(255),\n"
" salt varchar(255),\n"
" cert_subject varchar(255),\n"
" cert_common_name varchar(255),\n"
" is_superuser_str varchar(255),\n"
" is_superuser_int smallint,\n"
" is_superuser_bool boolean)"
@ -487,12 +519,21 @@ init_seeds() ->
).
create_user(Values) ->
Fields = [username, password_hash, salt, is_superuser_str, is_superuser_int, is_superuser_bool],
Fields = [
username,
password_hash,
salt,
cert_subject,
cert_common_name,
is_superuser_str,
is_superuser_int,
is_superuser_bool
],
InsertQuery =
"INSERT INTO users(username, password_hash, salt,"
"INSERT INTO users(username, password_hash, salt, cert_subject, cert_common_name, "
"is_superuser_str, is_superuser_int, is_superuser_bool) "
"VALUES($1, $2, $3, $4, $5, $6)",
"VALUES($1, $2, $3, $4, $5, $6, $7, $8)",
Params = [maps:get(F, Values, null) || F <- Fields],
{ok, 1} = q(InsertQuery, Params),

46
apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
View File

@ -475,6 +475,52 @@ user_seeds() ->
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
password_hash =>
<<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>,
is_superuser => <<"1">>
},
credentials => #{
clientid => <<"sha256_no_salt">>,
cn => <<"cert_common_name">>,
dn => <<"cert_subject_name">>,
password => <<"sha256_no_salt">>
},
key => <<"mqtt_user:cert_common_name">>,
config_params => #{
<<"cmd">> => <<"HMGET mqtt_user:${cert_common_name} password_hash is_superuser">>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"disable">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
password_hash =>
<<"a3c7f6b085c3e5897ffb9b86f18a9d905063f8550a74444b5892e193c1b50428">>,
is_superuser => <<"1">>
},
credentials => #{
clientid => <<"sha256_no_salt">>,
cn => <<"cert_common_name">>,
dn => <<"cert_subject_name">>,
password => <<"sha256_no_salt">>
},
key => <<"mqtt_user:cert_subject_name">>,
config_params => #{
<<"cmd">> => <<"HMGET mqtt_user:${cert_subject} password_hash is_superuser">>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"disable">>
}
},
result => {ok, #{is_superuser => true}}
}
].

9
apps/emqx_authz/src/emqx_authz_api_sources.erl
View File

@ -50,6 +50,8 @@
aggregate_metrics/1
]).
-define(TAGS, [<<"Authorization">>]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@ -70,6 +72,7 @@ schema("/authorization/sources") ->
get =>
#{
description => ?DESC(authorization_sources_get),
tags => ?TAGS,
responses =>
#{
200 => mk(
@ -81,6 +84,7 @@ schema("/authorization/sources") ->
post =>
#{
description => ?DESC(authorization_sources_post),
tags => ?TAGS,
'requestBody' => mk(
hoconsc:union(authz_sources_type_refs()),
#{desc => ?DESC(source_config)}
@ -101,6 +105,7 @@ schema("/authorization/sources/:type") ->
get =>
#{
description => ?DESC(authorization_sources_type_get),
tags => ?TAGS,
parameters => parameters_field(),
responses =>
#{
@ -114,6 +119,7 @@ schema("/authorization/sources/:type") ->
put =>
#{
description => ?DESC(authorization_sources_type_put),
tags => ?TAGS,
parameters => parameters_field(),
'requestBody' => mk(hoconsc:union(authz_sources_type_refs())),
responses =>
@ -125,6 +131,7 @@ schema("/authorization/sources/:type") ->
delete =>
#{
description => ?DESC(authorization_sources_type_delete),
tags => ?TAGS,
parameters => parameters_field(),
responses =>
#{
@ -139,6 +146,7 @@ schema("/authorization/sources/:type/status") ->
get =>
#{
description => ?DESC(authorization_sources_type_status_get),
tags => ?TAGS,
parameters => parameters_field(),
responses =>
#{
@ -159,6 +167,7 @@ schema("/authorization/sources/:type/move") ->
post =>
#{
description => ?DESC(authorization_sources_type_move_post),
tags => ?TAGS,
parameters => parameters_field(),
'requestBody' =>
emqx_dashboard_swagger:schema_with_examples(

2
apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auto_subscribe, [
{description, "An OTP application"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, []},
{mod, {emqx_auto_subscribe_app, []}},
{applications, [

2
apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
View File

@ -44,12 +44,14 @@ schema("/mqtt/auto_subscribe") ->
'operationId' => auto_subscribe,
get => #{
description => ?DESC(list_auto_subscribe_api),
tags => [<<"Auto subscribe">>],
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe")
}
},
put => #{
description => ?DESC(update_auto_subscribe_api),
tags => [<<"Auto subscribe">>],
'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),

2
apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
View File

@ -51,6 +51,7 @@ schema("/error_codes") ->
get => #{
security => [],
description => <<"API Error Codes">>,
tags => [<<"Error codes">>],
responses => #{
200 => hoconsc:array(hoconsc:ref(?MODULE, error_code))
}
@ -62,6 +63,7 @@ schema("/error_codes/:code") ->
get => #{
security => [],
description => <<"API Error Codes">>,
tags => [<<"Error codes">>],
parameters => [
{code,
hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{

8
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -37,7 +37,7 @@ schema("/monitor") ->
#{
'operationId' => monitor,
get => #{
tags => [dashboard],
tags => [<<"Metrics">>],
desc => <<"List monitor data.">>,
parameters => [parameter_latest()],
responses => #{
@ -50,7 +50,7 @@ schema("/monitor/nodes/:node") ->
#{
'operationId' => monitor,
get => #{
tags => [dashboard],
tags => [<<"Metrics">>],
desc => <<"List the monitor data on the node.">>,
parameters => [parameter_node(), parameter_latest()],
responses => #{
@ -63,7 +63,7 @@ schema("/monitor_current") ->
#{
'operationId' => monitor_current,
get => #{
tags => [dashboard],
tags => [<<"Metrics">>],
desc => <<"Current status. Gauge and rate.">>,
responses => #{
200 => hoconsc:mk(hoconsc:ref(sampler_current), #{})
@ -74,7 +74,7 @@ schema("/monitor_current/nodes/:node") ->
#{
'operationId' => monitor_current,
get => #{
tags => [dashboard],
tags => [<<"Metrics">>],
desc => <<"Node current status. Gauge and rate.">>,
parameters => [parameter_node()],
responses => #{

11
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -338,10 +338,17 @@ to_spec(Meta, Params, RequestBody, Responses) ->
maps:put('requestBody', RequestBody, Spec).
generate_method_desc(Spec = #{desc := _Desc}) ->
trans_description(maps:remove(desc, Spec), Spec);
Spec1 = trans_description(maps:remove(desc, Spec), Spec),
trans_tags(Spec1);
generate_method_desc(Spec = #{description := _Desc}) ->
trans_description(Spec, Spec);
Spec1 = trans_description(Spec, Spec),
trans_tags(Spec1);
generate_method_desc(Spec) ->
trans_tags(Spec).
trans_tags(Spec = #{tags := Tags}) ->
Spec#{tags => [string:titlecase(to_bin(Tag)) || Tag <- Tags]};
trans_tags(Spec) ->
Spec.
parameters(Params, Module) ->

2
apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl
View File

@ -196,7 +196,7 @@ t_in_mix(_Config) ->
}
],
ExpectMeta = #{
tags => [tags, good],
tags => [<<"Tags">>, <<"Good">>],
description => <<"good description">>,
summary => <<"good summary">>,
security => [],

2
apps/emqx_gateway/src/coap/emqx_coap_api.erl
View File

@ -48,7 +48,7 @@ schema(?PREFIX ++ "/request") ->
#{
operationId => request,
post => #{
tags => [<<"gateway|coap">>],
tags => [<<"CoAP gateway">>],
desc => ?DESC(send_coap_request),
parameters => request_parameters(),
requestBody => request_body(),

4
apps/emqx_management/src/emqx_mgmt_api_alarms.erl
View File

@ -26,6 +26,8 @@
-export([alarms/2]).
-define(TAGS, [<<"Alarms">>]).
%% internal export (for query)
-export([query/4]).
@ -40,6 +42,7 @@ schema("/alarms") ->
'operationId' => alarms,
get => #{
description => ?DESC(list_alarms_api),
tags => ?TAGS,
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit),
@ -59,6 +62,7 @@ schema("/alarms") ->
},
delete => #{
description => ?DESC(delete_alarms_api),
tags => ?TAGS,
responses => #{
204 => ?DESC(delete_alarms_api_response204)
}

6
apps/emqx_management/src/emqx_mgmt_api_app.erl
View File

@ -22,6 +22,7 @@
-export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
-export([api_key/2, api_key_by_name/2]).
-export([validate_name/1]).
-define(TAGS, [<<"API keys">>]).
namespace() -> "api_key".
@ -36,12 +37,14 @@ schema("/api_key") ->
'operationId' => api_key,
get => #{
description => "Return api_key list",
tags => ?TAGS,
responses => #{
200 => delete([api_secret], fields(app))
}
},
post => #{
description => "Create new api_key",
tags => ?TAGS,
'requestBody' => delete([created_at, api_key, api_secret], fields(app)),
responses => #{
200 => hoconsc:ref(app),
@ -54,6 +57,7 @@ schema("/api_key/:name") ->
'operationId' => api_key_by_name,
get => #{
description => "Return the specific api_key",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
200 => delete([api_secret], fields(app)),
@ -62,6 +66,7 @@ schema("/api_key/:name") ->
},
put => #{
description => "Update the specific api_key",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
'requestBody' => delete([created_at, api_key, api_secret, name], fields(app)),
responses => #{
@ -71,6 +76,7 @@ schema("/api_key/:name") ->
},
delete => #{
description => "Delete the specific api_key",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
204 => <<"Delete successfully">>,

4
apps/emqx_management/src/emqx_mgmt_api_banned.erl
View File

@ -39,6 +39,7 @@
]).
-define(TAB, emqx_banned).
-define(TAGS, [<<"Banned">>]).
-define(BANNED_TYPES, [clientid, username, peerhost]).
@ -55,6 +56,7 @@ schema("/banned") ->
'operationId' => banned,
get => #{
description => ?DESC(list_banned_api),
tags => ?TAGS,
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit)
@ -68,6 +70,7 @@ schema("/banned") ->
},
post => #{
description => ?DESC(create_banned_api),
tags => ?TAGS,
'requestBody' => hoconsc:mk(hoconsc:ref(ban)),
responses => #{
200 => [{data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})}],
@ -83,6 +86,7 @@ schema("/banned/:as/:who") ->
'operationId' => delete_banned,
delete => #{
description => ?DESC(delete_banned_api),
tags => ?TAGS,
parameters => [
{as,
hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{

12
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -54,6 +54,7 @@
-export([do_subscribe/3]).
-define(CLIENT_QTAB, emqx_channel_info).
-define(TAGS, [<<"Clients">>]).
-define(CLIENT_QSCHEMA, [
{<<"node">>, atom},
@ -100,6 +101,7 @@ schema("/clients") ->
'operationId' => clients,
get => #{
description => <<"List clients">>,
tags => ?TAGS,
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit),
@ -220,6 +222,7 @@ schema("/clients/:clientid") ->
'operationId' => client,
get => #{
description => <<"Get clients info by client ID">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}),
@ -230,6 +233,7 @@ schema("/clients/:clientid") ->
},
delete => #{
description => <<"Kick out client by client ID">>,
tags => ?TAGS,
parameters => [
{clientid, hoconsc:mk(binary(), #{in => path})}
],
@ -246,6 +250,7 @@ schema("/clients/:clientid/authorization/cache") ->
'operationId' => authz_cache,
get => #{
description => <<"Get client authz cache in the cluster.">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, authz_cache), #{}),
@ -256,6 +261,7 @@ schema("/clients/:clientid/authorization/cache") ->
},
delete => #{
description => <<"Clean client authz cache in the cluster.">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
204 => <<"Kick out client successfully">>,
@ -270,6 +276,7 @@ schema("/clients/:clientid/subscriptions") ->
'operationId' => subscriptions,
get => #{
description => <<"Get client subscriptions">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
200 => hoconsc:mk(
@ -286,6 +293,7 @@ schema("/clients/:clientid/subscribe") ->
'operationId' => subscribe,
post => #{
description => <<"Subscribe">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, subscribe)),
responses => #{
@ -301,6 +309,7 @@ schema("/clients/:clientid/subscribe/bulk") ->
'operationId' => subscribe_batch,
post => #{
description => <<"Subscribe">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscribe))),
responses => #{
@ -316,6 +325,7 @@ schema("/clients/:clientid/unsubscribe") ->
'operationId' => unsubscribe,
post => #{
description => <<"Unsubscribe">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, unsubscribe)),
responses => #{
@ -331,6 +341,7 @@ schema("/clients/:clientid/unsubscribe/bulk") ->
'operationId' => unsubscribe_batch,
post => #{
description => <<"Unsubscribe">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, unsubscribe))),
responses => #{
@ -346,6 +357,7 @@ schema("/clients/:clientid/keepalive") ->
'operationId' => set_keepalive,
put => #{
description => <<"Set the online client keepalive by seconds">>,
tags => ?TAGS,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, keepalive)),
responses => #{

3
apps/emqx_management/src/emqx_mgmt_api_cluster.erl
View File

@ -40,6 +40,7 @@ schema("/cluster") ->
'operationId' => cluster_info,
get => #{
description => "Get cluster info",
tags => [<<"Cluster">>],
responses => #{
200 => [
{name, ?HOCON(string(), #{desc => "Cluster name"})},
@ -54,6 +55,7 @@ schema("/cluster/:node/invite") ->
'operationId' => invite_node,
put => #{
description => "Invite node to cluster",
tags => [<<"Cluster">>],
parameters => [hoconsc:ref(node)],
responses => #{
200 => <<"ok">>,
@ -66,6 +68,7 @@ schema("/cluster/:node/force_leave") ->
'operationId' => force_leave,
delete => #{
description => "Force leave node from cluster",
tags => [<<"Cluster">>],
parameters => [hoconsc:ref(node)],
responses => #{
204 => <<"Delete successfully">>,

3
apps/emqx_management/src/emqx_mgmt_api_metrics.erl
View File

@ -22,7 +22,7 @@
-import(hoconsc, [mk/2, ref/2]).
%% minirest/dashbaord_swagger behaviour callbacks
%% minirest/dashboard_swagger behaviour callbacks
-export([
api_spec/0,
paths/0,
@ -74,6 +74,7 @@ schema("/metrics") ->
get =>
#{
description => <<"EMQX metrics">>,
tags => [<<"Metrics">>],
parameters =>
[
{aggregate,

4
apps/emqx_management/src/emqx_mgmt_api_nodes.erl
View File

@ -64,6 +64,7 @@ schema("/nodes") ->
get =>
#{
description => <<"List EMQX nodes">>,
tags => [<<"Nodes">>],
responses =>
#{
200 => mk(
@ -79,6 +80,7 @@ schema("/nodes/:node") ->
get =>
#{
description => <<"Get node info">>,
tags => [<<"Nodes">>],
parameters => [ref(node_name)],
responses =>
#{
@ -96,6 +98,7 @@ schema("/nodes/:node/metrics") ->
get =>
#{
description => <<"Get node metrics">>,
tags => [<<"Nodes">>],
parameters => [ref(node_name)],
responses =>
#{
@ -113,6 +116,7 @@ schema("/nodes/:node/stats") ->
get =>
#{
description => <<"Get node stats">>,
tags => [<<"Nodes">>],
parameters => [ref(node_name)],
responses =>
#{

7
apps/emqx_management/src/emqx_mgmt_api_plugins.erl
View File

@ -48,6 +48,7 @@
]).
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$").
-define(TAGS, [<<"Plugins">>]).
namespace() -> "plugins".
@ -72,6 +73,7 @@ schema("/plugins") ->
"List all install plugins.</br>"
"Plugins are launched in top-down order.</br>"
"Using `POST /plugins/{name}/move` to change the boot order.",
tags => ?TAGS,
responses => #{
200 => hoconsc:array(hoconsc:ref(plugin))
}
@ -85,6 +87,7 @@ schema("/plugins/install") ->
"Install a plugin(plugin-vsn.tar.gz)."
"Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) "
"to develop plugin.",
tags => ?TAGS,
'requestBody' => #{
content => #{
'multipart/form-data' => #{
@ -111,6 +114,7 @@ schema("/plugins/:name") ->
'operationId' => plugin,
get => #{
description => "Describe a plugin according `release.json` and `README.md`.",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
200 => hoconsc:ref(plugin),
@ -119,6 +123,7 @@ schema("/plugins/:name") ->
},
delete => #{
description => "Uninstall a plugin package.",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
204 => <<"Uninstall successfully">>,
@ -134,6 +139,7 @@ schema("/plugins/:name/:action") ->
"start/stop a installed plugin.</br>"
"- **start**: start the plugin.</br>"
"- **stop**: stop the plugin.</br>",
tags => ?TAGS,
parameters => [
hoconsc:ref(name),
{action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})}
@ -149,6 +155,7 @@ schema("/plugins/:name/move") ->
'operationId' => update_boot_order,
post => #{
description => "Setting the boot order of plugins.",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
'requestBody' => move_request_body(),
responses => #{200 => <<"OK">>}

2
apps/emqx_management/src/emqx_mgmt_api_publish.erl
View File

@ -43,6 +43,7 @@ schema("/publish") ->
'operationId' => publish,
post => #{
description => <<"Publish Message">>,
tags => [<<"Publish">>],
'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)),
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, publish_message_info))
@ -54,6 +55,7 @@ schema("/publish/bulk") ->
'operationId' => publish_batch,
post => #{
description => <<"Publish Messages">>,
tags => [<<"Publish">>],
'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}),
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message_info)), #{})

2
apps/emqx_management/src/emqx_mgmt_api_stats.erl
View File

@ -50,7 +50,7 @@ schema("/stats") ->
get =>
#{
description => <<"EMQX stats">>,
tags => [<<"stats">>],
tags => [<<"Metrics">>],
parameters => [ref(aggregate)],
responses =>
#{

1
apps/emqx_management/src/emqx_mgmt_api_status.erl
View File

@ -37,6 +37,7 @@ schema("/status") ->
get =>
#{
description => <<"Node running status">>,
tags => [<<"Status">>],
security => [],
responses =>
#{

1
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -60,6 +60,7 @@ schema("/subscriptions") ->
'operationId' => subscriptions,
get => #{
description => <<"List subscriptions">>,
tags => [<<"Subscriptions">>],
parameters => parameters(),
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{})

2
apps/emqx_management/src/emqx_mgmt_api_sys.erl
View File

@ -31,7 +31,7 @@
-export([sys/2]).
-define(TAGS, [<<"sys">>]).
-define(TAGS, [<<"System topics">>]).
namespace() -> "sys".

3
apps/emqx_management/src/emqx_mgmt_api_topics.erl
View File

@ -39,6 +39,7 @@
-define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND').
-define(TOPICS_QUERY_SCHEMA, [{<<"topic">>, binary}, {<<"node">>, atom}]).
-define(TAGS, [<<"Topics">>]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
@ -51,6 +52,7 @@ schema("/topics") ->
'operationId' => topics,
get => #{
description => <<"Topics list">>,
tags => ?TAGS,
parameters => [
topic_param(query),
node_param(),
@ -70,6 +72,7 @@ schema("/topics/:topic") ->
'operationId' => topic,
get => #{
description => <<"Lookup topic info by name">>,
tags => ?TAGS,
parameters => [topic_param(path)],
responses => #{
200 => hoconsc:mk(hoconsc:ref(topic), #{}),

8
apps/emqx_management/src/emqx_mgmt_api_trace.erl
View File

@ -47,6 +47,7 @@
-define(TO_BIN(_B_), iolist_to_binary(_B_)).
-define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}).
-define(TAGS, [<<"Trace">>]).
namespace() -> "trace".
@ -61,12 +62,14 @@ schema("/trace") ->
'operationId' => trace,
get => #{
description => "List all trace",
tags => ?TAGS,
responses => #{
200 => hoconsc:ref(trace)
}
},
post => #{
description => "Create new trace",
tags => ?TAGS,
'requestBody' => delete([status, log_size], fields(trace)),
responses => #{
200 => hoconsc:ref(trace),
@ -82,6 +85,7 @@ schema("/trace") ->
},
delete => #{
description => "Clear all traces",
tags => ?TAGS,
responses => #{
204 => <<"No Content">>
}
@ -92,6 +96,7 @@ schema("/trace/:name") ->
'operationId' => delete_trace,
delete => #{
description => "Delete trace by name",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
204 => <<"Delete successfully">>,
@ -104,6 +109,7 @@ schema("/trace/:name/stop") ->
'operationId' => update_trace,
put => #{
description => "Stop trace by name",
tags => ?TAGS,
parameters => [hoconsc:ref(name)],
responses => #{
200 => hoconsc:ref(trace),
@ -116,6 +122,7 @@ schema("/trace/:name/download") ->
'operationId' => download_trace_log,
get => #{
description => "Download trace log by name",
tags => ?TAGS,
parameters => [hoconsc:ref(name), hoconsc:ref(node)],
responses => #{
200 =>
@ -134,6 +141,7 @@ schema("/trace/:name/log") ->
'operationId' => stream_log_file,
get => #{
description => "view trace log",
tags => ?TAGS,
parameters => [
hoconsc:ref(name),
hoconsc:ref(bytes),

14
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -566,23 +566,23 @@ trace_type(_, _) -> error.
listeners([]) ->
lists:foreach(
fun({ID, Conf}) ->
{Host, Port} = maps:get(bind, Conf),
Bind = maps:get(bind, Conf),
Acceptors = maps:get(acceptors, Conf),
ProxyProtocol = maps:get(proxy_protocol, Conf, undefined),
Running = maps:get(running, Conf),
CurrentConns =
case emqx_listeners:current_conns(ID, {Host, Port}) of
case emqx_listeners:current_conns(ID, Bind) of
{error, _} -> [];
CC -> [{current_conn, CC}]
end,
MaxConn =
case emqx_listeners:max_conns(ID, {Host, Port}) of
case emqx_listeners:max_conns(ID, Bind) of
{error, _} -> [];
MC -> [{max_conns, MC}]
end,
Info =
[
{listen_on, {string, format_listen_on(Port)}},
{listen_on, {string, format_listen_on(Bind)}},
{acceptors, Acceptors},
{proxy_protocol, ProxyProtocol},
{running, Running}
@ -806,8 +806,10 @@ format_listen_on(Port) when is_integer(Port) ->
io_lib:format("0.0.0.0:~w", [Port]);
format_listen_on({Addr, Port}) when is_list(Addr) ->
io_lib:format("~ts:~w", [Addr, Port]);
format_listen_on({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]).
format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 4 ->
io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]);
format_listen_on({Addr, Port}) when is_tuple(Addr) andalso tuple_size(Addr) == 8 ->
io_lib:format("[~ts]:~w", [inet:ntoa(Addr), Port]).
name(Filter) ->
iolist_to_binary(["CLI-", Filter]).

3
apps/emqx_modules/src/emqx_telemetry_api.erl
View File

@ -52,12 +52,14 @@ schema("/telemetry/status") ->
get =>
#{
description => ?DESC(get_telemetry_status_api),
tags => [<<"Telemetry">>],
responses =>
#{200 => status_schema(?DESC(get_telemetry_status_api))}
},
put =>
#{
description => ?DESC(update_telemetry_status_api),
tags => [<<"Telemetry">>],
'requestBody' => status_schema(?DESC(update_telemetry_status_api)),
responses =>
#{
@ -71,6 +73,7 @@ schema("/telemetry/data") ->
get =>
#{
description => ?DESC(get_telemetry_data_api),
tags => [<<"Telemetry">>],
responses =>
#{200 => mk(ref(?MODULE, telemetry), #{desc => ?DESC(get_telemetry_data_api)})}
}

2
apps/emqx_prometheus/rebar.config
View File

@ -4,7 +4,7 @@
{emqx, {path, "../emqx"}},
%% FIXME: tag this as v3.1.3
{prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}
]}.
{edoc_opts, [{preprocess, true}]}.

2
apps/emqx_prometheus/src/emqx_prometheus.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_prometheus, [
{description, "Prometheus for EMQX"},
% strict semver, bump manually!
{vsn, "5.0.1"},
{vsn, "5.0.2"},
{modules, []},
{registered, [emqx_prometheus_sup]},
{applications, [kernel, stdlib, prometheus, emqx]},

4
apps/emqx_prometheus/src/emqx_prometheus_api.erl
View File

@ -34,6 +34,7 @@
]).
-define(SCHEMA_MODULE, emqx_prometheus_schema).
-define(TAGS, [<<"Monitor">>]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@ -50,12 +51,14 @@ schema("/prometheus") ->
get =>
#{
description => <<"Get Prometheus config info">>,
tags => ?TAGS,
responses =>
#{200 => prometheus_config_schema()}
},
put =>
#{
description => <<"Update Prometheus config">>,
tags => ?TAGS,
'requestBody' => prometheus_config_schema(),
responses =>
#{200 => prometheus_config_schema()}
@ -67,6 +70,7 @@ schema("/prometheus/stats") ->
get =>
#{
description => <<"Get Prometheus Data">>,
tags => ?TAGS,
security => [],
responses =>
#{200 => prometheus_data_schema()}

40
apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf
View File

@ -10,6 +10,46 @@ emqx_rule_engine_api {
zh: "列出所有规则"
}
}
api1_enable {
desc {
en: "Filter enable/disable rules"
zh: "根据规则是否开启条件过滤"
}
}
api1_from {
desc {
en: "Filter rules by from(topic), exact match"
zh: "根据规则来源 Topic 过滤, 需要完全匹配"
}
}
api1_like_id {
desc {
en: "Filter rules by id, Substring matching"
zh: "根据规则 id 过滤, 使用子串模糊匹配"
}
}
api1_like_from {
desc {
en: "Filter rules by from(topic), Substring matching"
zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配"
}
}
api1_like_description {
desc {
en: "Filter rules by description, Substring matching"
zh: "根据规则描述过滤, 使用子串模糊匹配"
}
}
api1_match_from {
desc {
en: "Filter rules by from(topic), Mqtt topic matching"
zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配"
}
}
api2 {
desc {

104
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -33,6 +33,9 @@
%% API callbacks
-export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
%% query callback
-export([query/4]).
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
-define(ERR_BADARGS(REASON), begin
R0 = err_msg(REASON),
@ -109,6 +112,15 @@ end).
}
).
-define(RULE_QS_SCHEMA, [
{<<"enable">>, atom},
{<<"from">>, binary},
{<<"like_id">>, binary},
{<<"like_from">>, binary},
{<<"match_from">>, binary},
{<<"like_description">>, binary}
]).
namespace() -> "rule".
api_spec() ->
@ -134,9 +146,31 @@ schema("/rules") ->
get => #{
tags => [<<"rules">>],
description => ?DESC("api1"),
parameters => [
{enable,
mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})},
{from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})},
{like_id,
mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})},
{like_from,
mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})},
{like_description,
mk(binary(), #{
desc => ?DESC("api1_like_description"), in => query, required => false
})},
{match_from,
mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})},
ref(emqx_dashboard_swagger, page),
ref(emqx_dashboard_swagger, limit)
],
summary => <<"List Rules">>,
responses => #{
200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})
200 =>
[
{data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})},
{meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
],
400 => error_schema('BAD_REQUEST', "Invalid Parameters")
}
},
post => #{
@ -236,9 +270,21 @@ param_path_id() ->
'/rule_events'(get, _Params) ->
{200, emqx_rule_events:event_info()}.
'/rules'(get, _Params) ->
Records = emqx_rule_engine:get_rules_ordered_by_ts(),
{200, format_rule_resp(Records)};
'/rules'(get, #{query_string := QueryString}) ->
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?RULE_TAB,
?RULE_QS_SCHEMA,
{?MODULE, query}
)
of
{error, page_limit_invalid} ->
{400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
Result ->
{200, Result}
end;
'/rules'(post, #{body := Params0}) ->
case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of
<<>> ->
@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
format_rule_resp(Rules) when is_list(Rules) ->
[format_rule_resp(R) || R <- Rules];
format_rule_resp({Id, Rule}) ->
format_rule_resp(Rule#{id => Id});
format_rule_resp(#{
id := Id,
name := Name,
@ -503,3 +551,51 @@ filter_out_request_body(Conf) ->
<<"node">>
],
maps:without(ExtraConfs, Conf).
query(Tab, {Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(),
FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
emqx_mgmt_api:select_table_with_count(
Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
).
%% rule is not a record, so everything is fuzzy filter.
qs2ms() ->
[{'_', [], ['$_']}].
fuzzy_match_fun(Qs, Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
fun(Rows) ->
Ls = ets:match_spec_run(Rows, MsC),
lists:filter(
fun(E) ->
run_qs_match(E, Qs) andalso
run_fuzzy_match(E, Fuzzy)
end,
Ls
)
end.
run_qs_match(_, []) ->
true;
run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
Enable =:= Pattern andalso run_qs_match(E, Qs);
run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
lists:member(Pattern, From) andalso run_qs_match(E, Qs);
run_qs_match(E, [_ | Qs]) ->
run_qs_match(E, Qs).
run_fuzzy_match(_, []) ->
true;
run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) ->
lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso
run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E, [_ | Fuzzy]) ->
run_fuzzy_match(E, Fuzzy).

77
apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
View File

@ -45,7 +45,7 @@ t_crud_rule_api(_Config) ->
),
?assertEqual(RuleID, maps:get(id, Rule)),
{200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}),
{200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
ct:pal("RList : ~p", [Rules]),
?assert(length(Rules) > 0),
@ -91,6 +91,81 @@ t_crud_rule_api(_Config) ->
),
ok.
t_list_rule_api(_Config) ->
AddIds =
lists:map(
fun(Seq0) ->
Seq = integer_to_binary(Seq0),
Params = #{
<<"description">> => <<"A simple rule">>,
<<"enable">> => true,
<<"actions">> => [#{<<"function">> => <<"console">>}],
<<"sql">> => <<"SELECT * from \"t/1\"">>,
<<"name">> => <<"test_rule", Seq/binary>>
},
{201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}),
Id
end,
lists:seq(1, 20)
),
{200, #{data := Rules, meta := #{count := Count}}} =
emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
?assertEqual(20, length(AddIds)),
?assertEqual(20, length(Rules)),
?assertEqual(20, Count),
[RuleID | _] = AddIds,
UpdateParams = #{
<<"description">> => <<"中文的描述也能搜索"/utf8>>,
<<"enable">> => false,
<<"actions">> => [#{<<"function">> => <<"console">>}],
<<"sql">> => <<"SELECT * from \"t/1/+\"">>,
<<"name">> => <<"test_rule_update1">>
},
{200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
bindings => #{id => RuleID},
body => UpdateParams
}),
QueryStr1 = #{query_string => #{<<"enable">> => false}},
{200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1),
?assertEqual(1, Count1),
QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
{200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
?assertEqual(Result1, Result2),
QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
{200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
?assertEqual(19, Count3),
QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
{200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
?assertEqual(Result1, Result4),
QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
{200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
?assertEqual(Result1, Result5),
QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
{200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
?assertEqual(Result1, Result6),
%% clean up
lists:foreach(
fun(Id) ->
?assertMatch(
{204},
emqx_rule_engine_api:'/rules/:id'(
delete,
#{bindings => #{id => Id}}
)
)
end,
AddIds
),
ok.
test_rule_params() ->
#{
body => #{

2
apps/emqx_slow_subs/src/emqx_slow_subs.app.src
View File

@ -1,7 +1,7 @@
{application, emqx_slow_subs, [
{description, "EMQX Slow Subscribers Statistics"},
% strict semver, bump manually!
{vsn, "1.0.0"},
{vsn, "1.0.1"},
{modules, []},
{registered, [emqx_slow_subs_sup]},
{applications, [kernel, stdlib, emqx]},

9
apps/emqx_slow_subs/src/emqx_slow_subs_api.erl
View File

@ -25,6 +25,7 @@
-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
-export([slow_subs/2, get_history/0, settings/2]).
-define(TAGS, [<<"Slow subscriptions">>]).
-import(hoconsc, [mk/2, ref/1, ref/2]).
-import(emqx_mgmt_util, [bad_request/0]).
@ -44,14 +45,14 @@ schema(("/slow_subscriptions")) ->
#{
'operationId' => slow_subs,
delete => #{
tags => [<<"slow subs">>],
tags => ?TAGS,
description => ?DESC(clear_records_api),
parameters => [],
'requestBody' => [],
responses => #{204 => <<"No Content">>}
},
get => #{
tags => [<<"slow subs">>],
tags => ?TAGS,
description => ?DESC(get_records_api),
parameters => [
ref(emqx_dashboard_swagger, page),
@ -65,12 +66,12 @@ schema("/slow_subscriptions/settings") ->
#{
'operationId' => settings,
get => #{
tags => [<<"slow subs">>],
tags => ?TAGS,
description => ?DESC(get_setting_api),
responses => #{200 => conf_schema()}
},
put => #{
tags => [<<"slow subs">>],
tags => ?TAGS,
description => ?DESC(update_setting_api),
'requestBody' => conf_schema(),
responses => #{200 => conf_schema()}

2
apps/emqx_statsd/src/emqx_statsd.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_statsd, [
{description, "An OTP application"},
{vsn, "5.0.0"},
{vsn, "5.0.1"},
{registered, []},
{mod, {emqx_statsd_app, []}},
{applications, [

2
apps/emqx_statsd/src/emqx_statsd_api.erl
View File

@ -32,7 +32,7 @@
schema/1
]).
-define(API_TAG_STATSD, [<<"statsd">>]).
-define(API_TAG_STATSD, [<<"Monitor">>]).
-define(SCHEMA_MODULE, emqx_statsd_schema).
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').

16
bin/emqx
View File

@ -169,6 +169,9 @@ usage() {
echo " --no-permanent Install release package VERSION but"
echo " don't make it permanent"
;;
check_config)
echo "Checks the EMQX config without generating any files"
;;
*)
echo "Usage: $REL_NAME COMMAND [help]"
echo ''
@ -184,6 +187,7 @@ usage() {
echo " Up/Down-grade: upgrade | downgrade | install | uninstall"
echo " Install info: ertspath | root_dir"
echo " Runtime info: pid | ping | versions"
echo " Validate Config: check_config"
echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-erl"
echo ''
echo "Execute '$REL_NAME COMMAND help' for more information"
@ -211,7 +215,7 @@ fi
## IS_BOOT_COMMAND is set for later to inspect node name and cookie from hocon config (or env variable)
case "${COMMAND}" in
start|console|console_clean|foreground)
start|console|console_clean|foreground|check_config)
IS_BOOT_COMMAND='yes'
;;
ertspath)
@ -525,6 +529,12 @@ relx_start_command() {
"$START_OPTION"
}
# Function to check configs without generating them
check_config() {
## this command checks the configs without generating any files
call_hocon -v -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf check_schema
}
# Function to generate app.config and vm.args
# sets two environment variables CONF_FILE and ARGS_FILE
generate_config() {
@ -1030,6 +1040,10 @@ case "${COMMAND}" in
shift
relx_nodetool "eval" "$@"
;;
check_config)
check_config
;;
*)
usage "$COMMAND"
exit 1

2
deploy/charts/emqx/Chart.yaml
View File

@ -14,7 +14,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 4.4.1
version: 5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.

14
deploy/charts/emqx/values.yaml
View File

@ -91,13 +91,13 @@ initContainers: {}
## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
emqxConfig:
EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
# EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
# EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
# EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
# EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
# EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
# EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
# EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
## The address type is used to extract host from k8s service.
## Value: ip | dns | hostname
## Note：Hostname is only supported after v4.0-rc.2

29
deploy/docker/docker-entrypoint.sh
View File

@ -16,22 +16,31 @@ shopt -s nullglob
LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1)
if [[ -z "$EMQX_NODE_NAME" ]]; then
EMQX_NAME="${EMQX_NAME:-emqx}"
if [[ -z "$EMQX_HOST" ]]; then
if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
export EMQX_NAME="${EMQX_NAME:-emqx}"
if [[ -z "$EMQX_HOST" ]]; then
if [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "dns" ]] && \
[[ "$EMQX_CLUSTER__DNS__RECORD_TYPE" == "srv" ]] && \
grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
[[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
[[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
else
EMQX_HOST="$LOCAL_IP"
fi
else
EMQX_HOST="$LOCAL_IP"
fi
export EMQX_HOST
fi
if [[ -z "$EMQX_NODE_NAME" ]]; then
export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST"
unset EMQX_NAME
unset EMQX_HOST
fi
# The default rpc port discovery 'stateless' is mostly for clusters

2
mix.exs
View File

@ -66,7 +66,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.28.3", override: true},
{:hocon, github: "emqx/hocon", tag: "0.29.0", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

2
rebar.config
View File

@ -67,7 +67,7 @@
, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

4
scripts/relup-build/download-base-packages.sh
View File

@ -14,7 +14,7 @@ export PROFILE
case $PROFILE in
"emqx-enterprise")
DIR='enterprise'
DIR='emqx-ee'
EDITION='enterprise'
;;
"emqx")
@ -51,7 +51,7 @@ mkdir -p _upgrade_base
pushd _upgrade_base >/dev/null
for tag in ${BASE_VERSIONS}; do
filename="$PROFILE-$(fullvsn "${tag#[e|v]}").tar.gz"
url="https://www.emqx.com/downloads/$DIR/$tag/$filename"
url="https://packages.emqx.io/$DIR/$tag/$filename"
echo "downloading ${filename} ..."
## if the file does not exist (not downloaded yet)
## and there is such a package to downlaod