diff --git a/.gitattributes b/.gitattributes
index 4ed73da9a..434addf38 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,5 +1,6 @@
* text=auto
*.* text eol=lf
+*.cmd text eol=crlf
*.jpg -text
*.png -text
*.pdf -text
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 322d38206..5ef58f411 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -140,7 +140,6 @@ jobs:
path: source/_packages/${{ matrix.profile }}/.
mac:
- runs-on: macos-10.15
needs: prepare
@@ -148,11 +147,16 @@ jobs:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+ macos:
+ - macos-11
+ - macos-10.15
otp:
- 24.0.5-emqx-1
exclude:
- profile: emqx-edge
+ runs-on: ${{ matrix.macos }}
+
steps:
- uses: actions/download-artifact@v2
with:
@@ -170,16 +174,12 @@ jobs:
id: cache
with:
path: ~/.kerl
- key: erl${{ matrix.otp }}-macos10.15
+ key: otp-${{ matrix.otp }}-${{ matrix.macos }}
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60
- env:
- KERL_BUILD_BACKEND: git
- OTP_GITHUB_URL: https://github.com/emqx/otp
run: |
- kerl update releases
- kerl build ${{ matrix.otp }}
+ kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build
working-directory: source
@@ -191,8 +191,8 @@ jobs:
- name: test
working-directory: source
run: |
- pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
- unzip -q _packages/${{ matrix.profile }}/$pkg_name
+ pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip | head)
+ unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
@@ -211,7 +211,7 @@ jobs:
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
- openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
+ openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
with:
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 2fb447d26..293bcb82b 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -13,6 +13,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
+ fail-fast: false
matrix:
otp:
- 24.0.5-emqx-1
@@ -53,13 +54,18 @@ jobs:
path: _packages/**/*.zip
mac:
- runs-on: macos-10.15
strategy:
+ fail-fast: false
matrix:
+ macos:
+ - macos-11
+ - macos-10.15
otp:
- 24.0.5-emqx-1
+ runs-on: ${{ matrix.macos }}
+
steps:
- uses: actions/checkout@v1
- name: prepare
@@ -82,16 +88,12 @@ jobs:
id: cache
with:
path: ~/.kerl
- key: erl${{ matrix.otp }}-macos10.15
+ key: otp-${{ matrix.otp }}-${{ matrix.macos }}
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60
- env:
- KERL_BUILD_BACKEND: git
- OTP_GITHUB_URL: https://github.com/emqx/otp
run: |
- kerl update releases
- kerl build ${{ matrix.otp }}
+ kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build
run: |
@@ -106,8 +108,7 @@ jobs:
path: ./rebar3.crashdump
- name: test
run: |
- pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
- unzip -q _packages/${EMQX_NAME}/$pkg_name
+ unzip -q $(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip | head)
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml
index af9be07e0..6b977900c 100644
--- a/.github/workflows/run_api_tests.yaml
+++ b/.github/workflows/run_api_tests.yaml
@@ -45,6 +45,13 @@ jobs:
- api_login
- api_banned
- api_alarms
+ - api_nodes
+ - api_topic_metrics
+ - api_retainer
+ - api_auto_subscribe
+ - api_delayed_publish
+ - api_topic_rewrite
+ - api_event_message
steps:
- uses: actions/checkout@v2
with:
@@ -74,7 +81,7 @@ jobs:
cd /tmp && tar -xvf apache-jmeter.tgz
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
- wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+ wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run ${{ matrix.script_name }}
run: |
diff --git a/.gitignore b/.gitignore
index 57be83882..4188126da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,3 +50,5 @@ _upgrade_base/
TAGS
erlang_ls.config
.els_cache/
+.vs/
+.vscode/
diff --git a/Makefile b/Makefile
index 5ba708fde..24e64bd7d 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DESC ?= EMQ X
-export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.14
+export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.15
ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none
endif
diff --git a/README.md b/README.md
index dac48c2f2..bf3d93b64 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [рус
*EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
-Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
+Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scale to 10+ million concurrent MQTT connections on one cluster.
- For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases).
- For more information, please visit [EMQ X homepage](https://www.emqx.io/).
diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf
index 267f9a7ec..df5ae9034 100644
--- a/apps/emqx/etc/emqx.conf
+++ b/apps/emqx/etc/emqx.conf
@@ -198,7 +198,7 @@ listeners.ssl.default {
ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
- ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+ # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
# TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
# TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
# PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
@@ -1350,7 +1350,7 @@ example_common_ssl_options {
## Default: true
ssl.honor_cipher_order = true
- ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+ # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
# TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
# TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
# PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index cf86338f9..4ec7c7dc5 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -13,7 +13,7 @@
, {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}}
- , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
+ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.5"}}}
diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl
index e47bb489e..db2376784 100644
--- a/apps/emqx/src/emqx_config_handler.erl
+++ b/apps/emqx/src/emqx_config_handler.erl
@@ -77,7 +77,7 @@ stop() ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config(SchemaModule, ConfKeyPath, UpdateArgs) ->
?ATOM_CONF_PATH(ConfKeyPath, gen_server:call(?MODULE, {change_config, SchemaModule,
- AtomKeyPath, UpdateArgs}), {error, ConfKeyPath}).
+ AtomKeyPath, UpdateArgs}), {error, {not_found, ConfKeyPath}}).
-spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok.
add_handler(ConfKeyPath, HandlerName) ->
diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl
index 088bb4085..f056b754e 100644
--- a/apps/emqx/src/emqx_hooks.erl
+++ b/apps/emqx/src/emqx_hooks.erl
@@ -67,7 +67,7 @@
%% - The execution order is the adding order of callbacks if they have
%% equal priority values.
--type(hookpoint() :: atom()).
+-type(hookpoint() :: atom() | binary()).
-type(action() :: {module(), atom(), [term()] | undefined}).
-type(filter() :: {module(), atom(), [term()] | undefined}).
@@ -158,12 +158,12 @@ del(HookPoint, Action) ->
gen_server:cast(?SERVER, {del, HookPoint, Action}).
%% @doc Run hooks.
--spec(run(atom(), list(Arg::term())) -> ok).
+-spec(run(hookpoint(), list(Arg::term())) -> ok).
run(HookPoint, Args) ->
do_run(lookup(HookPoint), Args).
%% @doc Run hooks with Accumulator.
--spec(run_fold(atom(), list(Arg::term()), Acc::term()) -> Acc::term()).
+-spec(run_fold(hookpoint(), list(Arg::term()), Acc::term()) -> Acc::term()).
run_fold(HookPoint, Args, Acc) ->
do_run_fold(lookup(HookPoint), Args, Acc).
diff --git a/apps/emqx/src/emqx_rule_actions_trans.erl b/apps/emqx/src/emqx_rule_actions_trans.erl
deleted file mode 100644
index df1e58797..000000000
--- a/apps/emqx/src/emqx_rule_actions_trans.erl
+++ /dev/null
@@ -1,66 +0,0 @@
--module(emqx_rule_actions_trans).
-
--include_lib("syntax_tools/include/merl.hrl").
-
--export([parse_transform/2]).
-
-parse_transform(Forms, _Options) ->
- trans(Forms, []).
-
-trans([], ResAST) ->
- lists:reverse(ResAST);
-trans([{eof, L} | AST], ResAST) ->
- lists:reverse([{eof, L} | ResAST]) ++ AST;
-trans([{function, LineNo, FuncName, Arity, Clauses} | AST], ResAST) ->
- NewClauses = trans_func_clauses(atom_to_list(FuncName), Clauses),
- trans(AST, [{function, LineNo, FuncName, Arity, NewClauses} | ResAST]);
-trans([Form | AST], ResAST) ->
- trans(AST, [Form | ResAST]).
-
-trans_func_clauses("on_action_create_" ++ _ = _FuncName , Clauses) ->
- NewClauses = [
- begin
- Bindings = lists:flatten(get_vars(Args) ++ get_vars(Body, lefth)),
- Body2 = append_to_result(Bindings, Body),
- {clause, LineNo, Args, Guards, Body2}
- end || {clause, LineNo, Args, Guards, Body} <- Clauses],
- NewClauses;
-trans_func_clauses(_FuncName, Clauses) ->
- Clauses.
-
-get_vars(Exprs) ->
- get_vars(Exprs, all).
-get_vars(Exprs, Type) ->
- do_get_vars(Exprs, [], Type).
-
-do_get_vars([], Vars, _Type) -> Vars;
-do_get_vars([Line | Expr], Vars, all) ->
- do_get_vars(Expr, [syntax_vars(erl_syntax:form_list([Line])) | Vars], all);
-do_get_vars([Line | Expr], Vars, lefth) ->
- do_get_vars(Expr,
- case (Line) of
- ?Q("_@LeftV = _@@_") -> Vars ++ syntax_vars(LeftV);
- _ -> Vars
- end, lefth).
-
-syntax_vars(Line) ->
- sets:to_list(erl_syntax_lib:variables(Line)).
-
-%% append bindings to the return value as the first tuple element.
-%% e.g. if the original result is R, then the new result will be {[binding()], R}.
-append_to_result(Bindings, Exprs) ->
- erl_syntax:revert_forms(do_append_to_result(to_keyword(Bindings), Exprs, [])).
-
-do_append_to_result(KeyWordVars, [Line], Res) ->
- case Line of
- ?Q("_@LeftV = _@RightV") ->
- lists:reverse([?Q("{[_@KeyWordVars], _@LeftV}"), Line | Res]);
- _ ->
- lists:reverse([?Q("{[_@KeyWordVars], _@Line}") | Res])
- end;
-do_append_to_result(KeyWordVars, [Line | Exprs], Res) ->
- do_append_to_result(KeyWordVars, Exprs, [Line | Res]).
-
-to_keyword(Vars) ->
- [erl_syntax:tuple([erl_syntax:atom(Var), merl:var(Var)])
- || Var <- Vars].
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 27688a868..202cd4ae9 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -55,7 +55,7 @@
% workaround: prevent being recognized as unused functions
-export([to_duration/1, to_duration_s/1, to_duration_ms/1,
- to_bytesize/1, to_wordsize/1,
+ mk_duration/2, to_bytesize/1, to_wordsize/1,
to_percent/1, to_comma_separated_list/1,
to_bar_separated_list/1, to_ip_port/1,
to_erl_cipher_suite/1,
@@ -159,11 +159,11 @@ fields("stats") ->
fields("authorization") ->
[ {"no_match",
- sc(hoconsc:union([allow, deny]),
+ sc(hoconsc:enum([allow, deny]),
#{ default => allow
})}
, {"deny_action",
- sc(hoconsc:union([ignore, disconnect]),
+ sc(hoconsc:enum([ignore, disconnect]),
#{ default => ignore
})}
, {"cache",
@@ -297,7 +297,7 @@ fields("mqtt") ->
})
}
, {"mqueue_default_priority",
- sc(union(highest, lowest),
+ sc(hoconsc:enum([highest, lowest]),
#{ default => lowest
})
}
@@ -312,11 +312,11 @@ fields("mqtt") ->
})
}
, {"peer_cert_as_username",
- sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]),
+ sc(hoconsc:enum([disabled, cn, dn, crt, pem, md5]),
#{ default => disabled
})}
, {"peer_cert_as_clientid",
- sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]),
+ sc(hoconsc:enum([disabled, cn, dn, crt, pem, md5]),
#{ default => disabled
})}
];
@@ -525,7 +525,7 @@ fields("ws_opts") ->
})
}
, {"mqtt_piggyback",
- sc(hoconsc:union([single, multiple]),
+ sc(hoconsc:enum([single, multiple]),
#{ default => multiple
})
}
@@ -637,23 +637,23 @@ fields("listener_ssl_opts") ->
server_ssl_opts_schema(
#{ depth => 10
, reuse_sessions => true
- , versions => tcp
- , ciphers => tcp_all
+ , versions => tls_all_available
+ , ciphers => tls_all_available
}, false);
fields("listener_wss_opts") ->
server_ssl_opts_schema(
#{ depth => 10
, reuse_sessions => true
- , versions => tcp
- , ciphers => tcp_all
+ , versions => tls_all_available
+ , ciphers => tls_all_available
}, true);
fields(ssl_client_opts) ->
client_ssl_opts_schema(#{});
fields("deflate_opts") ->
[ {"level",
- sc(hoconsc:union([none, default, best_compression, best_speed]),
+ sc(hoconsc:enum([none, default, best_compression, best_speed]),
#{})
}
, {"mem_level",
@@ -662,15 +662,15 @@ fields("deflate_opts") ->
})
}
, {"strategy",
- sc(hoconsc:union([default, filtered, huffman_only, rle]),
+ sc(hoconsc:enum([default, filtered, huffman_only, rle]),
#{})
}
, {"server_context_takeover",
- sc(hoconsc:union([takeover, no_takeover]),
+ sc(hoconsc:enum([takeover, no_takeover]),
#{})
}
, {"client_context_takeover",
- sc(hoconsc:union([takeover, no_takeover]),
+ sc(hoconsc:enum([takeover, no_takeover]),
#{})
}
, {"server_max_window_bits",
@@ -709,12 +709,12 @@ fields("broker") ->
})
}
, {"session_locking_strategy",
- sc(hoconsc:union([local, leader, quorum, all]),
+ sc(hoconsc:enum([local, leader, quorum, all]),
#{ default => quorum
})
}
, {"shared_subscription_strategy",
- sc(hoconsc:union([random, round_robin]),
+ sc(hoconsc:enum([random, round_robin]),
#{ default => round_robin
})
}
@@ -736,7 +736,7 @@ fields("broker") ->
fields("broker_perf") ->
[ {"route_lock_type",
- sc(hoconsc:union([key, tab, global]),
+ sc(hoconsc:enum([key, tab, global]),
#{ default => key
})}
, {"trie_compaction",
@@ -962,7 +962,7 @@ the file if it is to be added.
})
}
, {"verify",
- sc(hoconsc:union([verify_peer, verify_none]),
+ sc(hoconsc:enum([verify_peer, verify_none]),
#{ default => Df("verify", verify_none)
})
}
@@ -987,13 +987,14 @@ keyfile is password-protected."""
}
, {"versions",
sc(hoconsc:array(typerefl:atom()),
- #{ default => default_tls_vsns(maps:get(versions, Defaults, tcp))
+ #{ default => default_tls_vsns(maps:get(versions, Defaults, tls_all_available))
, desc =>
"""All TLS/DTLS versions to be supported.
NOTE: PSK ciphers are suppresed by 'tlsv1.3' version config
In case PSK cipher suites are intended, make sure to configured
['tlsv1.2', 'tlsv1.1'] here.
"""
+ , validator => fun validate_tls_versions/1
})
}
, {"ciphers", ciphers_schema(D("ciphers"))}
@@ -1086,7 +1087,7 @@ client_ssl_opts_schema(Defaults) ->
, desc =>
"""Specify the host name to be used in TLS Server Name Indication extension.
For instance, when connecting to \"server.example.net\", the genuine server
-which accedpts the connection and performs TSL handshake may differ from the
+which accedpts the connection and performs TLS handshake may differ from the
host the TLS client initially connects to, e.g. when connecting to an IP address
or when the host has multiple resolvable DNS records
If not specified, it will default to the host name string which is used
@@ -1099,12 +1100,12 @@ verification check."""
].
-default_tls_vsns(dtls) ->
- [<<"dtlsv1.2">>, <<"dtlsv1">>];
-default_tls_vsns(tcp) ->
- [<<"tlsv1.3">>, <<"tlsv1.2">>, <<"tlsv1.1">>, <<"tlsv1">>].
+default_tls_vsns(dtls_all_available) ->
+ proplists:get_value(available_dtls, ssl:versions());
+default_tls_vsns(tls_all_available) ->
+ emqx_tls_lib:default_versions().
--spec ciphers_schema(quic | dtls | tcp_all | undefined) -> hocon_schema:field_schema().
+-spec ciphers_schema(quic | dtls_all_available | tls_all_available | undefined) -> hocon_schema:field_schema().
ciphers_schema(Default) ->
sc(hoconsc:array(string()),
#{ default => default_ciphers(Default)
@@ -1113,7 +1114,10 @@ ciphers_schema(Default) ->
(Ciphers) when is_list(Ciphers) ->
Ciphers
end
- , validator => fun validate_ciphers/1
+ , validator => case Default =:= quic of
+ true -> undefined; %% quic has openssl statically linked
+ false -> fun validate_ciphers/1
+ end
, desc =>
"""TLS cipher suite names separated by comma, or as an array of strings
\"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or
@@ -1146,52 +1150,24 @@ RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA\"
end}).
default_ciphers(undefined) ->
- default_ciphers(tcp_all);
+ default_ciphers(tls_all_available);
default_ciphers(quic) -> [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256"
];
-default_ciphers(tcp_all) ->
- default_ciphers('tlsv1.3') ++
- default_ciphers('tlsv1.2') ++
- default_ciphers(psk);
-default_ciphers(dtls) ->
+default_ciphers(dtls_all_available) ->
%% as of now, dtls does not support tlsv1.3 ciphers
- default_ciphers('tlsv1.2') ++ default_ciphers('psk');
-default_ciphers('tlsv1.3') ->
- ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
- "TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
- "TLS_AES_128_CCM_8_SHA256"]
- ++ default_ciphers('tlsv1.2');
-default_ciphers('tlsv1.2') -> [
- "ECDHE-ECDSA-AES256-GCM-SHA384",
- "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384",
- "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384",
- "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384",
- "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256",
- "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256",
- "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256",
- "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256",
- "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256",
- "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA",
- "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA",
- "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
- "ECDH-RSA-AES128-SHA", "AES128-SHA"
- ];
-default_ciphers(psk) ->
- [ "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384",
- "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256",
- "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA",
- "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA"
- ].
+ emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']);
+default_ciphers(tls_all_available) ->
+ emqx_tls_lib:default_ciphers().
%% @private return a list of keys in a parent field
-spec(keys(string(), hocon:config()) -> [string()]).
keys(Parent, Conf) ->
[binary_to_list(B) || B <- maps:keys(conf_get(Parent, Conf, #{}))].
--spec ceiling(float()) -> integer().
+-spec ceiling(number()) -> integer().
ceiling(X) ->
T = erlang:trunc(X),
case (X - T) of
@@ -1210,6 +1186,15 @@ ref(Field) -> hoconsc:ref(?MODULE, Field).
ref(Module, Field) -> hoconsc:ref(Module, Field).
+mk_duration(Desc, OverrideMeta) ->
+ DefaultMeta = #{desc => Desc ++ " Time span. A text string with number followed by time units:
+ `ms` for milli-seconds,
+ `s` for seconds,
+ `m` for minutes,
+ `h` for hours;
+ or combined representation like `1h5m0s`"},
+ hoconsc:mk(typerefl:alias("string", duration()), maps:merge(DefaultMeta, OverrideMeta)).
+
to_duration(Str) ->
case hocon_postprocess:duration(Str) of
I when is_integer(I) -> {ok, I};
@@ -1218,13 +1203,15 @@ to_duration(Str) ->
to_duration_s(Str) ->
case hocon_postprocess:duration(Str) of
- I when is_integer(I) -> {ok, ceiling(I / 1000)};
+ I when is_number(I) -> {ok, ceiling(I / 1000)};
_ -> {error, Str}
end.
+-spec to_duration_ms(Input) -> {ok, integer()} | {error, Input}
+ when Input :: string() | binary().
to_duration_ms(Str) ->
case hocon_postprocess:duration(Str) of
- I when is_integer(I) -> {ok, ceiling(I)};
+ I when is_number(I) -> {ok, ceiling(I)};
_ -> {error, Str}
end.
@@ -1303,9 +1290,22 @@ parse_user_lookup_fun(StrConf) ->
{fun Mod:Fun/3, <<>>}.
validate_ciphers(Ciphers) ->
- All = ssl:cipher_suites(all, 'tlsv1.3', openssl) ++
- ssl:cipher_suites(all, 'tlsv1.2', openssl), %% includes older version ciphers
+ All = case is_tlsv13_available() of
+ true -> ssl:cipher_suites(all, 'tlsv1.3', openssl);
+ false -> []
+ end ++ ssl:cipher_suites(all, 'tlsv1.2', openssl),
case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of
[] -> ok;
Bad -> {error, {bad_ciphers, Bad}}
end.
+
+validate_tls_versions(Versions) ->
+ AvailableVersions = proplists:get_value(available, ssl:versions()) ++
+ proplists:get_value(available_dtls, ssl:versions()),
+ case lists:filter(fun(V) -> not lists:member(V, AvailableVersions) end, Versions) of
+ [] -> ok;
+ Vs -> {error, {unsupported_ssl_versions, Vs}}
+ end.
+
+is_tlsv13_available() ->
+ lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())).
diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl
index 24a9a15cf..11145f684 100644
--- a/apps/emqx/src/emqx_tls_lib.erl
+++ b/apps/emqx/src/emqx_tls_lib.erl
@@ -19,7 +19,7 @@
-export([ default_versions/0
, integral_versions/1
, default_ciphers/0
- , default_ciphers/1
+ , selected_ciphers/1
, integral_ciphers/2
, drop_tls13_for_old_otp/1
]).
@@ -31,9 +31,7 @@
%% @doc Returns the default supported tls versions.
-spec default_versions() -> [atom()].
-default_versions() ->
- OtpRelease = list_to_integer(erlang:system_info(otp_release)),
- integral_versions(default_versions(OtpRelease)).
+default_versions() -> available_versions().
%% @doc Validate a given list of desired tls versions.
%% raise an error exception if non of them are available.
@@ -51,7 +49,7 @@ integral_versions(Desired) when ?IS_STRING(Desired) ->
integral_versions(Desired) when is_binary(Desired) ->
integral_versions(parse_versions(Desired));
integral_versions(Desired) ->
- {_, Available} = lists:keyfind(available, 1, ssl:versions()),
+ Available = available_versions(),
case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
[] -> erlang:error(#{ reason => no_available_tls_version
, desired => Desired
@@ -61,27 +59,61 @@ integral_versions(Desired) ->
Filtered
end.
-%% @doc Return a list of default (openssl string format) cipher suites.
--spec default_ciphers() -> [string()].
-default_ciphers() -> default_ciphers(default_versions()).
-
%% @doc Return a list of (openssl string format) cipher suites.
--spec default_ciphers([ssl:tls_version()]) -> [string()].
-default_ciphers(['tlsv1.3']) ->
+-spec all_ciphers([ssl:tls_version()]) -> [string()].
+all_ciphers(['tlsv1.3']) ->
%% When it's only tlsv1.3 wanted, use 'exclusive' here
%% because 'all' returns legacy cipher suites too,
%% which does not make sense since tlsv1.3 can not use
%% legacy cipher suites.
ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
-default_ciphers(Versions) ->
+all_ciphers(Versions) ->
%% assert non-empty
[_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])).
+
+%% @doc All Pre-selected TLS ciphers.
+default_ciphers() ->
+ selected_ciphers(available_versions()).
+
+%% @doc Pre-selected TLS ciphers for given versions..
+selected_ciphers(Vsns) ->
+ All = all_ciphers(Vsns),
+ dedup(lists:filter(fun(Cipher) -> lists:member(Cipher, All) end,
+ lists:flatmap(fun do_selected_ciphers/1, Vsns))).
+
+do_selected_ciphers('tlsv1.3') ->
+ case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
+ true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
+ false -> []
+ end ++ do_selected_ciphers('tlsv1.2');
+do_selected_ciphers(_) ->
+ [ "ECDHE-ECDSA-AES256-GCM-SHA384",
+ "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384",
+ "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384",
+ "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384",
+ "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256",
+ "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256",
+ "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256",
+ "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256",
+ "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256",
+ "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA",
+ "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA",
+ "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
+ "ECDH-RSA-AES128-SHA", "AES128-SHA",
+
+ %% psk
+ "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384",
+ "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256",
+ "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA",
+ "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA"
+ ].
+
%% @doc Ensure version & cipher-suites integrity.
-spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined ->
%% not configured
- integral_ciphers(Versions, default_ciphers(Versions));
+ integral_ciphers(Versions, selected_ciphers(Versions));
integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) ->
%% ensure tlsv1.3 ciphers if none of them is found in Ciphers
dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers));
@@ -95,7 +127,7 @@ integral_ciphers(Versions, Ciphers) ->
%% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user
%% did not provide it from config --- which is a common mistake
ensure_tls13_cipher(true, Ciphers) ->
- Tls13Ciphers = default_ciphers(['tlsv1.3']),
+ Tls13Ciphers = selected_ciphers(['tlsv1.3']),
case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of
true -> Ciphers;
false -> Tls13Ciphers ++ Ciphers
@@ -103,11 +135,17 @@ ensure_tls13_cipher(true, Ciphers) ->
ensure_tls13_cipher(false, Ciphers) ->
Ciphers.
+%% default ssl versions based on available versions.
+-spec available_versions() -> [atom()].
+available_versions() ->
+ OtpRelease = list_to_integer(erlang:system_info(otp_release)),
+ default_versions(OtpRelease).
+
%% tlsv1.3 is available from OTP-22 but we do not want to use until 23.
default_versions(OtpRelease) when OtpRelease >= 23 ->
- ['tlsv1.3' | default_versions(22)];
+ proplists:get_value(available, ssl:versions());
default_versions(_) ->
- ['tlsv1.2', 'tlsv1.1', tlsv1].
+ lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())).
%% Deduplicate a list without re-ordering the elements.
dedup([]) -> [];
@@ -175,10 +213,12 @@ drop_tls13(SslOpts0) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+all_ciphers() -> all_ciphers(default_versions()).
+
drop_tls13_test() ->
Versions = default_versions(),
?assert(lists:member('tlsv1.3', Versions)),
- Ciphers = default_ciphers(),
+ Ciphers = all_ciphers(),
?assert(has_tlsv13_cipher(Ciphers)),
Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
Opts = drop_tls13(Opts0),
diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl
index 5fd2e47af..e4684649d 100644
--- a/apps/emqx/test/emqx_authentication_SUITE.erl
+++ b/apps/emqx/test/emqx_authentication_SUITE.erl
@@ -236,6 +236,9 @@ t_update_config(Config) when is_list(Config) ->
?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})),
?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)),
+ ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID2})),
+ ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)),
+
ListenerID = 'tcp:default',
ConfKeyPath = [listeners, tcp, default, authentication],
?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig1})),
diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl
index 3fb0c0130..e2825498d 100644
--- a/apps/emqx/test/emqx_schema_tests.erl
+++ b/apps/emqx/test/emqx_schema_tests.erl
@@ -19,8 +19,8 @@
-include_lib("eunit/include/eunit.hrl").
ssl_opts_dtls_test() ->
- Sc = emqx_schema:server_ssl_opts_schema(#{versions => dtls,
- ciphers => dtls}, false),
+ Sc = emqx_schema:server_ssl_opts_schema(#{versions => dtls_all_available,
+ ciphers => dtls_all_available}, false),
Checked = validate(Sc, #{<<"versions">> => [<<"dtlsv1.2">>, <<"dtlsv1">>]}),
?assertMatch(#{versions := ['dtlsv1.2', 'dtlsv1'],
ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _]
@@ -62,19 +62,14 @@ ssl_opts_cipher_comma_separated_string_test() ->
ssl_opts_tls_psk_test() ->
Sc = emqx_schema:server_ssl_opts_schema(#{}, false),
Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>]}),
- ?assertMatch(#{versions := ['tlsv1.2']}, Checked),
- #{ciphers := Ciphers} = Checked,
- PskCiphers = emqx_schema:default_ciphers(psk),
- lists:foreach(fun(Cipher) ->
- ?assert(lists:member(Cipher, Ciphers))
- end, PskCiphers).
+ ?assertMatch(#{versions := ['tlsv1.2']}, Checked).
bad_cipher_test() ->
Sc = emqx_schema:server_ssl_opts_schema(#{}, false),
Reason = {bad_ciphers, ["foo"]},
?assertThrow({_Sc, [{validation_error, #{reason := Reason}}]},
- [validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>],
- <<"ciphers">> => [<<"foo">>]})]),
+ validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>],
+ <<"ciphers">> => [<<"foo">>]})),
ok.
validate(Schema, Data0) ->
@@ -95,3 +90,10 @@ ciperhs_schema_test() ->
WSc = #{roots => [{ciphers, Sc}]},
?assertThrow({_, [{validation_error, _}]},
hocon_schema:check_plain(WSc, #{<<"ciphers">> => <<"foo,bar">>})).
+
+bad_tls_version_test() ->
+ Sc = emqx_schema:server_ssl_opts_schema(#{}, false),
+ Reason = {unsupported_ssl_versions, [foo]},
+ ?assertThrow({_Sc, [{validation_error, #{reason := Reason}}]},
+ validate(Sc, #{<<"versions">> => [<<"foo">>]})),
+ ok.
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index d115131a5..827e08dab 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -91,7 +91,7 @@
enable => true})).
-define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http-server">>,
- connect_timeout => 5000,
+ connect_timeout => "5s",
enable_pipelining => true,
headers => #{
<<"accept">> => <<"application/json">>,
@@ -102,8 +102,8 @@
},
max_retries => 5,
pool_size => 8,
- request_timeout => 5000,
- retry_interval => 1000,
+ request_timeout => "5s",
+ retry_interval => "1s",
enable => true})).
-define(INSTANCE_EXAMPLE_3, maps:merge(?EXAMPLE_3, #{id => <<"jwt">>,
@@ -1259,9 +1259,9 @@ definitions() ->
example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">>
},
query_timeout => #{
- type => integer,
- description => <<"Query timeout, Unit: Milliseconds">>,
- default => 5000
+ type => string,
+ description => <<"Query timeout">>,
+ default => "5s"
}
}
},
@@ -1528,16 +1528,16 @@ definitions() ->
type => object
},
connect_timeout => #{
- type => integer,
- default => 5000
+ type => string,
+ default => <<"5s">>
},
max_retries => #{
type => integer,
default => 5
},
retry_interval => #{
- type => integer,
- default => 1000
+ type => string,
+ default => <<"1s">>
},
request_timout => #{
type => integer,
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
index 536cd544a..22f701220 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
@@ -100,8 +100,8 @@ body(type) -> map();
body(validator) -> [fun check_body/1];
body(_) -> undefined.
-request_timeout(type) -> non_neg_integer();
-request_timeout(default) -> 5000;
+request_timeout(type) -> emqx_schema:duration_ms();
+request_timeout(default) -> "5s";
request_timeout(_) -> undefined.
%%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
index d15dff6e1..43a9bd252 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
@@ -65,8 +65,8 @@ salt_position(_) -> undefined.
query(type) -> string();
query(_) -> undefined.
-query_timeout(type) -> integer();
-query_timeout(default) -> 5000;
+query_timeout(type) -> emqx_schema:duration_ms();
+query_timeout(default) -> "5s";
query_timeout(_) -> undefined.
%%------------------------------------------------------------------------------
diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf
index 2948f2af7..a1cfd41d3 100644
--- a/apps/emqx_authz/etc/acl.conf
+++ b/apps/emqx_authz/etc/acl.conf
@@ -3,9 +3,9 @@
%%
%% -type(ipaddrs() :: {ipaddrs, string()}).
%%
-%% -type(username() :: {username, regex()}).
+%% -type(username() :: {user | username, string()} | {user | username, {re, regex()}}).
%%
-%% -type(clientid() :: {clientid, regex()}).
+%% -type(clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}).
%%
%% -type(who() :: ipaddr() | ipaddrs() |username() | clientid() |
%% {'and', [ipaddr() | ipaddrs()| username() | clientid()]} |
@@ -20,7 +20,7 @@
%%
%% -type(permission() :: allow | deny).
%%
-%% -type(rule() :: {permission(), who(), access(), topics()}).
+%% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}).
%%--------------------------------------------------------------------
{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}.
diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf
index 1111a7819..9e65517ad 100644
--- a/apps/emqx_authz/etc/emqx_authz.conf
+++ b/apps/emqx_authz/etc/emqx_authz.conf
@@ -55,8 +55,12 @@ authorization {
# collection: mqtt_authz
# selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] }
# },
+ {
+ type: built-in-database
+ }
{
type: file
+ # file is loaded into cache
path: "{{ platform_etc_dir }}/acl.conf"
}
]
diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl
index c83dfde0d..bf8371add 100644
--- a/apps/emqx_authz/include/emqx_authz.hrl
+++ b/apps/emqx_authz/include/emqx_authz.hrl
@@ -29,12 +29,32 @@
(A =:= all) orelse (A =:= <<"all">>)
)).
+-define(ACL_SHARDED, emqx_acl_sharded).
+
+-define(ACL_TABLE, emqx_acl).
+
+%% To save some space, use an integer for label, 0 for 'all', {1, Username} and {2, ClientId}.
+-define(ACL_TABLE_ALL, 0).
+-define(ACL_TABLE_USERNAME, 1).
+-define(ACL_TABLE_CLIENTID, 2).
+
+-record(emqx_acl, {
+ who :: ?ACL_TABLE_ALL| {?ACL_TABLE_USERNAME, binary()} | {?ACL_TABLE_CLIENTID, binary()},
+ rules :: [ {permission(), action(), emqx_topic:topic()} ]
+ }).
+
-record(authz_metrics, {
allow = 'client.authorize.allow',
deny = 'client.authorize.deny',
ignore = 'client.authorize.ignore'
}).
+-define(CMD_REPLCAE, replace).
+-define(CMD_DELETE, delete).
+-define(CMD_PREPEND, prepend).
+-define(CMD_APPEND, append).
+-define(CMD_MOVE, move).
+
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl
index e7ccbe5b0..7a417925e 100644
--- a/apps/emqx_authz/src/emqx_authz.erl
+++ b/apps/emqx_authz/src/emqx_authz.erl
@@ -39,7 +39,6 @@
-export([post_config_update/4, pre_config_update/2]).
-define(CONF_KEY_PATH, [authorization, sources]).
--define(SOURCE_TYPES, [file, http, mongodb, mysql, postgresql, redis]).
-spec(register_metrics() -> ok).
register_metrics() ->
@@ -50,228 +49,151 @@ init() ->
emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE),
Sources = emqx:get_config(?CONF_KEY_PATH, []),
ok = check_dup_types(Sources),
- NSources = [init_source(Source) || Source <- Sources],
+ NSources = init_sources(Sources),
ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1).
lookup() ->
{_M, _F, [A]}= find_action_in_hooks(),
A.
+
lookup(Type) ->
- try find_source_by_type(atom(Type), lookup()) of
- {_, Source} -> Source
- catch
- error:Reason -> {error, Reason}
- end.
+ {Source, _Front, _Rear} = take(Type),
+ Source.
move(Type, Cmd) ->
move(Type, Cmd, #{}).
move(Type, #{<<"before">> := Before}, Opts) ->
- emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"before">> => atom(Before)}}, Opts);
+ emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), #{<<"before">> => type(Before)}}, Opts);
move(Type, #{<<"after">> := After}, Opts) ->
- emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"after">> => atom(After)}}, Opts);
+ emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), #{<<"after">> => type(After)}}, Opts);
move(Type, Position, Opts) ->
- emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), Position}, Opts).
+ emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts).
update(Cmd, Sources) ->
update(Cmd, Sources, #{}).
-update({replace_once, Type}, Sources, Opts) ->
- emqx:update_config(?CONF_KEY_PATH, {{replace_once, atom(Type)}, Sources}, Opts);
-update({delete_once, Type}, Sources, Opts) ->
- emqx:update_config(?CONF_KEY_PATH, {{delete_once, atom(Type)}, Sources}, Opts);
+update({replace, Type}, Sources, Opts) ->
+ emqx:update_config(?CONF_KEY_PATH, {{replace, type(Type)}, Sources}, Opts);
+update({delete, Type}, Sources, Opts) ->
+ emqx:update_config(?CONF_KEY_PATH, {{delete, type(Type)}, Sources}, Opts);
update(Cmd, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts).
-pre_config_update({move, Type, <<"top">>}, Conf) when is_list(Conf) ->
- {Index, _} = find_source_by_type(Type),
- {List1, List2} = lists:split(Index, Conf),
- NConf = [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2,
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-
-pre_config_update({move, Type, <<"bottom">>}, Conf) when is_list(Conf) ->
- {Index, _} = find_source_by_type(Type),
- {List1, List2} = lists:split(Index, Conf),
- NConf = lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)],
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-
-pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) ->
- {Index1, _} = find_source_by_type(Type),
- Conf1 = lists:nth(Index1, Conf),
- {Index2, _} = find_source_by_type(Before),
- Conf2 = lists:nth(Index2, Conf),
-
- {List1, List2} = lists:split(Index2, Conf),
- NConf = lists:delete(Conf1, lists:droplast(List1))
- ++ [Conf1] ++ [Conf2]
- ++ lists:delete(Conf1, List2),
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-
-pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) ->
- {Index1, _} = find_source_by_type(Type),
- Conf1 = lists:nth(Index1, Conf),
- {Index2, _} = find_source_by_type(After),
-
- {List1, List2} = lists:split(Index2, Conf),
- NConf = lists:delete(Conf1, List1)
- ++ [Conf1]
- ++ lists:delete(Conf1, List2),
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-
-pre_config_update({head, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
+do_update({?CMD_MOVE, Type, <<"top">>}, Conf) when is_list(Conf) ->
+ {Source, Front, Rear} = take(Type, Conf),
+ [Source | Front] ++ Rear;
+do_update({?CMD_MOVE, Type, <<"bottom">>}, Conf) when is_list(Conf) ->
+ {Source, Front, Rear} = take(Type, Conf),
+ Front ++ Rear ++ [Source];
+do_update({?CMD_MOVE, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) ->
+ {S1, Front1, Rear1} = take(Type, Conf),
+ {S2, Front2, Rear2} = take(Before, Front1 ++ Rear1),
+ Front2 ++ [S1, S2] ++ Rear2;
+do_update({?CMD_MOVE, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) ->
+ {S1, Front1, Rear1} = take(Type, Conf),
+ {S2, Front2, Rear2} = take(After, Front1 ++ Rear1),
+ Front2 ++ [S2, S1] ++ Rear2;
+do_update({?CMD_PREPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
NConf = Sources ++ Conf,
- case check_dup_types(NConf) of
- ok -> {ok, Sources ++ Conf};
- Error -> Error
- end;
-pre_config_update({tail, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
+ ok = check_dup_types(NConf),
+ NConf;
+do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
NConf = Conf ++ Sources,
- case check_dup_types(NConf) of
- ok -> {ok, Conf ++ Sources};
- Error -> Error
- end;
-pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_list(Conf) ->
- {Index, _} = find_source_by_type(Type),
- {List1, List2} = lists:split(Index, Conf),
- NConf = lists:droplast(List1) ++ [Source] ++ List2,
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) ->
- {Index, _} = find_source_by_type(Type),
- {List1, List2} = lists:split(Index, Conf),
- NConf = lists:droplast(List1) ++ List2,
- case check_dup_types(NConf) of
- ok -> {ok, NConf};
- Error -> Error
- end;
-pre_config_update({_, Sources}, _Conf) when is_list(Sources)->
+ ok = check_dup_types(NConf),
+ NConf;
+do_update({{replace, Type}, Source}, Conf) when is_map(Source), is_list(Conf) ->
+ {_Old, Front, Rear} = take(Type, Conf),
+ NConf = Front ++ [Source | Rear],
+ ok = check_dup_types(NConf),
+ NConf;
+do_update({{delete, Type}, _Source}, Conf) when is_list(Conf) ->
+ {_Old, Front, Rear} = take(Type, Conf),
+ NConf = Front ++ Rear,
+ NConf;
+do_update({_, Sources}, _Conf) when is_list(Sources)->
%% overwrite the entire config!
- {ok, Sources}.
+ Sources.
+
+pre_config_update(Cmd, Conf) ->
+ {ok, do_update(Cmd, Conf)}.
+
post_config_update(_, undefined, _Conf, _AppEnvs) ->
ok;
-post_config_update({move, Type, <<"top">>}, _NewSources, _OldSources, _AppEnvs) ->
- InitedSources = lookup(),
- {Index, Source} = find_source_by_type(Type, InitedSources),
- {Sources1, Sources2 } = lists:split(Index, InitedSources),
- Sources3 = [Source] ++ lists:droplast(Sources1) ++ Sources2,
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1),
- ok = emqx_authz_cache:drain_cache();
-post_config_update({move, Type, <<"bottom">>}, _NewSources, _OldSources, _AppEnvs) ->
- InitedSources = lookup(),
- {Index, Source} = find_source_by_type(Type, InitedSources),
- {Sources1, Sources2 } = lists:split(Index, InitedSources),
- Sources3 = lists:droplast(Sources1) ++ Sources2 ++ [Source],
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1),
- ok = emqx_authz_cache:drain_cache();
-post_config_update({move, Type, #{<<"before">> := Before}}, _NewSources, _OldSources, _AppEnvs) ->
- InitedSources = lookup(),
- {_, Source0} = find_source_by_type(Type, InitedSources),
- {Index, Source1} = find_source_by_type(Before, InitedSources),
- {Sources1, Sources2} = lists:split(Index, InitedSources),
- Sources3 = lists:delete(Source0, lists:droplast(Sources1))
- ++ [Source0] ++ [Source1]
- ++ lists:delete(Source0, Sources2),
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1),
- ok = emqx_authz_cache:drain_cache();
-
-post_config_update({move, Type, #{<<"after">> := After}}, _NewSources, _OldSources, _AppEnvs) ->
- InitedSources = lookup(),
- {_, Source} = find_source_by_type(Type, InitedSources),
- {Index, _} = find_source_by_type(After, InitedSources),
- {Sources1, Sources2} = lists:split(Index, InitedSources),
- Sources3 = lists:delete(Source, Sources1)
- ++ [Source]
- ++ lists:delete(Source, Sources2),
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1),
- ok = emqx_authz_cache:drain_cache();
-
-post_config_update({head, Sources}, _NewSources, _OldConf, _AppEnvs) ->
- InitedSources = [init_source(R) || R <- check_sources(Sources)],
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources ++ lookup()]}, -1),
- ok = emqx_authz_cache:drain_cache();
-
-post_config_update({tail, Sources}, _NewSources, _OldConf, _AppEnvs) ->
- InitedSources = [init_source(R) || R <- check_sources(Sources)],
- emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1),
- ok = emqx_authz_cache:drain_cache();
-
-post_config_update({{replace_once, Type}, #{type := Type} = Source}, _NewSources, _OldConf, _AppEnvs) when is_map(Source) ->
- OldInitedSources = lookup(),
- {Index, OldSource} = find_source_by_type(Type, OldInitedSources),
- case maps:get(type, OldSource, undefined) of
- undefined -> ok;
- file -> ok;
- _ ->
- #{annotations := #{id := Id}} = OldSource,
- ok = emqx_resource:remove(Id)
- end,
- {OldSources1, OldSources2 } = lists:split(Index, OldInitedSources),
- InitedSources = [init_source(R) || R <- check_sources([Source])],
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:droplast(OldSources1) ++ InitedSources ++ OldSources2]}, -1),
- ok = emqx_authz_cache:drain_cache();
-post_config_update({{delete_once, Type}, _Source}, _NewSources, _OldConf, _AppEnvs) ->
- OldInitedSources = lookup(),
- {_, OldSource} = find_source_by_type(Type, OldInitedSources),
- case OldSource of
- #{annotations := #{id := Id}} ->
- ok = emqx_resource:remove(Id);
- _ -> ok
- end,
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:delete(OldSource, OldInitedSources)]}, -1),
- ok = emqx_authz_cache:drain_cache();
-post_config_update(_, NewSources, _OldConf, _AppEnvs) ->
- %% overwrite the entire config!
- OldInitedSources = lookup(),
- InitedSources = [init_source(Source) || Source <- NewSources],
- ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources]}, -1),
- lists:foreach(fun (#{type := _Type, enable := true, annotations := #{id := Id}}) ->
- ok = emqx_resource:remove(Id);
- (_) -> ok
- end, OldInitedSources),
+post_config_update(Cmd, NewSources, _OldSource, _AppEnvs) ->
+ ok = do_post_update(Cmd, NewSources),
ok = emqx_authz_cache:drain_cache().
-%%--------------------------------------------------------------------
-%% Initialize source
-%%--------------------------------------------------------------------
+do_post_update({?CMD_MOVE, _Type, _Where} = Cmd, _NewSources) ->
+ InitedSources = lookup(),
+ MovedSources = do_update(Cmd, InitedSources),
+ ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [MovedSources]}, -1),
+ ok = emqx_authz_cache:drain_cache();
+do_post_update({?CMD_PREPEND, Sources}, _NewSources) ->
+ InitedSources = init_sources(check_sources(Sources)),
+ ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources ++ lookup()]}, -1),
+ ok = emqx_authz_cache:drain_cache();
+do_post_update({?CMD_APPEND, Sources}, _NewSources) ->
+ InitedSources = init_sources(check_sources(Sources)),
+ emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1),
+ ok = emqx_authz_cache:drain_cache();
+do_post_update({{replace, Type}, #{type := Type} = Source}, _NewSources) when is_map(Source) ->
+ OldInitedSources = lookup(),
+ {OldSource, Front, Rear} = take(Type, OldInitedSources),
+ ok = ensure_resource_deleted(OldSource),
+ InitedSources = init_sources(check_sources([Source])),
+ ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1),
+ ok = emqx_authz_cache:drain_cache();
+do_post_update({{delete, Type}, _Source}, _NewSources) ->
+ OldInitedSources = lookup(),
+ {OldSource, Front, Rear} = take(Type, OldInitedSources),
+ ok = ensure_resource_deleted(OldSource),
+ ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, Front ++ Rear}, -1),
+ ok = emqx_authz_cache:drain_cache();
+do_post_update(_, NewSources) ->
+ %% overwrite the entire config!
+ OldInitedSources = lookup(),
+ InitedSources = init_sources(NewSources),
+ ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources]}, -1),
+ lists:foreach(fun ensure_resource_deleted/1, OldInitedSources),
+ ok = emqx_authz_cache:drain_cache().
+
+ensure_resource_deleted(#{type := file}) -> ok;
+ensure_resource_deleted(#{type := 'built-in-database'}) -> ok;
+ensure_resource_deleted(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove(Id).
check_dup_types(Sources) ->
- check_dup_types(Sources, ?SOURCE_TYPES).
-check_dup_types(_Sources, []) -> ok;
-check_dup_types(Sources, [T0 | Tail]) ->
- case lists:foldl(fun (#{type := T1}, AccIn) ->
- case T0 =:= T1 of
- true -> AccIn + 1;
- false -> AccIn
- end;
- (#{<<"type">> := T1}, AccIn) ->
- case T0 =:= atom(T1) of
- true -> AccIn + 1;
- false -> AccIn
- end
- end, 0, Sources) > 1 of
+ check_dup_types(Sources, []).
+
+check_dup_types([], _Checked) -> ok;
+check_dup_types([Source | Sources], Checked) ->
+ %% the input might be raw or type-checked result, so lookup both 'type' and <<"type">>
+ %% TODO: check: really?
+ Type = case maps:get(<<"type">>, Source, maps:get(type, Source, undefined)) of
+ undefined ->
+ %% this should never happen if the value is type checked by honcon schema
+ error({bad_source_input, Source});
+ Type0 ->
+ type(Type0)
+ end,
+ case lists:member(Type, Checked) of
true ->
- ?LOG(error, "The type is duplicated in the Authorization source"),
- {error, 'The type is duplicated in the Authorization source'};
- false -> check_dup_types(Sources, Tail)
+ %% we have made it clear not to support more than one authz instance for each type
+ error({duplicated_authz_source_type, Type});
+ false ->
+ check_dup_types(Sources, [Type | Checked])
end.
-init_source(#{enable := true,
- type := file,
+init_sources(Sources) ->
+ {Enabled, Disabled} = lists:partition(fun(#{enable := Enable}) -> Enable end, Sources),
+ case Disabled =/= [] of
+ true -> ?SLOG(info, #{msg => "disabled_sources_ignored", sources => Disabled});
+ false -> ok
+ end,
+ lists:map(fun init_source/1, Enabled).
+
+init_source(#{type := file,
path := Path
} = Source) ->
Rules = case file:consult(Path) of
@@ -288,8 +210,7 @@ init_source(#{enable := true,
error(Reason)
end,
Source#{annotations => #{rules => Rules}};
-init_source(#{enable := true,
- type := http,
+init_source(#{type := http,
url := Url
} = Source) ->
NSource= maps:put(base_url, maps:remove(query, Url), Source),
@@ -297,16 +218,17 @@ init_source(#{enable := true,
{error, Reason} -> error({load_config_error, Reason});
Id -> Source#{annotations => #{id => Id}}
end;
-init_source(#{enable := true,
- type := DB
+init_source(#{type := 'built-in-database'
+ } = Source) ->
+ Source;
+init_source(#{type := DB
} = Source) when DB =:= redis;
DB =:= mongodb ->
case create_resource(Source) of
{error, Reason} -> error({load_config_error, Reason});
Id -> Source#{annotations => #{id => Id}}
end;
-init_source(#{enable := true,
- type := DB,
+init_source(#{type := DB,
query := SQL
} = Source) when DB =:= mysql;
DB =:= postgresql ->
@@ -318,8 +240,7 @@ init_source(#{enable := true,
query => Mod:parse_query(SQL)
}
}
- end;
-init_source(#{enable := false} = Source) ->Source.
+ end.
%%--------------------------------------------------------------------
%% AuthZ callbacks
@@ -373,13 +294,17 @@ check_sources(RawSources) ->
#{sources := Sources} = hocon_schema:check_plain(Schema, Conf, #{atom_key => true}),
Sources.
-find_source_by_type(Type) -> find_source_by_type(Type, lookup()).
-find_source_by_type(Type, Sources) -> find_source_by_type(Type, Sources, 1).
-find_source_by_type(_, [], _N) -> error(not_found_source);
-find_source_by_type(Type, [ Source = #{type := T} | Tail], N) ->
- case Type =:= T of
- true -> {N, Source};
- false -> find_source_by_type(Type, Tail, N + 1)
+take(Type) -> take(Type, lookup()).
+
+%% Take the source of give type, the sources list is split into two parts
+%% front part and rear part.
+take(Type, Sources) ->
+ {Front, Rear} = lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources),
+ case Rear =:= [] of
+ true ->
+ error({authz_source_of_type_not_found, Type});
+ _ ->
+ {hd(Rear), Front, tl(Rear)}
end.
find_action_in_hooks() ->
@@ -404,6 +329,8 @@ create_resource(#{type := DB} = Source) ->
{error, Reason} -> {error, Reason}
end.
+authz_module('built-in-database') ->
+ emqx_authz_mnesia;
authz_module(Type) ->
list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)).
@@ -414,9 +341,20 @@ connector_module(postgresql) ->
connector_module(Type) ->
list_to_existing_atom("emqx_connector_" ++ atom_to_list(Type)).
-atom(B) when is_binary(B) ->
- try binary_to_existing_atom(B, utf8)
- catch
- _ -> binary_to_atom(B)
- end;
-atom(A) when is_atom(A) -> A.
+type(#{type := Type}) -> type(Type);
+type(#{<<"type">> := Type}) -> type(Type);
+type(file) -> file;
+type(<<"file">>) -> file;
+type(http) -> http;
+type(<<"http">>) -> http;
+type(mongodb) -> mongodb;
+type(<<"mongodb">>) -> mongodb;
+type(mysql) -> mysql;
+type(<<"mysql">>) -> mysql;
+type(redis) -> redis;
+type(<<"redis">>) -> redis;
+type(postgresql) -> postgresql;
+type(<<"postgresql">>) -> postgresql;
+type('built-in-database') -> 'built-in-database';
+type(<<"built-in-database">>) -> 'built-in-database';
+type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema
diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
new file mode 100644
index 000000000..3cc08c61c
--- /dev/null
+++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
@@ -0,0 +1,590 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_api_mnesia).
+
+-behavior(minirest_api).
+
+-include("emqx_authz.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+
+-define(EXAMPLE_USERNAME, #{username => user1,
+ rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+-define(EXAMPLE_CLIENTID, #{clientid => client1,
+ rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+-define(EXAMPLE_ALL , #{rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+
+-export([ api_spec/0
+ , purge/2
+ , records/2
+ , record/2
+ ]).
+
+api_spec() ->
+ {[ purge_api()
+ , records_api()
+ , record_api()
+ ], definitions()}.
+
+definitions() ->
+ Rules = #{
+ type => array,
+ items => #{
+ type => object,
+ required => [topic, permission, action],
+ properties => #{
+ topic => #{
+ type => string,
+ example => <<"test/topic/1">>
+ },
+ permission => #{
+ type => string,
+ enum => [<<"allow">>, <<"deny">>],
+ example => <<"allow">>
+ },
+ action => #{
+ type => string,
+ enum => [<<"publish">>, <<"subscribe">>, <<"all">>],
+ example => <<"publish">>
+ }
+ }
+ }
+ },
+ Username = #{
+ type => object,
+ required => [username, rules],
+ properties => #{
+ username => #{
+ type => string,
+ example => <<"username">>
+ },
+ rules => minirest:ref(<<"rules">>)
+ }
+ },
+ Clientid = #{
+ type => object,
+ required => [clientid, rules],
+ properties => #{
+ clientid => #{
+ type => string,
+ example => <<"clientid">>
+ },
+ rules => minirest:ref(<<"rules">>)
+ }
+ },
+ ALL = #{
+ type => object,
+ required => [rules],
+ properties => #{
+ rules => minirest:ref(<<"rules">>)
+ }
+ },
+ [ #{<<"rules">> => Rules}
+ , #{<<"username">> => Username}
+ , #{<<"clientid">> => Clientid}
+ , #{<<"all">> => ALL}
+ ].
+
+purge_api() ->
+ Metadata = #{
+ delete => #{
+ description => "Purge all records",
+ responses => #{
+ <<"204">> => #{description => <<"No Content">>},
+ <<"400">> => emqx_mgmt_util:bad_request()
+ }
+ }
+ },
+ {"/authorization/sources/built-in-database/purge-all", Metadata, purge}.
+
+records_api() ->
+ Metadata = #{
+ get => #{
+ description => "List records",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"username">>, <<"clientid">>, <<"all">>]
+ },
+ required => true
+ },
+ #{
+ name => page,
+ in => query,
+ required => false,
+ description => <<"Page Index">>,
+ schema => #{type => integer}
+ },
+ #{
+ name => limit,
+ in => query,
+ required => false,
+ description => <<"Page limit">>,
+ schema => #{type => integer}
+ }
+ ],
+ responses => #{
+ <<"200">> => #{
+ description => <<"OK">>,
+ content => #{
+ 'application/json' => #{
+ schema => #{
+ type => array,
+ items => #{
+ oneOf => [ minirest:ref(<<"username">>)
+ , minirest:ref(<<"clientid">>)
+ , minirest:ref(<<"all">>)
+ ]
+ }
+ },
+ examples => #{
+ username => #{
+ summary => <<"Username">>,
+ value => jsx:encode([?EXAMPLE_USERNAME])
+ },
+ clientid => #{
+ summary => <<"Clientid">>,
+ value => jsx:encode([?EXAMPLE_CLIENTID])
+ },
+ all => #{
+ summary => <<"All">>,
+ value => jsx:encode([?EXAMPLE_ALL])
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ post => #{
+ description => "Add new records",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"username">>, <<"clientid">>]
+ },
+ required => true
+ }
+ ],
+ requestBody => #{
+ content => #{
+ 'application/json' => #{
+ schema => #{
+ type => array,
+ items => #{
+ oneOf => [ minirest:ref(<<"username">>)
+ , minirest:ref(<<"clientid">>)
+ ]
+ }
+ },
+ examples => #{
+ username => #{
+ summary => <<"Username">>,
+ value => jsx:encode([?EXAMPLE_USERNAME])
+ },
+ clientid => #{
+ summary => <<"Clientid">>,
+ value => jsx:encode([?EXAMPLE_CLIENTID])
+ }
+ }
+ }
+ }
+ },
+ responses => #{
+ <<"204">> => #{description => <<"Created">>},
+ <<"400">> => emqx_mgmt_util:bad_request()
+ }
+ },
+ put => #{
+ description => "Set the list of rules for all",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"all">>]
+ },
+ required => true
+ }
+ ],
+ requestBody => #{
+ content => #{
+ 'application/json' => #{
+ schema => #{
+ type => array,
+ items => #{
+ oneOf => [ minirest:ref(<<"username">>)
+ , minirest:ref(<<"clientid">>)
+ , minirest:ref(<<"all">>)
+ ]
+ }
+ },
+ examples => #{
+ username => #{
+ summary => <<"Username">>,
+ value => jsx:encode(?EXAMPLE_USERNAME)
+ },
+ clientid => #{
+ summary => <<"Clientid">>,
+ value => jsx:encode(?EXAMPLE_CLIENTID)
+ },
+ all => #{
+ summary => <<"All">>,
+ value => jsx:encode(?EXAMPLE_ALL)
+ }
+ }
+ }
+ }
+ },
+ responses => #{
+ <<"204">> => #{description => <<"Created">>},
+ <<"400">> => emqx_mgmt_util:bad_request()
+ }
+ }
+ },
+ {"/authorization/sources/built-in-database/:type", Metadata, records}.
+
+record_api() ->
+ Metadata = #{
+ get => #{
+ description => "Get record info",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"username">>, <<"clientid">>]
+ },
+ required => true
+ },
+ #{
+ name => key,
+ in => path,
+ schema => #{
+ type => string
+ },
+ required => true
+ }
+ ],
+ responses => #{
+ <<"200">> => #{
+ description => <<"OK">>,
+ content => #{
+ 'application/json' => #{
+ schema => #{
+ oneOf => [ minirest:ref(<<"username">>)
+ , minirest:ref(<<"clientid">>)
+ ]
+ },
+ examples => #{
+ username => #{
+ summary => <<"Username">>,
+ value => jsx:encode(?EXAMPLE_USERNAME)
+ },
+ clientid => #{
+ summary => <<"Clientid">>,
+ value => jsx:encode(?EXAMPLE_CLIENTID)
+ }
+ }
+ }
+ }
+ },
+ <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>)
+ }
+ },
+ put => #{
+ description => "Update one record",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"username">>, <<"clientid">>]
+ },
+ required => true
+ },
+ #{
+ name => key,
+ in => path,
+ schema => #{
+ type => string
+ },
+ required => true
+ }
+ ],
+ requestBody => #{
+ content => #{
+ 'application/json' => #{
+ schema => #{
+ oneOf => [ minirest:ref(<<"username">>)
+ , minirest:ref(<<"clientid">>)
+ ]
+ },
+ examples => #{
+ username => #{
+ summary => <<"Username">>,
+ value => jsx:encode(?EXAMPLE_USERNAME)
+ },
+ clientid => #{
+ summary => <<"Clientid">>,
+ value => jsx:encode(?EXAMPLE_CLIENTID)
+ }
+ }
+ }
+ }
+ },
+ responses => #{
+ <<"204">> => #{description => <<"Updated">>},
+ <<"400">> => emqx_mgmt_util:bad_request()
+ }
+ },
+ delete => #{
+ description => "Delete one record",
+ parameters => [
+ #{
+ name => type,
+ in => path,
+ schema => #{
+ type => string,
+ enum => [<<"username">>, <<"clientid">>]
+ },
+ required => true
+ },
+ #{
+ name => key,
+ in => path,
+ schema => #{
+ type => string
+ },
+ required => true
+ }
+ ],
+ responses => #{
+ <<"204">> => #{description => <<"No Content">>},
+ <<"400">> => emqx_mgmt_util:bad_request()
+ }
+ }
+ },
+ {"/authorization/sources/built-in-database/:type/:key", Metadata, record}.
+
+purge(delete, _) ->
+ case emqx_authz_api_sources:get_raw_source(<<"built-in-database">>) of
+ [#{enable := false}] ->
+ ok = lists:foreach(fun(Key) ->
+ ok = ekka_mnesia:dirty_delete(?ACL_TABLE, Key)
+ end, mnesia:dirty_all_keys(?ACL_TABLE)),
+ {204};
+ _ ->
+ {400, #{code => <<"BAD_REQUEST">>,
+ message => <<"'built-in-database' type source must be disabled before purge.">>}}
+ end.
+
+records(get, #{bindings := #{type := <<"username">>},
+ query_string := Qs
+ }) ->
+ MatchSpec = ets:fun2ms(
+ fun({?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}, Rules}) ->
+ [{username, Username}, {rules, Rules}]
+ end),
+ Format = fun ([{username, Username}, {rules, Rules}]) ->
+ #{username => Username,
+ rules => [ #{topic => Topic,
+ action => Action,
+ permission => Permission
+ } || {Permission, Action, Topic} <- Rules]
+ }
+ end,
+ case Qs of
+ #{<<"limit">> := _, <<"page">> := _} = Page ->
+ {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)};
+ #{<<"limit">> := Limit} ->
+ case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of
+ {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]};
+ '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}
+ end;
+ _ ->
+ {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]}
+ end;
+
+records(get, #{bindings := #{type := <<"clientid">>},
+ query_string := Qs
+ }) ->
+ MatchSpec = ets:fun2ms(
+ fun({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}, Rules}) ->
+ [{clientid, Clientid}, {rules, Rules}]
+ end),
+ Format = fun ([{clientid, Clientid}, {rules, Rules}]) ->
+ #{clientid => Clientid,
+ rules => [ #{topic => Topic,
+ action => Action,
+ permission => Permission
+ } || {Permission, Action, Topic} <- Rules]
+ }
+ end,
+ case Qs of
+ #{<<"limit">> := _, <<"page">> := _} = Page ->
+ {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)};
+ #{<<"limit">> := Limit} ->
+ case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of
+ {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]};
+ '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}
+ end;
+ _ ->
+ {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]}
+ end;
+records(get, #{bindings := #{type := <<"all">>}}) ->
+ MatchSpec = ets:fun2ms(
+ fun({?ACL_TABLE, ?ACL_TABLE_ALL, Rules}) ->
+ [{rules, Rules}]
+ end),
+ {200, [ #{rules => [ #{topic => Topic,
+ action => Action,
+ permission => Permission
+ } || {Permission, Action, Topic} <- Rules]
+ } || [{rules, Rules}] <- ets:select(?ACL_TABLE, MatchSpec)]};
+records(post, #{bindings := #{type := <<"username">>},
+ body := Body}) when is_list(Body) ->
+ lists:foreach(fun(#{<<"username">> := Username, <<"rules">> := Rules}) ->
+ ekka_mnesia:dirty_write(#emqx_acl{
+ who = {?ACL_TABLE_USERNAME, Username},
+ rules = format_rules(Rules)
+ })
+ end, Body),
+ {204};
+records(post, #{bindings := #{type := <<"clientid">>},
+ body := Body}) when is_list(Body) ->
+ lists:foreach(fun(#{<<"clientid">> := Clientid, <<"rules">> := Rules}) ->
+ ekka_mnesia:dirty_write(#emqx_acl{
+ who = {?ACL_TABLE_CLIENTID, Clientid},
+ rules = format_rules(Rules)
+ })
+ end, Body),
+ {204};
+records(put, #{bindings := #{type := <<"all">>},
+ body := #{<<"rules">> := Rules}}) ->
+ ekka_mnesia:dirty_write(#emqx_acl{
+ who = ?ACL_TABLE_ALL,
+ rules = format_rules(Rules)
+ }),
+ {204}.
+
+record(get, #{bindings := #{type := <<"username">>, key := Key}}) ->
+ case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_USERNAME, Key}) of
+ [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}};
+ [#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}] ->
+ {200, #{username => Username,
+ rules => [ #{topic => Topic,
+ action => Action,
+ permission => Permission
+ } || {Permission, Action, Topic} <- Rules]}
+ }
+ end;
+record(get, #{bindings := #{type := <<"clientid">>, key := Key}}) ->
+ case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Key}) of
+ [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}};
+ [#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}] ->
+ {200, #{clientid => Clientid,
+ rules => [ #{topic => Topic,
+ action => Action,
+ permission => Permission
+ } || {Permission, Action, Topic} <- Rules]}
+ }
+ end;
+record(put, #{bindings := #{type := <<"username">>, key := Username},
+ body := #{<<"username">> := Username, <<"rules">> := Rules}}) ->
+ ekka_mnesia:dirty_write(#emqx_acl{
+ who = {?ACL_TABLE_USERNAME, Username},
+ rules = format_rules(Rules)
+ }),
+ {204};
+record(put, #{bindings := #{type := <<"clientid">>, key := Clientid},
+ body := #{<<"clientid">> := Clientid, <<"rules">> := Rules}}) ->
+ ekka_mnesia:dirty_write(#emqx_acl{
+ who = {?ACL_TABLE_CLIENTID, Clientid},
+ rules = format_rules(Rules)
+ }),
+ {204};
+record(delete, #{bindings := #{type := <<"username">>, key := Key}}) ->
+ ekka_mnesia:dirty_delete({?ACL_TABLE, {?ACL_TABLE_USERNAME, Key}}),
+ {204};
+record(delete, #{bindings := #{type := <<"clientid">>, key := Key}}) ->
+ ekka_mnesia:dirty_delete({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Key}}),
+ {204}.
+
+format_rules(Rules) when is_list(Rules) ->
+ lists:foldl(fun(#{<<"topic">> := Topic,
+ <<"action">> := Action,
+ <<"permission">> := Permission
+ }, AccIn) when ?PUBSUB(Action)
+ andalso ?ALLOW_DENY(Permission) ->
+ AccIn ++ [{ atom(Permission), atom(Action), Topic }]
+ end, [], Rules).
+
+atom(B) when is_binary(B) ->
+ try binary_to_existing_atom(B, utf8)
+ catch
+ _ -> binary_to_atom(B)
+ end;
+atom(A) when is_atom(A) -> A.
diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl
index b41aaf3b3..b05476b03 100644
--- a/apps/emqx_authz/src/emqx_authz_api_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl
@@ -21,6 +21,7 @@
definitions() ->
Sources = #{
oneOf => [ minirest:ref(<<"http">>)
+ , minirest:ref(<<"built-in-database">>)
, minirest:ref(<<"mongo_single">>)
, minirest:ref(<<"mongo_rs">>)
, minirest:ref(<<"mongo_sharded">>)
@@ -79,9 +80,9 @@ definitions() ->
},
headers => #{type => object},
body => #{type => object},
- connect_timeout => #{type => integer},
+ connect_timeout => #{type => string},
max_retries => #{type => integer},
- retry_interval => #{type => integer},
+ retry_interval => #{type => string},
pool_type => #{
type => string,
enum => [<<"random">>, <<"hash">>],
@@ -133,8 +134,8 @@ definitions() ->
properties => #{
pool_size => #{type => integer},
max_overflow => #{type => integer},
- overflow_ttl => #{type => integer},
- overflow_check_period => #{type => integer},
+ overflow_ttl => #{type => string},
+ overflow_check_period => #{type => string},
local_threshold_ms => #{type => integer},
connect_timeout_ms => #{type => integer},
socket_timeout_ms => #{type => integer},
@@ -191,8 +192,8 @@ definitions() ->
properties => #{
pool_size => #{type => integer},
max_overflow => #{type => integer},
- overflow_ttl => #{type => integer},
- overflow_check_period => #{type => integer},
+ overflow_ttl => #{type => string},
+ overflow_check_period => #{type => string},
local_threshold_ms => #{type => integer},
connect_timeout_ms => #{type => integer},
socket_timeout_ms => #{type => integer},
@@ -247,8 +248,8 @@ definitions() ->
properties => #{
pool_size => #{type => integer},
max_overflow => #{type => integer},
- overflow_ttl => #{type => integer},
- overflow_check_period => #{type => integer},
+ overflow_ttl => #{type => string},
+ overflow_check_period => #{type => string},
local_threshold_ms => #{type => integer},
connect_timeout_ms => #{type => integer},
socket_timeout_ms => #{type => integer},
@@ -446,6 +447,21 @@ definitions() ->
ssl => minirest:ref(<<"ssl">>)
}
},
+ Mnesia = #{
+ type => object,
+ required => [type, enable],
+ properties => #{
+ type => #{
+ type => string,
+ enum => [<<"redis">>],
+ example => <<"redis">>
+ },
+ enable => #{
+ type => boolean,
+ example => true
+ }
+ }
+ },
File = #{
type => object,
required => [type, enable, rules],
@@ -475,6 +491,7 @@ definitions() ->
[ #{<<"sources">> => Sources}
, #{<<"ssl">> => SSL}
, #{<<"http">> => HTTP}
+ , #{<<"built-in-database">> => Mnesia}
, #{<<"mongo_single">> => MongoSingle}
, #{<<"mongo_rs">> => MongoRs}
, #{<<"mongo_sharded">> => MongoSharded}
diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl
index 15820b4ed..efe402fa7 100644
--- a/apps/emqx_authz/src/emqx_authz_api_sources.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl
@@ -41,6 +41,10 @@
]
}).
+-export([ get_raw_sources/0
+ , get_raw_source/1
+ ]).
+
-export([ api_spec/0
, sources/2
, source/2
@@ -147,7 +151,15 @@ source_api() ->
name => type,
in => path,
schema => #{
- type => string
+ type => string,
+ enum => [ <<"file">>
+ , <<"http">>
+ , <<"mongodb">>
+ , <<"mysql">>
+ , <<"postgresql">>
+ , <<"redis">>
+ , <<"built-in-database">>
+ ]
},
required => true
}
@@ -181,7 +193,15 @@ source_api() ->
name => type,
in => path,
schema => #{
- type => string
+ type => string,
+ enum => [ <<"file">>
+ , <<"http">>
+ , <<"mongodb">>
+ , <<"mysql">>
+ , <<"postgresql">>
+ , <<"redis">>
+ , <<"built-in-database">>
+ ]
},
required => true
}
@@ -216,7 +236,15 @@ source_api() ->
name => type,
in => path,
schema => #{
- type => string
+ type => string,
+ enum => [ <<"file">>
+ , <<"http">>
+ , <<"mongodb">>
+ , <<"mysql">>
+ , <<"postgresql">>
+ , <<"redis">>
+ , <<"built-in-database">>
+ ]
},
required => true
}
@@ -238,7 +266,15 @@ move_source_api() ->
name => type,
in => path,
schema => #{
- type => string
+ type => string,
+ enum => [ <<"file">>
+ , <<"http">>
+ , <<"mongodb">>
+ , <<"mysql">>
+ , <<"postgresql">>
+ , <<"redis">>
+ , <<"built-in-database">>
+ ]
},
required => true
}
@@ -290,7 +326,7 @@ move_source_api() ->
{"/authorization/sources/:type/move", Metadata, move_source}.
sources(get, _) ->
- Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) ->
+ Sources = lists:foldl(fun (#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}, AccIn) ->
case file:read_file(Path) of
{ok, Rules} ->
lists:append(AccIn, [#{type => file,
@@ -309,7 +345,7 @@ sources(get, _) ->
{200, #{sources => Sources}};
sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) ->
{ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules),
- update_config(head, [#{type => file, enable => true, path => Filename}]);
+ update_config(head, [#{<<"type">> => <<"file">>, <<"enable">> => true, <<"path">> => Filename}]);
sources(post, #{body := Body}) when is_map(Body) ->
update_config(head, [write_cert(Body)]);
sources(put, #{body := Body}) when is_list(Body) ->
@@ -317,16 +353,16 @@ sources(put, #{body := Body}) when is_list(Body) ->
case Source of
#{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable} ->
{ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules),
- #{type => file, enable => Enable, path => Filename};
+ #{<<"type">> => <<"file">>, <<"enable">> => Enable, <<"path">> => Filename};
_ -> write_cert(Source)
end
end || Source <- Body],
- update_config(replace, NBody).
+ update_config(?CMD_REPLCAE, NBody).
source(get, #{bindings := #{type := Type}}) ->
case get_raw_source(Type) of
[] -> {404, #{message => <<"Not found ", Type/binary>>}};
- [#{type := <<"file">>, enable := Enable, path := Path}] ->
+ [#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}] ->
case file:read_file(Path) of
{ok, Rules} ->
{200, #{type => file,
@@ -336,23 +372,23 @@ source(get, #{bindings := #{type := Type}}) ->
};
{error, Reason} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}}
+ message => bin(Reason)}}
end;
[Source] ->
{200, read_cert(Source)}
end;
source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) ->
{ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), Rules),
- case emqx_authz:update({replace_once, file}, #{type => file, enable => Enable, path => Filename}) of
+ case emqx_authz:update({?CMD_REPLCAE, file}, #{<<"type">> => file, <<"enable">> => Enable, <<"path">> => Filename}) of
{ok, _} -> {204};
{error, Reason} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}}
+ message => bin(Reason)}}
end;
source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) ->
- update_config({replace_once, Type}, write_cert(Body));
+ update_config({?CMD_REPLCAE, Type}, write_cert(Body));
source(delete, #{bindings := #{type := Type}}) ->
- update_config({delete_once, Type}, #{}).
+ update_config({?CMD_DELETE, Type}, #{}).
move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) ->
case emqx_authz:move(Type, Position) of
@@ -362,18 +398,18 @@ move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos
message => <<"source ", Type/binary, " not found">>}};
{error, Reason} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}}
+ message => bin(Reason)}}
end.
get_raw_sources() ->
RawSources = emqx:get_raw_config([authorization, sources]),
Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}},
Conf = #{<<"sources">> => RawSources},
- #{sources := Sources} = hocon_schema:check_plain(Schema, Conf, #{atom_key => true, no_conversion => true}),
+ #{<<"sources">> := Sources} = hocon_schema:check_plain(Schema, Conf, #{only_fill_defaults => true}),
Sources.
get_raw_source(Type) ->
- lists:filter(fun (#{type := T}) ->
+ lists:filter(fun (#{<<"type">> := T}) ->
T =:= Type
end, get_raw_sources()).
@@ -382,16 +418,16 @@ update_config(Cmd, Sources) ->
{ok, _} -> {204};
{error, {pre_config_update, emqx_authz, Reason}} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}};
+ message => bin(Reason)}};
{error, {post_config_update, emqx_authz, Reason}} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}};
+ message => bin(Reason)}};
{error, Reason} ->
{400, #{code => <<"BAD_REQUEST">>,
- message => atom_to_binary(Reason)}}
+ message => bin(Reason)}}
end.
-read_cert(#{ssl := #{enable := true} = SSL} = Source) ->
+read_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) ->
CaCert = case file:read_file(maps:get(cacertfile, SSL, "")) of
{ok, CaCert0} -> CaCert0;
_ -> ""
@@ -459,3 +495,6 @@ do_write_file(Filename, Bytes) ->
?LOG(error, "Write File ~p Error: ~p", [Filename, Reason]),
error(Reason)
end.
+
+bin(Term) ->
+ erlang:iolist_to_binary(io_lib:format("~p", [Term])).
diff --git a/apps/emqx_authz/src/emqx_authz_app.erl b/apps/emqx_authz/src/emqx_authz_app.erl
index 460d7cbf9..f868ac342 100644
--- a/apps/emqx_authz/src/emqx_authz_app.erl
+++ b/apps/emqx_authz/src/emqx_authz_app.erl
@@ -7,9 +7,12 @@
-behaviour(application).
+-include("emqx_authz.hrl").
+
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
+ ok = ekka_rlog:wait_for_shards([?ACL_SHARDED], infinity),
{ok, Sup} = emqx_authz_sup:start_link(),
ok = emqx_authz:init(),
{ok, Sup}.
diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl
new file mode 100644
index 000000000..ab755403e
--- /dev/null
+++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl
@@ -0,0 +1,76 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_mnesia).
+
+-include("emqx_authz.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+%% AuthZ Callbacks
+-export([ mnesia/1
+ , authorize/4
+ , description/0
+ ]).
+
+-ifdef(TEST).
+-compile(export_all).
+-compile(nowarn_export_all).
+-endif.
+
+-boot_mnesia({mnesia, [boot]}).
+-copy_mnesia({mnesia, [copy]}).
+
+-spec(mnesia(boot | copy) -> ok).
+mnesia(boot) ->
+ ok = ekka_mnesia:create_table(?ACL_TABLE, [
+ {type, ordered_set},
+ {rlog_shard, ?ACL_SHARDED},
+ {disc_copies, [node()]},
+ {attributes, record_info(fields, ?ACL_TABLE)},
+ {storage_properties, [{ets, [{read_concurrency, true}]}]}]);
+mnesia(copy) ->
+ ok = ekka_mnesia:copy_table(?ACL_TABLE, disc_copies).
+
+description() ->
+ "AuthZ with Mnesia".
+
+authorize(#{username := Username,
+ clientid := Clientid
+ } = Client, PubSub, Topic, #{type := 'built-in-database'}) ->
+
+ Rules = case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}) of
+ [] -> [];
+ [#emqx_acl{rules = Rules0}] when is_list(Rules0) -> Rules0
+ end
+ ++ case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}) of
+ [] -> [];
+ [#emqx_acl{rules = Rules1}] when is_list(Rules1) -> Rules1
+ end
+ ++ case mnesia:dirty_read(?ACL_TABLE, ?ACL_TABLE_ALL) of
+ [] -> [];
+ [#emqx_acl{rules = Rules2}] when is_list(Rules2) -> Rules2
+ end,
+ do_authorize(Client, PubSub, Topic, Rules).
+
+do_authorize(_Client, _PubSub, _Topic, []) -> nomatch;
+do_authorize(Client, PubSub, Topic, [ {Permission, Action, TopicFilter} | Tail]) ->
+ case emqx_authz_rule:match(Client, PubSub, Topic,
+ emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]})
+ ) of
+ {matched, Permission} -> {matched, Permission};
+ nomatch -> do_authorize(Client, PubSub, Topic, Tail)
+ end.
diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl
index 6c0fb126a..ca65e6f53 100644
--- a/apps/emqx_authz/src/emqx_authz_mongodb.erl
+++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl
@@ -58,9 +58,9 @@ do_authorize(Client, PubSub, Topic, [Rule | Tail]) ->
end.
replvar(Selector, #{clientid := Clientid,
- username := Username,
- peerhost := IpAddress
- }) ->
+ username := Username,
+ peerhost := IpAddress
+ }) ->
Fun = fun
_Fun(K, V, AccIn) when is_map(V) -> maps:put(K, maps:fold(_Fun, AccIn, V), AccIn);
_Fun(K, V, AccIn) when is_list(V) ->
diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl
index ac8f04f32..6ad206d90 100644
--- a/apps/emqx_authz/src/emqx_authz_mysql.erl
+++ b/apps/emqx_authz/src/emqx_authz_mysql.erl
@@ -69,7 +69,6 @@ do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) ->
nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail)
end.
-
format_result(Columns, Row) ->
Permission = lists:nth(index(<<"permission">>, Columns), Row),
Action = lists:nth(index(<<"action">>, Columns), Row),
diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl
index deb8968c6..5f4dcfcab 100644
--- a/apps/emqx_authz/src/emqx_authz_rule.erl
+++ b/apps/emqx_authz/src/emqx_authz_rule.erl
@@ -32,16 +32,21 @@
-export_type([rule/0]).
+compile({Permission, all}) when ?ALLOW_DENY(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]};
compile({Permission, Who, Action, TopicFilters}) when ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(TopicFilters) ->
{atom(Permission), compile_who(Who), atom(Action), [compile_topic(Topic) || Topic <- TopicFilters]}.
compile_who(all) -> all;
-compile_who({username, Username}) ->
+compile_who({user, Username}) -> compile_who({username, Username});
+compile_who({username, {re, Username}}) ->
{ok, MP} = re:compile(bin(Username)),
{username, MP};
-compile_who({clientid, Clientid}) ->
+compile_who({username, Username}) -> {username, {eq, bin(Username)}};
+compile_who({client, Clientid}) -> compile_who({clientid, Clientid});
+compile_who({clientid, {re, Clientid}}) ->
{ok, MP} = re:compile(bin(Clientid)),
{clientid, MP};
+compile_who({clientid, Clientid}) -> {clientid, {eq, bin(Clientid)}};
compile_who({ipaddr, CIDR}) ->
{ipaddr, esockd_cidr:parse(CIDR, true)};
compile_who({ipaddrs, CIDRs}) ->
@@ -102,14 +107,16 @@ match_action(_, all) -> true;
match_action(_, _) -> false.
match_who(_, all) -> true;
-match_who(#{username := undefined}, {username, _MP}) ->
+match_who(#{username := undefined}, {username, _}) ->
false;
-match_who(#{username := Username}, {username, MP}) ->
+match_who(#{username := Username}, {username, {eq, Username}}) -> true;
+match_who(#{username := Username}, {username, {re_pattern, _, _, _, _} = MP}) ->
case re:run(Username, MP) of
{match, _} -> true;
_ -> false
end;
-match_who(#{clientid := Clientid}, {clientid, MP}) ->
+match_who(#{clientid := Clientid}, {clientid, {eq, Clientid}}) -> true;
+match_who(#{clientid := Clientid}, {clientid, {re_pattern, _, _, _, _} = MP}) ->
case re:run(Clientid, MP) of
{match, _} -> true;
_ -> false
diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl
index c880b7669..50c2ec3ff 100644
--- a/apps/emqx_authz/src/emqx_authz_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_schema.erl
@@ -18,6 +18,8 @@
, fields/1
]).
+-import(emqx_schema, [mk_duration/2]).
+
namespace() -> authz.
%% @doc authorization schema is not exported
@@ -29,6 +31,7 @@ fields("authorization") ->
[ hoconsc:ref(?MODULE, file)
, hoconsc:ref(?MODULE, http_get)
, hoconsc:ref(?MODULE, http_post)
+ , hoconsc:ref(?MODULE, mnesia)
, hoconsc:ref(?MODULE, mongo_single)
, hoconsc:ref(?MODULE, mongo_rs)
, hoconsc:ref(?MODULE, mongo_sharded)
@@ -45,11 +48,7 @@ fields(file) ->
, {enable, #{type => boolean(),
default => true}}
, {path, #{type => string(),
- validator => fun(S) -> case filelib:is_file(S) of
- true -> ok;
- _ -> {error, "File does not exist"}
- end
- end
+ desc => "Path to the file which contains the ACL rules."
}}
];
fields(http_get) ->
@@ -77,7 +76,7 @@ fields(http_get) ->
end
}
}
- , {request_timeout, #{type => timeout(), default => 30000 }}
+ , {request_timeout, mk_duration("request timeout", #{default => "30s"})}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config));
fields(http_post) ->
[ {type, #{type => http}}
@@ -107,12 +106,17 @@ fields(http_post) ->
end
}
}
- , {request_timeout, #{type => timeout(), default => 30000 }}
+ , {request_timeout, mk_duration("request timeout", #{default => "30s"})}
, {body, #{type => map(),
nullable => true
}
}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config));
+fields(mnesia) ->
+ [ {type, #{type => 'built-in-database'}}
+ , {enable, #{type => boolean(),
+ default => true}}
+ ];
fields(mongo_single) ->
[ {collection, #{type => atom()}}
, {selector, #{type => map()}}
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index bfdc131a0..16bf39d49 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -50,14 +50,14 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
- {ok, _} = emqx_authz:update(replace, []),
+ {ok, _} = emqx_authz:update(?CMD_REPLCAE, []),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_resource),
meck:unload(emqx_schema),
ok.
init_per_testcase(_, Config) ->
- {ok, _} = emqx_authz:update(replace, []),
+ {ok, _} = emqx_authz:update(?CMD_REPLCAE, []),
Config.
-define(SOURCE1, #{<<"type">> => <<"http">>,
@@ -120,12 +120,12 @@ init_per_testcase(_, Config) ->
%%------------------------------------------------------------------------------
t_update_source(_) ->
- {ok, _} = emqx_authz:update(replace, [?SOURCE3]),
- {ok, _} = emqx_authz:update(head, [?SOURCE2]),
- {ok, _} = emqx_authz:update(head, [?SOURCE1]),
- {ok, _} = emqx_authz:update(tail, [?SOURCE4]),
- {ok, _} = emqx_authz:update(tail, [?SOURCE5]),
- {ok, _} = emqx_authz:update(tail, [?SOURCE6]),
+ {ok, _} = emqx_authz:update(?CMD_REPLCAE, [?SOURCE3]),
+ {ok, _} = emqx_authz:update(?CMD_PREPEND, [?SOURCE2]),
+ {ok, _} = emqx_authz:update(?CMD_PREPEND, [?SOURCE1]),
+ {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE4]),
+ {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE5]),
+ {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE6]),
?assertMatch([ #{type := http, enable := true}
, #{type := mongodb, enable := true}
@@ -135,12 +135,12 @@ t_update_source(_) ->
, #{type := file, enable := true}
], emqx:get_config([authorization, sources], [])),
- {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}),
- {ok, _} = emqx_authz:update({replace_once, mongodb}, ?SOURCE2#{<<"enable">> := false}),
- {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}),
- {ok, _} = emqx_authz:update({replace_once, postgresql}, ?SOURCE4#{<<"enable">> := false}),
- {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}),
- {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, http}, ?SOURCE1#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, mongodb}, ?SOURCE2#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, mysql}, ?SOURCE3#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, postgresql}, ?SOURCE4#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, redis}, ?SOURCE5#{<<"enable">> := false}),
+ {ok, _} = emqx_authz:update({?CMD_REPLCAE, file}, ?SOURCE6#{<<"enable">> := false}),
?assertMatch([ #{type := http, enable := false}
, #{type := mongodb, enable := false}
@@ -150,10 +150,10 @@ t_update_source(_) ->
, #{type := file, enable := false}
], emqx:get_config([authorization, sources], [])),
- {ok, _} = emqx_authz:update(replace, []).
+ {ok, _} = emqx_authz:update(?CMD_REPLCAE, []).
t_move_source(_) ->
- {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]),
+ {ok, _} = emqx_authz:update(?CMD_REPLCAE, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]),
?assertMatch([ #{type := http}
, #{type := mongodb}
, #{type := mysql}
diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
new file mode 100644
index 000000000..1ea942c10
--- /dev/null
+++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
@@ -0,0 +1,224 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_api_mnesia_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include("emqx_authz.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(CONF_DEFAULT, <<"authorization: {sources: []}">>).
+
+-import(emqx_ct_http, [ request_api/3
+ , request_api/5
+ , get_http_data/1
+ , create_default_app/0
+ , delete_default_app/0
+ , default_auth_header/0
+ , auth_header/2
+ ]).
+
+-define(HOST, "http://127.0.0.1:18083/").
+-define(API_VERSION, "v5").
+-define(BASE_PATH, "api").
+
+-define(EXAMPLE_USERNAME, #{username => user1,
+ rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+-define(EXAMPLE_CLIENTID, #{clientid => client1,
+ rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+-define(EXAMPLE_ALL , #{rules => [ #{topic => <<"test/toopic/1">>,
+ permission => <<"allow">>,
+ action => <<"publish">>
+ }
+ , #{topic => <<"test/toopic/2">>,
+ permission => <<"allow">>,
+ action => <<"subscribe">>
+ }
+ , #{topic => <<"eq test/#">>,
+ permission => <<"deny">>,
+ action => <<"all">>
+ }
+ ]
+ }).
+
+all() ->
+ []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error
+ % emqx_ct:all(?MODULE).
+
+groups() ->
+ [].
+
+init_per_suite(Config) ->
+ meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
+ meck:expect(emqx_schema, fields, fun("authorization") ->
+ meck:passthrough(["authorization"]) ++
+ emqx_authz_schema:fields("authorization");
+ (F) -> meck:passthrough([F])
+ end),
+
+ ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT),
+
+ ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1),
+ {ok, _} = emqx:update_config([authorization, cache, enable], false),
+ {ok, _} = emqx:update_config([authorization, no_match], deny),
+
+ Config.
+
+end_per_suite(_Config) ->
+ {ok, _} = emqx_authz:update(replace, []),
+ emqx_ct_helpers:stop_apps([emqx_authz, emqx_dashboard]),
+ meck:unload(emqx_schema),
+ ok.
+
+set_special_configs(emqx_dashboard) ->
+ Config = #{
+ default_username => <<"admin">>,
+ default_password => <<"public">>,
+ listeners => [#{
+ protocol => http,
+ port => 18083
+ }]
+ },
+ emqx_config:put([emqx_dashboard], Config),
+ ok;
+set_special_configs(emqx_authz) ->
+ emqx_config:put([authorization], #{sources => [#{type => 'built-in-database',
+ enable => true}
+ ]}),
+ ok;
+set_special_configs(_App) ->
+ ok.
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_api(_) ->
+ {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "username"]), [?EXAMPLE_USERNAME]),
+ {ok, 200, Request1} = request(get, uri(["authorization", "sources", "built-in-database", "username"]), []),
+ {ok, 200, Request2} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []),
+ [#{<<"username">> := <<"user1">>, <<"rules">> := Rules1}] = jsx:decode(Request1),
+ #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2),
+ ?assertEqual(3, length(Rules1)),
+
+ {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "username", "user1"]), ?EXAMPLE_USERNAME#{rules => []}),
+ {ok, 200, Request3} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []),
+ #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3),
+ ?assertEqual(0, length(Rules2)),
+
+ {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []),
+ {ok, 404, _} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []),
+
+ {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "clientid"]), [?EXAMPLE_CLIENTID]),
+ {ok, 200, Request4} = request(get, uri(["authorization", "sources", "built-in-database", "clientid"]), []),
+ {ok, 200, Request5} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []),
+ [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}] = jsx:decode(Request4),
+ #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5),
+ ?assertEqual(3, length(Rules3)),
+
+ {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), ?EXAMPLE_CLIENTID#{rules => []}),
+ {ok, 200, Request6} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []),
+ #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6),
+ ?assertEqual(0, length(Rules4)),
+
+ {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []),
+ {ok, 404, _} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []),
+
+ {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "all"]), ?EXAMPLE_ALL),
+ {ok, 200, Request7} = request(get, uri(["authorization", "sources", "built-in-database", "all"]), []),
+ [#{<<"rules">> := Rules5}] = jsx:decode(Request7),
+ ?assertEqual(3, length(Rules5)),
+
+ {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "all"]), ?EXAMPLE_ALL#{rules => []}),
+ {ok, 200, Request8} = request(get, uri(["authorization", "sources", "built-in-database", "all"]), []),
+ [#{<<"rules">> := Rules6}] = jsx:decode(Request8),
+ ?assertEqual(0, length(Rules6)),
+
+ {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "username"]), [ #{username => N, rules => []} || N <- lists:seq(1, 20) ]),
+ {ok, 200, Request9} = request(get, uri(["authorization", "sources", "built-in-database", "username?page=2&limit=5"]), []),
+ #{<<"data">> := Data1} = jsx:decode(Request9),
+ ?assertEqual(5, length(Data1)),
+
+ {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "clientid"]), [ #{clientid => N, rules => []} || N <- lists:seq(1, 20) ]),
+ {ok, 200, Request10} = request(get, uri(["authorization", "sources", "built-in-database", "clientid?limit=5"]), []),
+ ?assertEqual(5, length(jsx:decode(Request10))),
+
+ {ok, 400, _} = request(delete, uri(["authorization", "sources", "built-in-database", "purge-all"]), []),
+ {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database"]), #{<<"enable">> => false}),
+ {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "purge-all"]), []),
+ ?assertEqual([], mnesia:dirty_all_keys(?ACL_TABLE)),
+
+ ok.
+
+%%--------------------------------------------------------------------
+%% HTTP Request
+%%--------------------------------------------------------------------
+
+request(Method, Url, Body) ->
+ Request = case Body of
+ [] -> {Url, [auth_header_()]};
+ _ -> {Url, [auth_header_()], "application/json", jsx:encode(Body)}
+ end,
+ ct:pal("Method: ~p, Request: ~p", [Method, Request]),
+ case httpc:request(Method, Request, [], [{body_format, binary}]) of
+ {error, socket_closed_remotely} ->
+ {error, socket_closed_remotely};
+ {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } ->
+ {ok, Code, Return};
+ {ok, {Reason, _, _}} ->
+ {error, Reason}
+ end.
+
+uri() -> uri([]).
+uri(Parts) when is_list(Parts) ->
+ NParts = [E || E <- Parts],
+ ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]).
+
+get_sources(Result) -> jsx:decode(Result).
+
+auth_header_() ->
+ Username = <<"admin">>,
+ Password = <<"public">>,
+ {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
+ {"Authorization", "Bearer " ++ binary_to_list(Token)}.
diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
index d000162c0..86b347b98 100644
--- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
@@ -42,7 +42,7 @@
<<"url">> => <<"https://fake.com:443/">>,
<<"headers">> => #{},
<<"method">> => <<"get">>,
- <<"request_timeout">> => 5000
+ <<"request_timeout">> => <<"5s">>
}).
-define(SOURCE2, #{<<"type">> => <<"mongodb">>,
<<"enable">> => true,
diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
new file mode 100644
index 000000000..8b221d3e7
--- /dev/null
+++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
@@ -0,0 +1,109 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_mnesia_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include("emqx_authz.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(CONF_DEFAULT, <<"authorization: {sources: []}">>).
+
+all() ->
+ emqx_ct:all(?MODULE).
+
+groups() ->
+ [].
+
+init_per_suite(Config) ->
+ meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
+ meck:expect(emqx_schema, fields, fun("authorization") ->
+ meck:passthrough(["authorization"]) ++
+ emqx_authz_schema:fields("authorization");
+ (F) -> meck:passthrough([F])
+ end),
+
+ ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT),
+ ok = emqx_ct_helpers:start_apps([emqx_authz]),
+
+ {ok, _} = emqx:update_config([authorization, cache, enable], false),
+ {ok, _} = emqx:update_config([authorization, no_match], deny),
+ Rules = [#{<<"type">> => <<"built-in-database">>}],
+ {ok, _} = emqx_authz:update(replace, Rules),
+ Config.
+
+end_per_suite(_Config) ->
+ {ok, _} = emqx_authz:update(replace, []),
+ emqx_ct_helpers:stop_apps([emqx_authz]),
+ meck:unload(emqx_schema),
+ ok.
+
+init_per_testcase(t_authz, Config) ->
+ mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = {?ACL_TABLE_USERNAME, <<"test_username">>},
+ rules = [{allow, publish, <<"test/%u">>},
+ {allow, subscribe, <<"eq #">>}
+ ]
+ }]),
+ mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = {?ACL_TABLE_CLIENTID, <<"test_clientid">>},
+ rules = [{allow, publish, <<"test/%c">>},
+ {deny, subscribe, <<"eq #">>}
+ ]
+ }]),
+ mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = ?ACL_TABLE_ALL,
+ rules = [{deny, all, <<"#">>}]
+ }]),
+ Config;
+init_per_testcase(_, Config) -> Config.
+
+end_per_testcase(t_authz, Config) ->
+ [ ekka_mnesia:dirty_delete(?ACL_TABLE, K) || K <- mnesia:dirty_all_keys(?ACL_TABLE)],
+ Config;
+end_per_testcase(_, Config) -> Config.
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_authz(_) ->
+ ClientInfo1 = #{clientid => <<"test">>,
+ username => <<"test">>,
+ peerhost => {127,0,0,1},
+ listener => {tcp, default}
+ },
+ ClientInfo2 = #{clientid => <<"fake_clientid">>,
+ username => <<"test_username">>,
+ peerhost => {127,0,0,1},
+ listener => {tcp, default}
+ },
+ ClientInfo3 = #{clientid => <<"test_clientid">>,
+ username => <<"fake_username">>,
+ peerhost => {127,0,0,1},
+ listener => {tcp, default}
+ },
+
+ ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)),
+ ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"#">>)),
+
+ ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_username">>)),
+ ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, subscribe, <<"#">>)),
+
+ ?assertEqual(allow, emqx_access_control:authorize(ClientInfo3, publish, <<"test/test_clientid">>)),
+ ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, subscribe, <<"#">>)),
+
+ ok.
+
diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
index c38d99cba..3c7e314cd 100644
--- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
@@ -22,11 +22,11 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
--define(SOURCE1, {deny, all, all, ["#"]}).
+-define(SOURCE1, {deny, all}).
-define(SOURCE2, {allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}).
-define(SOURCE3, {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, ["%c"]}).
--define(SOURCE4, {allow, {'and', [{clientid, "^test?"}, {username, "^test?"}]}, publish, ["topic/test"]}).
--define(SOURCE5, {allow, {'or', [{username, "^test"}, {clientid, "test?"}]}, publish, ["%u", "%c"]}).
+-define(SOURCE4, {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}).
+-define(SOURCE5, {allow, {'or', [{username, {re, "^test"}}, {clientid, {re, "test?"}}]}, publish, ["%u", "%c"]}).
all() ->
emqx_ct:all(?MODULE).
@@ -52,7 +52,7 @@ t_compile(_) ->
}, emqx_authz_rule:compile(?SOURCE3)),
?assertMatch({allow,
- {'and', [{clientid, {re_pattern, _, _, _, _}}, {username, {re_pattern, _, _, _, _}}]},
+ {'and', [{clientid, {eq, <<"test">>}}, {username, {eq, <<"test">>}}]},
publish,
[[<<"topic">>, <<"test">>]]
}, emqx_authz_rule:compile(?SOURCE4)),
diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf
index e8af40341..f26172ef6 100644
--- a/apps/emqx_bridge/etc/emqx_bridge.conf
+++ b/apps/emqx_bridge/etc/emqx_bridge.conf
@@ -45,3 +45,30 @@
# retain = false
# }
#}
+#
+#bridges.http.my_http_bridge {
+# base_url: "http://localhost:9901"
+# connect_timeout: "30s"
+# max_retries: 3
+# retry_interval = "10s"
+# pool_type = "random"
+# pool_size = 4
+# enable_pipelining = true
+# ssl {
+# enable = false
+# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem"
+# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem"
+# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
+# }
+# egress_channels.post_messages {
+# subscribe_local_topic = "emqx_http/#"
+# request_timeout: "30s"
+# ## following config entries can use placehodler variables
+# method = post
+# path = "/messages/${topic}"
+# body = "${payload}"
+# headers {
+# "content-type": "application/json"
+# }
+# }
+#}
diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index e3458adca..351e6aeca 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -15,9 +15,15 @@
%%--------------------------------------------------------------------
-module(emqx_bridge).
-behaviour(emqx_config_handler).
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
-export([post_config_update/4]).
+-export([reload_hook/0, unload_hook/0]).
+
+-export([on_message_publish/1]).
+
-export([ load_bridges/0
, get_bridge/2
, get_bridge/3
@@ -28,6 +34,7 @@
, start_bridge/2
, stop_bridge/2
, restart_bridge/2
+ , send_message/2
]).
-export([ config_key_path/0
@@ -38,24 +45,57 @@
, resource_id/1
, resource_id/2
, parse_bridge_id/1
+ , channel_id/4
+ , parse_channel_id/1
]).
+reload_hook() ->
+ unload_hook(),
+ Bridges = emqx:get_config([bridges], #{}),
+ lists:foreach(fun({_Type, Bridge}) ->
+ lists:foreach(fun({_Name, BridgeConf}) ->
+ load_hook(BridgeConf)
+ end, maps:to_list(Bridge))
+ end, maps:to_list(Bridges)).
+
+load_hook(#{egress_channels := Channels}) ->
+ case has_subscribe_local_topic(Channels) of
+ true -> ok;
+ false -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []})
+ end;
+load_hook(_Conf) -> ok.
+
+unload_hook() ->
+ ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}).
+
+on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
+ case maps:get(sys, Flags, false) of
+ false ->
+ ChannelIds = get_matched_channels(Topic),
+ lists:foreach(fun(ChannelId) ->
+ send_message(ChannelId, emqx_message:to_map(Message))
+ end, ChannelIds);
+ true -> ok
+ end,
+ {ok, Message}.
+
+%% TODO: remove this clause, treat mqtt bridges the same as other bridges
+send_message(ChannelId, Message) ->
+ {BridgeType, BridgeName, _, _} = parse_channel_id(ChannelId),
+ ResId = emqx_bridge:resource_id(BridgeType, BridgeName),
+ do_send_message(ResId, ChannelId, Message).
+
+do_send_message(ResId, ChannelId, Message) ->
+ emqx_resource:query(ResId, {send_message, ChannelId, Message}).
+
config_key_path() ->
[bridges].
resource_type(mqtt) -> emqx_connector_mqtt;
-resource_type(mysql) -> emqx_connector_mysql;
-resource_type(pgsql) -> emqx_connector_pgsql;
-resource_type(mongo) -> emqx_connector_mongo;
-resource_type(redis) -> emqx_connector_redis;
-resource_type(ldap) -> emqx_connector_ldap.
+resource_type(http) -> emqx_connector_http.
bridge_type(emqx_connector_mqtt) -> mqtt;
-bridge_type(emqx_connector_mysql) -> mysql;
-bridge_type(emqx_connector_pgsql) -> pgsql;
-bridge_type(emqx_connector_mongo) -> mongo;
-bridge_type(emqx_connector_redis) -> redis;
-bridge_type(emqx_connector_ldap) -> ldap.
+bridge_type(emqx_connector_http) -> http.
post_config_update(_Req, NewConf, OldConf, _AppEnv) ->
#{added := Added, removed := Removed, changed := Updated}
@@ -100,11 +140,23 @@ bridge_id(BridgeType, BridgeName) ->
<>.
parse_bridge_id(BridgeId) ->
- try
- [Type, Name] = string:split(str(BridgeId), ":", leading),
- {list_to_existing_atom(Type), list_to_atom(Name)}
- catch
- _ : _ -> error({invalid_bridge_id, BridgeId})
+ case string:split(bin(BridgeId), ":", all) of
+ [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)};
+ _ -> error({invalid_bridge_id, BridgeId})
+ end.
+
+channel_id(BridgeType, BridgeName, ChannelType, ChannelName) ->
+ BType = bin(BridgeType),
+ BName = bin(BridgeName),
+ CType = bin(ChannelType),
+ CName = bin(ChannelName),
+ <>.
+
+parse_channel_id(ChannelId) ->
+ case string:split(bin(ChannelId), ":", all) of
+ [BridgeType, BridgeName, ChannelType, ChannelName] ->
+ {BridgeType, BridgeName, ChannelType, ChannelName};
+ _ -> error({invalid_bridge_id, ChannelId})
end.
list_bridges() ->
@@ -137,7 +189,8 @@ restart_bridge(Type, Name) ->
emqx_resource:restart(resource_id(Type, Name)).
create_bridge(Type, Name, Conf) ->
- logger:info("create ~p bridge ~p use config: ~p", [Type, Name, Conf]),
+ ?SLOG(info, #{msg => "create bridge", type => Type, name => Name,
+ config => Conf}),
ResId = resource_id(Type, Name),
case emqx_resource:create(ResId,
emqx_bridge:resource_type(Type), Conf) of
@@ -158,12 +211,13 @@ update_bridge(Type, Name, {_OldConf, Conf}) ->
%% `egress_channels` are changed, then we should not restart the bridge, we only restart/start
%% the channels.
%%
- logger:info("update ~p bridge ~p use config: ~p", [Type, Name, Conf]),
+ ?SLOG(info, #{msg => "update bridge", type => Type, name => Name,
+ config => Conf}),
emqx_resource:recreate(resource_id(Type, Name),
emqx_bridge:resource_type(Type), Conf, []).
remove_bridge(Type, Name, _Conf) ->
- logger:info("remove ~p bridge ~p", [Type, Name]),
+ ?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}),
case emqx_resource:remove(resource_id(Type, Name)) of
ok -> ok;
{error, not_found} -> ok;
@@ -184,13 +238,35 @@ flatten_confs(Conf0) ->
do_flatten_confs(Type, Conf0) ->
[{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)].
+has_subscribe_local_topic(Channels) ->
+ lists:any(fun (#{subscribe_local_topic := _}) -> true;
+ (_) -> false
+ end, maps:to_list(Channels)).
+
+get_matched_channels(Topic) ->
+ Bridges = emqx:get_config([bridges], #{}),
+ maps:fold(fun
+ %% TODO: also trigger 'message.publish' for mqtt bridges.
+ (mqtt, _Conf, Acc0) -> Acc0;
+ (BType, Conf, Acc0) ->
+ maps:fold(fun
+ (BName, #{egress_channels := Channels}, Acc1) ->
+ do_get_matched_channels(Topic, Channels, BType, BName, egress_channels)
+ ++ Acc1;
+ (_Name, _BridgeConf, Acc1) -> Acc1
+ end, Acc0, Conf)
+ end, [], Bridges).
+
+do_get_matched_channels(Topic, Channels, BType, BName, CType) ->
+ maps:fold(fun
+ (ChannName, #{subscribe_local_topic := Filter}, Acc) ->
+ case emqx_topic:match(Topic, Filter) of
+ true -> [channel_id(BType, BName, CType, ChannName) | Acc];
+ false -> Acc
+ end;
+ (_ChannName, _ChannConf, Acc) -> Acc
+ end, [], Channels).
+
bin(Bin) when is_binary(Bin) -> Bin;
bin(Str) when is_list(Str) -> list_to_binary(Str);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
-
-str(A) when is_atom(A) ->
- atom_to_list(A);
-str(B) when is_binary(B) ->
- binary_to_list(B);
-str(S) when is_list(S) ->
- S.
diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl
index 004b32787..8cb325e20 100644
--- a/apps/emqx_bridge/src/emqx_bridge_app.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_app.erl
@@ -22,10 +22,12 @@
start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_bridge_sup:start_link(),
ok = emqx_bridge:load_bridges(),
+ ok = emqx_bridge:reload_hook(),
emqx_config_handler:add_handler(emqx_bridge:config_key_path(), emqx_bridge),
{ok, Sup}.
stop(_State) ->
+ ok = emqx_bridge:unload_hook(),
ok.
%% internal functions
\ No newline at end of file
diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl
index 87eb40372..2072d15ec 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl
@@ -1,5 +1,7 @@
-module(emqx_bridge_schema).
+-include_lib("typerefl/include/types.hrl").
+
-export([roots/0, fields/1]).
%%======================================================================================
@@ -8,7 +10,16 @@
roots() -> [bridges].
fields(bridges) ->
- [{mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))}];
+ [ {mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))}
+ , {http, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "http_bridge")))}
+ ];
fields("mqtt_bridge") ->
- emqx_connector_mqtt:fields("config").
+ emqx_connector_mqtt:fields("config");
+
+fields("http_bridge") ->
+ emqx_connector_http:fields(config) ++ http_channels().
+
+http_channels() ->
+ [{egress_channels, hoconsc:mk(hoconsc:map(id,
+ hoconsc:ref(emqx_connector_http, "http_request")))}].
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 0f8c23986..aff6e7255 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -21,6 +21,8 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
+
%% callbacks of behaviour emqx_resource
-export([ on_start/2
, on_stop/2
@@ -38,7 +40,7 @@
-export([ check_ssl_opts/2 ]).
--type connect_timeout() :: non_neg_integer() | infinity.
+-type connect_timeout() :: emqx_schema:duration() | infinity.
-type pool_type() :: random | hash.
-reflect_type([ connect_timeout/0
@@ -50,6 +52,22 @@
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].
+fields("http_request") ->
+ [ {subscribe_local_topic, hoconsc:mk(binary())}
+ , {method, hoconsc:mk(method(), #{default => post})}
+ , {path, hoconsc:mk(binary(), #{default => <<"">>})}
+ , {headers, hoconsc:mk(map(),
+ #{default => #{
+ <<"accept">> => <<"application/json">>,
+ <<"cache-control">> => <<"no-cache">>,
+ <<"connection">> => <<"keep-alive">>,
+ <<"content-type">> => <<"application/json">>,
+ <<"keep-alive">> => <<"timeout=5">>}})
+ }
+ , {body, hoconsc:mk(binary(), #{default => <<"${payload}">>})}
+ , {request_timeout, hoconsc:mk(emqx_schema:duration_ms(), #{default => <<"30s">>})}
+ ];
+
fields(config) ->
[ {base_url, fun base_url/1}
, {connect_timeout, fun connect_timeout/1}
@@ -60,6 +78,13 @@ fields(config) ->
, {enable_pipelining, fun enable_pipelining/1}
] ++ emqx_connector_schema_lib:ssl_fields().
+method() ->
+ hoconsc:union([ typerefl:atom(post)
+ , typerefl:atom(put)
+ , typerefl:atom(get)
+ , typerefl:atom(delete)
+ ]).
+
validations() ->
[ {check_ssl_opts, fun check_ssl_opts/1} ].
@@ -71,16 +96,16 @@ base_url(validator) -> fun(#{query := _Query}) ->
end;
base_url(_) -> undefined.
-connect_timeout(type) -> connect_timeout();
-connect_timeout(default) -> 5000;
+connect_timeout(type) -> emqx_schema:duration_ms();
+connect_timeout(default) -> "5s";
connect_timeout(_) -> undefined.
max_retries(type) -> non_neg_integer();
max_retries(default) -> 5;
max_retries(_) -> undefined.
-retry_interval(type) -> non_neg_integer();
-retry_interval(default) -> 1000;
+retry_interval(type) -> emqx_schema:duration();
+retry_interval(default) -> "1s";
retry_interval(_) -> undefined.
pool_type(type) -> pool_type();
@@ -105,13 +130,14 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
retry_interval := RetryInterval,
pool_type := PoolType,
pool_size := PoolSize} = Config) ->
- logger:info("starting http connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting http connector",
+ connector => InstId, config => Config}),
{Transport, TransportOpts} = case Scheme of
http ->
{tcp, []};
https ->
SSLOpts = emqx_plugin_libs_ssl:save_files_return_opts(
- maps:get(ssl_opts, Config), "connectors", InstId),
+ maps:get(ssl, Config), "connectors", InstId),
{tls, SSLOpts}
end,
NTransportOpts = emqx_misc:ipv6_probe(TransportOpts),
@@ -126,30 +152,51 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
, {transport, Transport}
, {transport_opts, NTransportOpts}],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
- {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts),
- {ok, #{pool_name => PoolName,
- host => Host,
- port => Port,
- base_path => BasePath}}.
+ State = #{
+ pool_name => PoolName,
+ host => Host,
+ port => Port,
+ base_path => BasePath,
+ channels => preproc_channels(InstId, Config)
+ },
+ case ehttpc_sup:start_pool(PoolName, PoolOpts) of
+ {ok, _} -> {ok, State};
+ {error, {already_started, _}} -> {ok, State};
+ {error, Reason} ->
+ {error, Reason}
+ end.
on_stop(InstId, #{pool_name := PoolName}) ->
- logger:info("stopping http connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping http connector",
+ connector => InstId}),
ehttpc_sup:stop_pool(PoolName).
+on_query(InstId, {send_message, ChannelId, Msg}, AfterQuery, #{channels := Channels} = State) ->
+ case maps:find(ChannelId, Channels) of
+ error -> ?SLOG(error, #{msg => "channel not found", channel_id => ChannelId});
+ {ok, ChannConf} ->
+ #{method := Method, path := Path, body := Body, headers := Headers,
+ request_timeout := Timeout} = proc_channel_conf(ChannConf, Msg),
+ on_query(InstId, {Method, {Path, Headers, Body}, Timeout}, AfterQuery, State)
+ end;
on_query(InstId, {Method, Request}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State);
on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
-on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName,
- base_path := BasePath} = State) ->
- logger:debug("http connector ~p received request: ~p, at state: ~p", [InstId, Request, State]),
+on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
+ #{pool_name := PoolName, base_path := BasePath} = State) ->
+ ?SLOG(debug, #{msg => "http connector received request",
+ request => Request, connector => InstId,
+ state => State}),
NRequest = update_path(BasePath, Request),
case Result = ehttpc:request(case KeyOrNum of
undefined -> PoolName;
_ -> {PoolName, KeyOrNum}
end, Method, NRequest, Timeout) of
{error, Reason} ->
- logger:debug("http connector ~p do reqeust failed, sql: ~p, reason: ~p", [InstId, NRequest, Reason]),
+ ?SLOG(error, #{msg => "http connector do reqeust failed",
+ request => NRequest, reason => Reason,
+ connector => InstId}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)
@@ -169,6 +216,52 @@ on_health_check(_InstId, #{host := Host, port := Port} = State) ->
%% Internal functions
%%--------------------------------------------------------------------
+preproc_channels(<<"bridge:", BridgeId/binary>>, Config) ->
+ {BridgeType, BridgeName} = emqx_bridge:parse_bridge_id(BridgeId),
+ maps:fold(fun(ChannName, ChannConf, Acc) ->
+ Acc#{emqx_bridge:channel_id(BridgeType, BridgeName, egress_channels, ChannName) =>
+ preproc_channel_conf(ChannConf)}
+ end, #{}, maps:get(egress_channels, Config, #{})).
+
+preproc_channel_conf(#{
+ method := Method,
+ path := Path,
+ body := Body,
+ headers := Headers} = Conf) ->
+ Conf#{ method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method))
+ , path => emqx_plugin_libs_rule:preproc_tmpl(Path)
+ , body => emqx_plugin_libs_rule:preproc_tmpl(Body)
+ , headers => preproc_headers(Headers)
+ }.
+
+preproc_headers(Headers) ->
+ maps:fold(fun(K, V, Acc) ->
+ Acc#{emqx_plugin_libs_rule:preproc_tmpl(bin(K)) =>
+ emqx_plugin_libs_rule:preproc_tmpl(bin(V))}
+ end, #{}, Headers).
+
+proc_channel_conf(#{
+ method := MethodTks,
+ path := PathTks,
+ body := BodyTks,
+ headers := HeadersTks} = Conf, Msg) ->
+ Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg))
+ , path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg)
+ , body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg)
+ , headers => maps:to_list(proc_headers(HeadersTks, Msg))
+ }.
+
+proc_headers(HeaderTks, Msg) ->
+ maps:fold(fun(K, V, Acc) ->
+ Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) =>
+ emqx_plugin_libs_rule:proc_tmpl(V, Msg)}
+ end, #{}, HeaderTks).
+
+make_method(M) when M == <<"POST">>; M == <<"post">> -> post;
+make_method(M) when M == <<"PUT">>; M == <<"put">> -> put;
+make_method(M) when M == <<"GET">>; M == <<"get">> -> get;
+make_method(M) when M == <<"DELETE">>; M == <<"delete">> -> delete.
+
check_ssl_opts(Conf) ->
check_ssl_opts("base_url", Conf).
@@ -185,3 +278,10 @@ update_path(BasePath, {Path, Headers}) ->
{filename:join(BasePath, Path), Headers};
update_path(BasePath, {Path, Headers, Body}) ->
{filename:join(BasePath, Path), Headers, Body}.
+
+bin(Bin) when is_binary(Bin) ->
+ Bin;
+bin(Str) when is_list(Str) ->
+ list_to_binary(Str);
+bin(Atom) when is_atom(Atom) ->
+ atom_to_binary(Atom, utf8).
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index fadf7f56f..85e42b0f3 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -18,6 +18,7 @@
-include("emqx_connector.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
-export([roots/0, fields/1]).
@@ -53,7 +54,8 @@ on_start(InstId, #{servers := Servers0,
pool_size := PoolSize,
auto_reconnect := AutoReconn,
ssl := SSL} = Config) ->
- logger:info("starting ldap connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting ldap connector",
+ connector => InstId, config => Config}),
Servers = [begin proplists:get_value(host, S) end || S <- Servers0],
SslOpts = case maps:get(enable, SSL) of
true ->
@@ -75,14 +77,20 @@ on_start(InstId, #{servers := Servers0,
{ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) ->
- logger:info("stopping ldap connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping ldap connector",
+ connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
- logger:debug("ldap connector ~p received request: ~p, at state: ~p", [InstId, {Base, Filter, Attributes}, State]),
+ Request = {Base, Filter, Attributes},
+ ?SLOG(debug, #{msg => "ldap connector received request",
+ request => Request, connector => InstId,
+ state => State}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, search, [Base, Filter, Attributes]}, no_handover) of
{error, Reason} ->
- logger:debug("ldap connector ~p do request failed, request: ~p, reason: ~p", [InstId, {Base, Filter, Attributes}, Reason]),
+ ?SLOG(error, #{msg => "ldap connector do request failed",
+ request => Request, connector => InstId,
+ reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 906b57fb3..0cb40adbb 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -18,6 +18,7 @@
-include("emqx_connector.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
-type server() :: emqx_schema:ip_port().
-reflect_type([server/0]).
@@ -93,7 +94,8 @@ on_jsonify(Config) ->
%% ===================================================================
on_start(InstId, Config = #{server := Server,
mongo_type := single}) ->
- logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting mongodb single connector",
+ connector => InstId, config => Config}),
Opts = [{type, single},
{hosts, [emqx_connector_schema_lib:ip_port_to_string(Server)]}
],
@@ -102,7 +104,8 @@ on_start(InstId, Config = #{server := Server,
on_start(InstId, Config = #{servers := Servers,
mongo_type := rs,
replica_set_name := RsName}) ->
- logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting mongodb rs connector",
+ connector => InstId, config => Config}),
Opts = [{type, {rs, RsName}},
{hosts, [emqx_connector_schema_lib:ip_port_to_string(S)
|| S <- Servers]}
@@ -111,7 +114,8 @@ on_start(InstId, Config = #{servers := Servers,
on_start(InstId, Config = #{servers := Servers,
mongo_type := sharded}) ->
- logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting mongodb sharded connector",
+ connector => InstId, config => Config}),
Opts = [{type, sharded},
{hosts, [emqx_connector_schema_lib:ip_port_to_string(S)
|| S <- Servers]}
@@ -119,14 +123,20 @@ on_start(InstId, Config = #{servers := Servers,
do_start(InstId, Opts, Config).
on_stop(InstId, #{poolname := PoolName}) ->
- logger:info("stopping mongodb connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping mongodb connector",
+ connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {Action, Collection, Selector, Docs}, AfterQuery, #{poolname := PoolName} = State) ->
- logger:debug("mongodb connector ~p received request: ~p, at state: ~p", [InstId, {Action, Collection, Selector, Docs}, State]),
+ Request = {Action, Collection, Selector, Docs},
+ ?SLOG(debug, #{msg => "mongodb connector received request",
+ request => Request, connector => InstId,
+ state => State}),
case ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of
{error, Reason} ->
- logger:debug("mongodb connector ~p do sql query failed, request: ~p, reason: ~p", [InstId, {Action, Collection, Selector, Docs}, Reason]),
+ ?SLOG(error, #{msg => "mongodb connector do query failed",
+ request => Request, reason => Reason,
+ connector => InstId}),
emqx_resource:query_failed(AfterQuery),
{error, Reason};
{ok, Cursor} when is_pid(Cursor) ->
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index a03c888d3..a4527984a 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -17,6 +17,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
-behaviour(supervisor).
@@ -88,13 +89,15 @@ drop_bridge(Name) ->
%% ===================================================================
%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called
%% if the bridge received msgs from the remote broker.
-on_message_received(Msg, ChannelName) ->
- emqx:run_hook(ChannelName, [Msg]).
+on_message_received(Msg, ChannId) ->
+ Name = atom_to_binary(ChannId, utf8),
+ emqx:run_hook(<<"$bridges/", Name/binary>>, [Msg]).
%% ===================================================================
on_start(InstId, Conf) ->
- logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]),
- NamePrefix = binary_to_list(InstId),
+ ?SLOG(info, #{msg => "starting mqtt connector",
+ connector => InstId, config => Conf}),
+ "bridge:" ++ NamePrefix = binary_to_list(InstId),
BasicConf = basic_config(Conf),
InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, channels => []}},
InOutConfigs = taged_map_list(ingress_channels, maps:get(ingress_channels, Conf, #{}))
@@ -110,7 +113,8 @@ on_start(InstId, Conf) ->
end, InitRes, InOutConfigs).
on_stop(InstId, #{channels := NameList}) ->
- logger:info("stopping mqtt connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping mqtt connector",
+ connector => InstId}),
lists:foreach(fun(Name) ->
remove_channel(Name)
end, NameList).
@@ -120,9 +124,10 @@ on_stop(InstId, #{channels := NameList}) ->
on_query(_InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix,
baisc_conf := BasicConf}) ->
create_channel(Conf, Prefix, BasicConf);
-on_query(_InstId, {send_to_remote, ChannelName, Msg}, _AfterQuery, _State) ->
- logger:debug("send msg to remote node on channel: ~p, msg: ~p", [ChannelName, Msg]),
- emqx_connector_mqtt_worker:send_to_remote(ChannelName, Msg).
+on_query(_InstId, {send_message, ChannelId, Msg}, _AfterQuery, _State) ->
+ ?SLOG(debug, #{msg => "send msg to remote node", message => Msg,
+ channel_id => ChannelId}),
+ emqx_connector_mqtt_worker:send_to_remote(ChannelId, Msg).
on_health_check(_InstId, #{channels := NameList} = State) ->
Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList],
@@ -134,35 +139,43 @@ on_health_check(_InstId, #{channels := NameList} = State) ->
create_channel({{ingress_channels, Id}, #{subscribe_remote_topic := RemoteT} = Conf},
NamePrefix, BasicConf) ->
LocalT = maps:get(local_topic, Conf, undefined),
- Name = ingress_channel_name(NamePrefix, Id),
- logger:info("creating ingress channel ~p, remote ~s -> local ~s", [Name, RemoteT, LocalT]),
+ ChannId = ingress_channel_id(NamePrefix, Id),
+ ?SLOG(info, #{msg => "creating ingress channel",
+ remote_topic => RemoteT,
+ local_topic => LocalT,
+ channel_id => ChannId}),
do_create_channel(BasicConf#{
- name => Name,
- clientid => clientid(Name),
+ name => ChannId,
+ clientid => clientid(ChannId),
subscriptions => Conf#{
local_topic => LocalT,
- on_message_received => {fun ?MODULE:on_message_received/2, [Name]}
+ on_message_received => {fun ?MODULE:on_message_received/2, [ChannId]}
},
forwards => undefined});
create_channel({{egress_channels, Id}, #{remote_topic := RemoteT} = Conf},
NamePrefix, BasicConf) ->
LocalT = maps:get(subscribe_local_topic, Conf, undefined),
- Name = egress_channel_name(NamePrefix, Id),
- logger:info("creating egress channel ~p, local ~s -> remote ~s", [Name, LocalT, RemoteT]),
+ ChannId = egress_channel_id(NamePrefix, Id),
+ ?SLOG(info, #{msg => "creating egress channel",
+ remote_topic => RemoteT,
+ local_topic => LocalT,
+ channel_id => ChannId}),
do_create_channel(BasicConf#{
- name => Name,
- clientid => clientid(Name),
+ name => ChannId,
+ clientid => clientid(ChannId),
subscriptions => undefined,
forwards => Conf#{subscribe_local_topic => LocalT}}).
-remove_channel(ChannelName) ->
- logger:info("removing channel ~p", [ChannelName]),
- case ?MODULE:drop_bridge(ChannelName) of
+remove_channel(ChannId) ->
+ ?SLOG(info, #{msg => "removing channel",
+ channel_id => ChannId}),
+ case ?MODULE:drop_bridge(ChannId) of
ok -> ok;
{error, not_found} -> ok;
{error, Reason} ->
- logger:error("stop channel ~p failed, error: ~p", [ChannelName, Reason])
+ ?SLOG(error, #{msg => "stop channel failed",
+ channel_id => ChannId, reason => Reason})
end.
do_create_channel(#{name := Name} = Conf) ->
@@ -215,9 +228,9 @@ basic_config(#{
taged_map_list(Tag, Map) ->
[{{Tag, K}, V} || {K, V} <- maps:to_list(Map)].
-ingress_channel_name(Prefix, Id) ->
+ingress_channel_id(Prefix, Id) ->
channel_name("ingress_channels", Prefix, Id).
-egress_channel_name(Prefix, Id) ->
+egress_channel_id(Prefix, Id) ->
channel_name("egress_channels", Prefix, Id).
channel_name(Type, Prefix, Id) ->
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 0cd051512..845c96161 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -17,6 +17,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
%% callbacks of behaviour emqx_resource
-export([ on_start/2
@@ -54,7 +55,8 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL } = Config) ->
- logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting mysql connector",
+ connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of
true ->
[{ssl, [{server_name_indication, disable} |
@@ -73,7 +75,8 @@ on_start(InstId, #{server := {Host, Port},
{ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) ->
- logger:info("stopping mysql connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping mysql connector",
+ connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
@@ -81,10 +84,12 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
- logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]),
+ ?SLOG(debug, #{msg => "mysql connector received sql query",
+ connector => InstId, sql => SQL, state => State}),
case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL, Params, Timeout]}, no_handover) of
{error, Reason} ->
- logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]),
+ ?SLOG(error, #{msg => "mysql connector do sql query failed",
+ connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 31e954b3c..5b0adbeb9 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -17,6 +17,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
-export([roots/0, fields/1]).
@@ -54,7 +55,8 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL } = Config) ->
- logger:info("starting postgresql connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting postgresql connector",
+ connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of
true ->
[{ssl, [{server_name_indication, disable} |
@@ -73,16 +75,20 @@ on_start(InstId, #{server := {Host, Port},
{ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) ->
- logger:info("stopping postgresql connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping postgresql connector",
+ connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, []}, AfterQuery, State);
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
- logger:debug("postgresql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]),
+ ?SLOG(debug, #{msg => "postgresql connector received sql query",
+ connector => InstId, sql => SQL, state => State}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of
{error, Reason} ->
- logger:debug("postgresql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]),
+ ?SLOG(error, #{
+ msg => "postgresql connector do sql query failed",
+ connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 44b036f39..aed06e724 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -18,6 +18,7 @@
-include("emqx_connector.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+-include_lib("emqx/include/logger.hrl").
-type server() :: tuple().
@@ -85,7 +86,8 @@ on_start(InstId, #{redis_type := Type,
pool_size := PoolSize,
auto_reconnect := AutoReconn,
ssl := SSL } = Config) ->
- logger:info("starting redis connector: ~p, config: ~p", [InstId, Config]),
+ ?SLOG(info, #{msg => "starting redis connector",
+ connector => InstId, config => Config}),
Servers = case Type of
single -> [{servers, [maps:get(server, Config)]}];
_ ->[{servers, maps:get(servers, Config)}]
@@ -116,18 +118,21 @@ on_start(InstId, #{redis_type := Type,
{ok, #{poolname => PoolName, type => Type}}.
on_stop(InstId, #{poolname := PoolName}) ->
- logger:info("stopping redis connector: ~p", [InstId]),
+ ?SLOG(info, #{msg => "stopping redis connector",
+ connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
- logger:debug("redis connector ~p received cmd query: ~p, at state: ~p", [InstId, Command, State]),
+ ?SLOG(debug, #{msg => "redis connector received cmd query",
+ connector => InstId, sql => Command, state => State}),
Result = case Type of
cluster -> eredis_cluster:q(PoolName, Command);
_ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
end,
case Result of
{error, Reason} ->
- logger:debug("redis connector ~p do cmd query failed, cmd: ~p, reason: ~p", [InstId, Command, Reason]),
+ ?SLOG(error, #{msg => "redis connector do cmd query failed",
+ connector => InstId, sql => Command, reason => Reason}),
emqx_resource:query_failed(AfterCommand);
_ ->
emqx_resource:query_success(AfterCommand)
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
index 8b0aa5051..853221eec 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
@@ -155,14 +155,18 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
Parent ! {batch_ack, PktId}, ok;
handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
- ?LOG(warning, "publish ~p to remote node falied, reason_code: ~p", [PktId, RC]).
+ ?SLOG(warning, #{msg => "publish to remote node falied",
+ packet_id => PktId, reason_code => RC}).
handle_publish(Msg, undefined) ->
- ?LOG(error, "cannot publish to local broker as 'bridge.mqtt..in' not configured, msg: ~p", [Msg]);
+ ?SLOG(error, #{msg => "cannot publish to local broker as"
+ " ingress_channles' is not configured",
+ message => Msg});
handle_publish(Msg, #{on_message_received := {OnMsgRcvdFunc, Args}} = Vars) ->
- ?LOG(debug, "publish to local broker, msg: ~p, vars: ~p", [Msg, Vars]),
+ ?SLOG(debug, #{msg => "publish to local broker",
+ message => Msg, vars => Vars}),
emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1),
- _ = erlang:apply(OnMsgRcvdFunc, [Msg, Args]),
+ _ = erlang:apply(OnMsgRcvdFunc, [Msg | Args]),
case maps:get(local_topic, Vars, undefined) of
undefined -> ok;
_Topic ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index a00b76b97..b0aaeb8b6 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -23,19 +23,21 @@
-export([ roots/0
, fields/1]).
+-import(emqx_schema, [mk_duration/2]).
+
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, "config")}}].
fields("config") ->
[ {server, hoconsc:mk(emqx_schema:ip_port(), #{default => "127.0.0.1:1883"})}
- , {reconnect_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})}
+ , {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})}
, {proto_ver, fun proto_ver/1}
, {bridge_mode, hoconsc:mk(boolean(), #{default => true})}
, {username, hoconsc:mk(string())}
, {password, hoconsc:mk(string())}
, {clean_start, hoconsc:mk(boolean(), #{default => true})}
- , {keepalive, hoconsc:mk(integer(), #{default => 300})}
- , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})}
+ , {keepalive, mk_duration("keepalive", #{default => "300s"})}
+ , {retry_interval, mk_duration("retry interval", #{default => "30s"})}
, {max_inflight, hoconsc:mk(integer(), #{default => 32})}
, {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))}
, {ingress_channels, hoconsc:mk(hoconsc:map(id, hoconsc:ref(?MODULE, "ingress_channels")), #{default => []})}
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index c98efd322..990d15ef5 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -63,6 +63,7 @@
-behaviour(gen_statem).
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("emqx/include/logger.hrl").
%% APIs
-export([ start_link/1
@@ -189,7 +190,8 @@ callback_mode() -> [state_functions].
%% @doc Config should be a map().
init(#{name := Name} = ConnectOpts) ->
- ?LOG(debug, "starting bridge worker for ~p", [Name]),
+ ?SLOG(debug, #{msg => "starting bridge worker",
+ name => Name}),
erlang:process_flag(trap_exit, true),
Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
State = init_state(ConnectOpts),
@@ -335,8 +337,9 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
NewQ = replayq:append(Q, [Msg]),
{keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
common(StateName, Type, Content, #{name := Name} = State) ->
- ?LOG(notice, "Bridge ~p discarded ~p type event at state ~p:~p",
- [Name, Type, StateName, Content]),
+ ?SLOG(notice, #{msg => "Bridge discarded event",
+ name => Name, type => Type, state_name => StateName,
+ content => Content}),
{keep_state, State}.
do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards},
@@ -352,8 +355,8 @@ do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards},
{ok, State#{connection => Conn}};
{error, Reason} ->
ConnectOpts1 = obfuscate(ConnectOpts),
- ?LOG(error, "Failed to connect \n"
- "config=~p\nreason:~p", [ConnectOpts1, Reason]),
+ ?SLOG(error, #{msg => "Failed to connect",
+ config => ConnectOpts1, reason => Reason}),
{error, Reason, State}
end.
@@ -399,7 +402,9 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
%% Assert non-empty batch because we have a is_empty check earlier.
do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Batch) ->
- ?LOG(error, "cannot forward messages to remote broker as 'bridge.mqtt..in' not configured, msg: ~p", [Batch]);
+ ?SLOG(error, #{msg => "cannot forward messages to remote broker"
+ " as egress_channel is not configured",
+ messages => Batch});
do_send(#{inflight := Inflight,
connection := Connection,
mountpoint := Mountpoint,
@@ -409,14 +414,16 @@ do_send(#{inflight := Inflight,
emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
end,
- ?LOG(debug, "publish to remote broker, msg: ~p, vars: ~p", [Batch, Vars]),
+ ?SLOG(debug, #{msg => "publish to remote broker",
+ message => Batch, vars => Vars}),
case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(M) || M <- Batch]) of
{ok, Refs} ->
{ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef,
send_ack_ref => map_set(Refs),
batch => Batch}]}};
{error, Reason} ->
- ?LOG(info, "mqtt_bridge_produce_failed ~p", [Reason]),
+ ?SLOG(info, #{msg => "mqtt_bridge_produce_failed",
+ reason => Reason}),
{error, State}
end.
@@ -436,7 +443,8 @@ handle_batch_ack(#{inflight := Inflight0, replayq := Q} = State, Ref) ->
State#{inflight := Inflight}.
do_ack([], Ref) ->
- ?LOG(debug, "stale_batch_ack_reference ~p", [Ref]),
+ ?SLOG(debug, #{msg => "stale_batch_ack_reference",
+ ref => Ref}),
[];
do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) ->
case maps:is_key(Ref, Refs) of
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
index b477bd779..5af983b4d 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl
@@ -139,7 +139,10 @@ update_pwd(Username, Fun) ->
-spec(lookup_user(binary()) -> [mqtt_admin()]).
-lookup_user(Username) when is_binary(Username) -> mnesia:dirty_read(mqtt_admin, Username).
+lookup_user(Username) when is_binary(Username) ->
+ Fun = fun() -> mnesia:read(mqtt_admin, Username) end,
+ {atomic, User} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun),
+ User.
-spec(all_users() -> [#mqtt_admin{}]).
all_users() -> ets:tab2list(mqtt_admin).
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
index 8b0576342..0e2adf7c3 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
@@ -162,7 +162,8 @@ flush({Connection, Route, Subscription}, {Received0, Sent0, Dropped0}) ->
diff(Sent, Sent0),
diff(Dropped, Dropped0)},
Ts = get_local_time(),
- _ = mnesia:dirty_write(emqx_collect, #mqtt_collect{timestamp = Ts, collect = Collect}),
+ ekka_mnesia:transaction(ekka_mnesia:local_content_shard(),
+ fun mnesia:write/1, [#mqtt_collect{timestamp = Ts, collect = Collect}]),
{Received, Sent, Dropped}.
avg(Items) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 9033a9b40..75b3ab201 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -6,6 +6,11 @@
%% API
-export([spec/1, spec/2]).
-export([translate_req/2]).
+-export([namespace/0, fields/1]).
+-export([error_codes/1, error_codes/2]).
+-define(MAX_ROW_LIMIT, 100).
+
+%% API
-ifdef(TEST).
-compile(export_all).
@@ -22,7 +27,8 @@
-define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}).
-define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
--define(TO_COMPONENTS(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])).
+-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])).
+-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])).
%% @equiv spec(Module, #{check_schema => false})
-spec(spec(module()) ->
@@ -54,7 +60,6 @@ spec(Module, Options) ->
end, {[], []}, Paths),
{ApiSpec, components(lists:usort(AllRefs))}.
-
-spec(translate_req(#{binding => list(), query_string => list(), body => map()},
#{module => module(), path => string(), method => atom()}) ->
{ok, #{binding => list(), query_string => list(), body => map()}}|
@@ -64,7 +69,7 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) ->
try
Params = maps:get(parameters, Spec, []),
Body = maps:get(requestBody, Spec, []),
- {Bindings, QueryStr} = check_parameters(Request, Params),
+ {Bindings, QueryStr} = check_parameters(Request, Params, Module),
NewBody = check_requestBody(Request, Body, Module, hoconsc:is_schema(Body)),
{ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}}
catch throw:Error ->
@@ -73,6 +78,30 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) ->
{400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~s : ~p", [Key, Reason]))}
end.
+namespace() -> "public".
+
+fields(page) ->
+ Desc = <<"Page number of the results to fetch.">>,
+ Meta = #{in => query, desc => Desc, default => 1, example => 1},
+ [{page, hoconsc:mk(integer(), Meta)}];
+fields(limit) ->
+ Desc = iolist_to_binary([<<"Results per page(max ">>,
+ integer_to_binary(?MAX_ROW_LIMIT), <<")">>]),
+ Meta = #{in => query, desc => Desc, default => ?MAX_ROW_LIMIT, example => 50},
+ [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}].
+
+error_codes(Codes) ->
+ error_codes(Codes, <<"Error code to troubleshoot problems.">>).
+
+error_codes(Codes = [_ | _], MsgExample) ->
+ [
+ {code, hoconsc:mk(hoconsc:enum(Codes))},
+ {message, hoconsc:mk(string(), #{
+ desc => <<"Details description of the error.">>,
+ example => MsgExample
+ })}
+ ].
+
support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER;
support_check_schema(#{check_schema := Func})when is_function(Func, 2) -> #{filter => Func};
support_check_schema(_) -> #{filter => undefined}.
@@ -93,23 +122,28 @@ parse_spec_ref(Module, Path) ->
maps:without([operationId], Schema)),
{maps:get(operationId, Schema), Specs, Refs}.
-check_parameters(Request, Spec) ->
+check_parameters(Request, Spec, Module) ->
#{bindings := Bindings, query_string := QueryStr} = Request,
BindingsBin = maps:fold(fun(Key, Value, Acc) -> Acc#{atom_to_binary(Key) => Value} end, #{}, Bindings),
- check_parameter(Spec, BindingsBin, QueryStr, #{}, #{}).
+ check_parameter(Spec, BindingsBin, QueryStr, Module, #{}, #{}).
-check_parameter([], _Bindings, _QueryStr, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr};
-check_parameter([{Name, Type} | Spec], Bindings, QueryStr, BindingsAcc, QueryStrAcc) ->
+check_parameter([?REF(Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
+ check_parameter([?R_REF(LocalMod, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
+check_parameter([?R_REF(Module, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
+ Params = apply(Module, fields, [Fields]),
+ check_parameter(Params ++ Spec, Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
+check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr};
+check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, QueryStrAcc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
case hocon_schema:field_schema(Type, in) of
path ->
NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true, override_env => false}),
NewBindingsAcc = maps:merge(BindingsAcc, NewBindings),
- check_parameter(Spec, Bindings, QueryStr, NewBindingsAcc, QueryStrAcc);
+ check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc);
query ->
NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}),
NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
- check_parameter(Spec, Bindings, QueryStr, BindingsAcc, NewQueryStrAcc)
+ check_parameter(Spec, Bindings, QueryStr, Module, BindingsAcc, NewQueryStrAcc)
end.
check_requestBody(#{body := Body}, Schema, Module, true) ->
@@ -154,19 +188,28 @@ to_spec(Meta, Params, RequestBody, Responses) ->
parameters(Params, Module) ->
{SpecList, AllRefs} =
- lists:foldl(fun({Name, Type}, {Acc, RefsAcc}) ->
- In = hocon_schema:field_schema(Type, in),
- In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}),
- Nullable = hocon_schema:field_schema(Type, nullable),
- Default = hocon_schema:field_schema(Type, default),
- HoconType = hocon_schema:field_schema(Type, type),
- Meta = init_meta(Nullable, Default),
- {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module),
- Spec0 = init_prop([required | ?DEFAULT_FIELDS],
- #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type),
- Spec1 = trans_required(Spec0, Nullable, In),
- Spec2 = trans_desc(Spec1, Type),
- {[Spec2 | Acc], Refs ++ RefsAcc}
+ lists:foldl(fun(Param, {Acc, RefsAcc}) ->
+ case Param of
+ ?REF(StructName) ->
+ {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} |Acc],
+ [{Module, StructName, parameter}|RefsAcc]};
+ ?R_REF(RModule, StructName) ->
+ {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} |Acc],
+ [{RModule, StructName, parameter}|RefsAcc]};
+ {Name, Type} ->
+ In = hocon_schema:field_schema(Type, in),
+ In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}),
+ Nullable = hocon_schema:field_schema(Type, nullable),
+ Default = hocon_schema:field_schema(Type, default),
+ HoconType = hocon_schema:field_schema(Type, type),
+ Meta = init_meta(Nullable, Default),
+ {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module),
+ Spec0 = init_prop([required | ?DEFAULT_FIELDS],
+ #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type),
+ Spec1 = trans_required(Spec0, Nullable, In),
+ Spec2 = trans_desc(Spec1, Type),
+ {[Spec2 | Acc], Refs ++ RefsAcc}
+ end
end, {[], []}, Params),
{lists:reverse(SpecList), AllRefs}.
@@ -196,7 +239,7 @@ trans_required(Spec, _, _) -> Spec.
trans_desc(Spec, Hocon) ->
case hocon_schema:field_schema(Hocon, desc) of
undefined -> Spec;
- Desc -> Spec#{description => Desc}
+ Desc -> Spec#{description => to_bin(Desc)}
end.
requestBody([], _Module) -> {[], []};
@@ -248,6 +291,13 @@ components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) ->
Namespace = namespace(Module),
{Object, SubRefs} = parse_object(Props, Module),
NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object},
+ components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc);
+%% parameters in ref only have one value, not array
+components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) ->
+ Props = apply(Module, fields, [Field]),
+ {[Param], SubRefs} = parameters(Props, Module),
+ Namespace = namespace(Module),
+ NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param},
components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc).
namespace(Module) ->
@@ -257,10 +307,10 @@ namespace(Module) ->
end.
hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) ->
- {#{<<"$ref">> => ?TO_COMPONENTS(Module, StructName)},
+ {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)},
[{Module, StructName}]};
hocon_schema_to_spec(?REF(StructName), LocalModule) ->
- {#{<<"$ref">> => ?TO_COMPONENTS(LocalModule, StructName)},
+ {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)},
[{LocalModule, StructName}]};
hocon_schema_to_spec(Type, _LocalModule) when ?IS_TYPEREFL(Type) ->
{typename_to_spec(typerefl:name(Type)), []};
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl
index 2acf00f13..c1ca15cb3 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl
@@ -103,7 +103,8 @@ do_sign(Username, Password) ->
},
Signed = jose_jwt:sign(JWK, JWS, JWT),
{_, Token} = jose_jws:compact(Signed),
- ok = ekka_mnesia:dirty_write(format(Token, Username, ExpTime)),
+ JWTRec = format(Token, Username, ExpTime),
+ ekka_mnesia:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [JWTRec]),
{ok, Token}.
do_verify(Token)->
@@ -111,8 +112,9 @@ do_verify(Token)->
{ok, JWT = #mqtt_admin_jwt{exptime = ExpTime}} ->
case ExpTime > erlang:system_time(millisecond) of
true ->
- ekka_mnesia:dirty_write(JWT#mqtt_admin_jwt{exptime = jwt_expiration_time()}),
- ok;
+ NewJWT = JWT#mqtt_admin_jwt{exptime = jwt_expiration_time()},
+ {atomic, Res} = ekka_mnesia:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [NewJWT]),
+ Res;
_ ->
{error, token_timeout}
end;
@@ -132,14 +134,18 @@ do_destroy_by_username(Username) ->
%% jwt internal util function
-spec(lookup(Token :: binary()) -> {ok, #mqtt_admin_jwt{}} | {error, not_found}).
lookup(Token) ->
- case mnesia:dirty_read(?TAB, Token) of
- [JWT] -> {ok, JWT};
- [] -> {error, not_found}
+ Fun = fun() -> mnesia:read(?TAB, Token) end,
+ case ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun) of
+ {atomic, [JWT]} -> {ok, JWT};
+ {atomic, []} -> {error, not_found}
end.
lookup_by_username(Username) ->
Spec = [{{mqtt_admin_jwt, '_', Username, '_'}, [], ['$_']}],
- mnesia:dirty_select(?TAB, Spec).
+ Fun = fun() -> mnesia:select(?TAB, Spec) end,
+ {atomic, List} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun),
+ List.
+
jwk(Username, Password, Salt) ->
Key = erlang:md5(<>),
@@ -187,7 +193,8 @@ handle_info(clean_jwt, State) ->
timer_clean(self()),
Now = erlang:system_time(millisecond),
Spec = [{{mqtt_admin_jwt, '_', '_', '$1'}, [{'<', '$1', Now}], ['$_']}],
- JWTList = mnesia:dirty_select(?TAB, Spec),
+ {atomic, JWTList} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD,
+ fun() -> mnesia:select(?TAB, Spec) end),
destroy(JWTList),
{noreply, State};
handle_info(_Info, State) ->
diff --git a/apps/emqx_dashboard/src/emqx_swagger_util.erl b/apps/emqx_dashboard/src/emqx_swagger_util.erl
deleted file mode 100644
index e2f279941..000000000
--- a/apps/emqx_dashboard/src/emqx_swagger_util.erl
+++ /dev/null
@@ -1,13 +0,0 @@
-%%%-------------------------------------------------------------------
-%%% @author zhongwen
-%%% @copyright (C) 2021,
-%%% @doc
-%%%
-%%% @end
-%%% Created : 22. 9月 2021 13:38
-%%%-------------------------------------------------------------------
--module(emqx_swagger_util).
--author("zhongwen").
-
-%% API
--export([]).
diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl
index a5c458ffa..cea0a915d 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl
@@ -3,10 +3,10 @@
-behaviour(hocon_schema).
%% API
--export([paths/0, api_spec/0, schema/1]).
--export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1]).
+-export([paths/0, api_spec/0, schema/1, fields/1]).
+-export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]).
-export([t_require/1, t_nullable/1, t_method/1, t_api_spec/1]).
--export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1]).
+-export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1, t_ref_trans/1]).
-export([t_in_path_trans_error/1, t_in_query_trans_error/1, t_in_mix_trans_error/1]).
-export([all/0, suite/0, groups/0]).
@@ -20,9 +20,9 @@
all() -> [{group, spec}, {group, validation}].
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
- {spec, [parallel], [t_api_spec, t_in_path, t_in_query, t_in_mix,
- t_without_in, t_require, t_nullable, t_method]},
- {validation, [parallel], [t_in_path_trans, t_in_query_trans, t_in_mix_trans,
+ {spec, [parallel], [t_api_spec, t_in_path, t_ref, t_in_query, t_in_mix,
+ t_without_in, t_require, t_nullable, t_method, t_public_ref]},
+ {validation, [parallel], [t_in_path_trans, t_ref_trans, t_in_query_trans, t_in_mix_trans,
t_in_path_trans_error, t_in_query_trans_error, t_in_mix_trans_error]}
].
@@ -44,6 +44,41 @@ t_in_query(_Config) ->
validate("/test/in/query", Expect),
ok.
+t_ref(_Config) ->
+ LocalPath = "/test/in/ref/local",
+ Path = "/test/in/ref",
+ Expect = [#{<<"$ref">> => <<"#/components/parameters/emqx_swagger_parameter_SUITE.page">>}],
+ {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path),
+ {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, LocalPath),
+ ?assertEqual(test, OperationId),
+ Params = maps:get(parameters, maps:get(post, Spec)),
+ ?assertEqual(Expect, Params),
+ ?assertEqual([{?MODULE, page, parameter}], Refs),
+ ok.
+
+t_public_ref(_Config) ->
+ Path = "/test/in/ref/public",
+ Expect = [
+ #{<<"$ref">> => <<"#/components/parameters/public.page">>},
+ #{<<"$ref">> => <<"#/components/parameters/public.limit">>}
+ ],
+ {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path),
+ ?assertEqual(test, OperationId),
+ Params = maps:get(parameters, maps:get(post, Spec)),
+ ?assertEqual(Expect, Params),
+ ?assertEqual([
+ {emqx_dashboard_swagger, limit, parameter},
+ {emqx_dashboard_swagger, page, parameter}
+ ], Refs),
+ ExpectRefs = [
+ #{<<"public.limit">> => #{description => <<"Results per page(max 100)">>, example => 50,in => query,name => limit,
+ schema => #{default => 100,example => 1,maximum => 100, minimum => 1,type => integer}}},
+ #{<<"public.page">> => #{description => <<"Page number of the results to fetch.">>,
+ example => 1,in => query,name => page,
+ schema => #{default => 1,example => 100,type => integer}}}],
+ ?assertEqual(ExpectRefs, emqx_dashboard_swagger:components(Refs)),
+ ok.
+
t_in_mix(_Config) ->
Expect =
[#{description => <<"Indicates which sorts of issues to return">>,
@@ -115,6 +150,18 @@ t_in_query_trans(_Config) ->
?assertEqual(Expect, trans_parameters(Path, #{}, #{<<"per_page">> => 100})),
ok.
+t_ref_trans(_Config) ->
+ LocalPath = "/test/in/ref/local",
+ Path = "/test/in/ref",
+ Expect = {ok, #{bindings => #{},body => #{},
+ query_string => #{<<"per_page">> => 100}}},
+ ?assertEqual(Expect, trans_parameters(Path, #{}, #{<<"per_page">> => 100})),
+ ?assertEqual(Expect, trans_parameters(LocalPath, #{}, #{<<"per_page">> => 100})),
+ {400,'BAD_REQUEST', Reason} = trans_parameters(Path, #{}, #{<<"per_page">> => 1010}),
+ ?assertNotEqual(nomatch, binary:match(Reason, [<<"per_page">>])),
+ {400,'BAD_REQUEST', Reason} = trans_parameters(LocalPath, #{}, #{<<"per_page">> => 1010}),
+ ok.
+
t_in_mix_trans(_Config) ->
Path = "/test/in/mix/:state",
Bindings = #{
@@ -186,7 +233,7 @@ trans_parameters(Path, Bindings, QueryStr) ->
api_spec() -> emqx_dashboard_swagger:spec(?MODULE).
-paths() -> ["/test/in/:filter", "/test/in/query", "/test/in/mix/:state",
+paths() -> ["/test/in/:filter", "/test/in/query", "/test/in/mix/:state", "/test/in/ref",
"/required/false", "/nullable/false", "/nullable/true", "/method/ok"].
schema("/test/in/:filter") ->
@@ -213,6 +260,33 @@ schema("/test/in/query") ->
responses => #{200 => <<"ok">>}
}
};
+schema("/test/in/ref/local") ->
+ #{
+ operationId => test,
+ post => #{
+ parameters => [hoconsc:ref(page)],
+ responses => #{200 => <<"ok">>}
+ }
+ };
+schema("/test/in/ref") ->
+ #{
+ operationId => test,
+ post => #{
+ parameters => [hoconsc:ref(?MODULE, page)],
+ responses => #{200 => <<"ok">>}
+ }
+ };
+schema("/test/in/ref/public") ->
+ #{
+ operationId => test,
+ post => #{
+ parameters => [
+ hoconsc:ref(emqx_dashboard_swagger, page),
+ hoconsc:ref(emqx_dashboard_swagger, limit)
+ ],
+ responses => #{200 => <<"ok">>}
+ }
+ };
schema("/test/in/mix/:state") ->
#{
operationId => test,
@@ -257,6 +331,13 @@ schema("/method/ok") ->
#{operationId => test}, ?METHODS);
schema("/method/error") ->
#{operationId => test, bar => #{200 => <<"ok">>}}.
+
+fields(page) ->
+ [
+ {per_page,
+ mk(range(1, 100),
+ #{in => query, desc => <<"results per page (max 100)">>, example => 1})}
+ ].
to_schema(Params) ->
#{
operationId => test,
diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
index 49dca926f..84cbfe5fb 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
@@ -101,7 +101,7 @@ t_remote_ref(_Config) ->
{<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}},
#{<<"emqx_swagger_remote_schema.ref3">> => #{<<"properties">> => [
{<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}},
- {<<"version">>, #{description => "a good version", example => <<"1.0.0">>,type => string}}],
+ {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>,type => string}}],
<<"type">> => object}}],
?assertEqual(ExpectComponents, Components),
ok.
@@ -116,7 +116,7 @@ t_nest_ref(_Config) ->
ExpectComponents = lists:sort([
#{<<"emqx_swagger_requestBody_SUITE.nest_ref">> => #{<<"properties">> => [
{<<"env">>, #{enum => [test,dev,prod],type => string}},
- {<<"another_ref">>, #{description => "nest ref", <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
+ {<<"another_ref">>, #{description => <<"nest ref">>, <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
<<"type">> => object}},
#{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [
{<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}},
diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
index c2140d2c0..fd6920549 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
@@ -12,7 +12,7 @@
-export([all/0, suite/0, groups/0]).
-export([paths/0, api_spec/0, schema/1, fields/1]).
--export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1,
+-export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1,
t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1,
t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]).
@@ -21,7 +21,7 @@ all() -> [{group, spec}].
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
{spec, [parallel], [
- t_api_spec, t_simple_binary, t_object, t_nest_object,
+ t_api_spec, t_simple_binary, t_object, t_nest_object, t_error,
t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function,
t_local_ref, t_remote_ref, t_bad_ref, t_none_ref,
t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]}
@@ -48,6 +48,33 @@ t_object(_config) ->
validate(Path, Object, ExpectRefs),
ok.
+t_error(_Config) ->
+ Path = "/error",
+ Error400 = #{<<"content">> =>
+ #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object,
+ <<"properties">> =>
+ [
+ {<<"code">>, #{enum => ['Bad1','Bad2'], type => string}},
+ {<<"message">>, #{description => <<"Details description of the error.">>,
+ example => <<"Bad request desc">>, type => string}}]
+ }}}},
+ Error404 = #{<<"content">> =>
+ #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object,
+ <<"properties">> =>
+ [
+ {<<"code">>, #{enum => ['Not-Found'], type => string}},
+ {<<"message">>, #{description => <<"Details description of the error.">>,
+ example => <<"Error code to troubleshoot problems.">>, type => string}}]
+ }}}},
+ {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path),
+ ?assertEqual(test, OperationId),
+ Response = maps:get(responses, maps:get(get, Spec)),
+ ?assertEqual(Error400, maps:get(<<"400">>, Response)),
+ ?assertEqual(Error404, maps:get(<<"404">>, Response)),
+ ?assertEqual(#{}, maps:without([<<"400">>, <<"404">>], Response)),
+ ?assertEqual([], Refs),
+ ok.
+
t_nest_object(_Config) ->
Path = "/nest/object",
Object =
@@ -175,7 +202,7 @@ t_hocon_schema_function(_Config) ->
#{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object,
<<"properties">> => [
{<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}},
- {<<"version">>, #{description => "a good version", example => <<"1.0.0">>, type => string}}]
+ {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}]
}},
#{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>],
<<"properties">> => [{<<"listeners">>, #{items =>
@@ -255,7 +282,15 @@ schema("/ref/array/with/key") ->
schema("/ref/array/without/key") ->
to_schema(mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{}));
schema("/ref/hocon/schema/function") ->
- to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "root"), #{})).
+ to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "root"), #{}));
+schema("/error") ->
+ #{
+ operationId => test,
+ get => #{responses => #{
+ 400 => emqx_dashboard_swagger:error_codes(['Bad1', 'Bad2'], <<"Bad request desc">>),
+ 404 => emqx_dashboard_swagger:error_codes(['Not-Found'])
+ }}
+ }.
validate(Path, ExpectObject, ExpectRefs) ->
{OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path),
diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf
index 2ce48bf75..4695d41ba 100644
--- a/apps/emqx_gateway/etc/emqx_gateway.conf
+++ b/apps/emqx_gateway/etc/emqx_gateway.conf
@@ -5,296 +5,345 @@
## TODO: These configuration options are temporary example here.
## In the final version, it will be commented out.
-gateway.stomp {
-
- ## How long time the connection will be disconnected if the
- ## connection is established but no bytes received
- idle_timeout = 30s
-
- ## To control whether write statistics data into ETS table
- ## for dashbord to read.
- enable_stats = true
-
- ## When publishing or subscribing, prefix all topics with a mountpoint string.
- mountpoint = ""
-
- frame {
- max_headers = 10
- max_headers_length = 1024
- max_body_length = 8192
- }
-
- clientinfo_override {
- username = "${Packet.headers.login}"
- password = "${Packet.headers.passcode}"
- }
-
- authentication: [
- # {
- # name = "authenticator1"
- # type = "password-based:built-in-database"
- # user_id_type = clientid
- # }
- ]
-
- listeners.tcp.default {
- bind = 61613
- acceptors = 16
- max_connections = 1024000
- max_conn_rate = 1000
-
- access_rules = [
- "allow all"
- ]
-
- ## TCP options
- ## See ${example_common_tcp_options} for more information
- tcp.active_n = 100
- tcp.backlog = 1024
- tcp.buffer = 4KB
- }
-
- listeners.ssl.default {
- bind = 61614
- acceptors = 16
- max_connections = 1024000
- max_conn_rate = 1000
-
- ## TCP options
- ## See ${example_common_tcp_options} for more information
- tcp.active_n = 100
- tcp.backlog = 1024
- tcp.buffer = 4KB
-
- ## SSL options
- ## See ${example_common_ssl_options} for more information
- ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
- ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
- ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
- ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
- #ssl.verify = verify_none
- #ssl.fail_if_no_peer_cert = false
- #ssl.server_name_indication = disable
- #ssl.secure_renegotiate = false
- #ssl.reuse_sessions = false
- #ssl.honor_cipher_order = false
- #ssl.handshake_timeout = 15s
- #ssl.depth = 10
- #ssl.password = foo
- #ssl.dhfile = path-to-your-file
- }
-}
-
-gateway.coap {
-
- ## How long time the connection will be disconnected if the
- ## connection is established but no bytes received
- idle_timeout = 30s
-
- ## To control whether write statistics data into ETS table
- ## for dashbord to read.
- enable_stats = true
-
- ## When publishing or subscribing, prefix all topics with a mountpoint string.
- mountpoint = ""
-
- notify_type = qos
-
- ## if true, you need to establish a connection before use
- connection_required = false
- subscribe_qos = qos0
- publish_qos = qos1
-
- listeners.udp.default {
- bind = 5683
- acceptors = 4
- max_connections = 102400
- max_conn_rate = 1000
-
- ## UDP Options
- ## See ${example_common_udp_options} for more information
- udp.active_n = 100
- udp.buffer = 16KB
- }
- listeners.dtls.default {
- bind = 5684
- acceptors = 4
- max_connections = 102400
- max_conn_rate = 1000
-
- ## UDP Options
- ## See ${example_common_udp_options} for more information
- udp.active_n = 100
- udp.buffer = 16KB
-
- ## DTLS Options
- ## See #{example_common_dtls_options} for more information
- dtls.versions = ["dtlsv1.2", "dtlsv1"]
- dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
- dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
- dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
- }
-}
-
-gateway.mqttsn {
-
- ## How long time the connection will be disconnected if the
- ## connection is established but no bytes received
- idle_timeout = 30s
-
- ## To control whether write statistics data into ETS table
- ## for dashbord to read.
- enable_stats = true
-
- ## When publishing or subscribing, prefix all topics with a mountpoint string.
- mountpoint = ""
-
- ## The MQTT-SN Gateway ID in ADVERTISE message.
- gateway_id = 1
-
- ## Enable broadcast this gateway to WLAN
- broadcast = true
-
- ## To control whether accept and process the received
- ## publish message with qos=-1.
- enable_qos3 = true
-
- ## The pre-defined topic name corresponding to the pre-defined topic
- ## id of N.
- ## Note that the pre-defined topic id of 0 is reserved.
- predefined = [
- { id = 1
- topic = "/predefined/topic/name/hello"
- },
- { id = 2
- topic = "/predefined/topic/name/nice"
- }
- ]
-
- ### ClientInfo override
- clientinfo_override {
- username = "mqtt_sn_user"
- password = "abc"
- }
-
- listeners.udp.default {
- bind = 1884
- max_connections = 10240000
- max_conn_rate = 1000
- }
-
- listeners.dtls.default {
- bind = 1885
- acceptors = 4
- max_connections = 102400
- max_conn_rate = 1000
-
- ## UDP Options
- ## See ${example_common_udp_options} for more information
- udp.active_n = 100
- udp.buffer = 16KB
-
- ## DTLS Options
- ## See #{example_common_dtls_options} for more information
- dtls.versions = ["dtlsv1.2", "dtlsv1"]
- dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
- dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
- dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
- }
-
-}
-
-gateway.lwm2m {
-
- ## How long time the connection will be disconnected if the
- ## connection is established but no bytes received
- idle_timeout = 30s
-
- ## To control whether write statistics data into ETS table
- ## for dashbord to read.
- enable_stats = true
-
- ## When publishing or subscribing, prefix all topics with a mountpoint string.
- mountpoint = "lwm2m/%u"
-
- xml_dir = "{{ platform_etc_dir }}/lwm2m_xml"
-
- lifetime_min = 1s
- lifetime_max = 86400s
- qmode_time_windonw = 22
- auto_observe = false
-
- ## always | contains_object_list
- update_msg_publish_condition = contains_object_list
-
-
- translators {
- command {
- topic = "/dn/#"
- qos = 0
- }
-
- response {
- topic = "/up/resp"
- qos = 0
- }
-
- notify {
- topic = "/up/notify"
- qos = 0
- }
-
- register {
- topic = "/up/resp"
- qos = 0
- }
-
- update {
- topic = "/up/resp"
- qos = 0
- }
- }
-
- listeners.udp.default {
- bind = 5783
- }
-}
-
-gateway.exproto {
-
- ## How long time the connection will be disconnected if the
- ## connection is established but no bytes received
- idle_timeout = 30s
-
- ## To control whether write statistics data into ETS table
- ## for dashbord to read.
- enable_stats = true
-
- ## When publishing or subscribing, prefix all topics with a mountpoint string.
- mountpoint = ""
-
- ## The gRPC server to accept requests
- server {
- bind = 9100
- #ssl.keyfile:
- #ssl.certfile:
- #ssl.cacertfile:
- }
-
- handler {
- address = "http://127.0.0.1:9001"
- #ssl.keyfile:
- #ssl.certfile:
- #ssl.cacertfile:
- }
-
- listeners.tcp.default {
- bind = 7993
- acceptors = 8
- max_connections = 10240
- max_conn_rate = 1000
- }
- #listeners.ssl.default: {}
- #listeners.udp.default: {}
- #listeners.dtls.default: {}
-}
+#gateway.stomp {
+#
+# ## How long time the connection will be disconnected if the
+# ## connection is established but no bytes received
+# idle_timeout = 30s
+#
+# ## To control whether write statistics data into ETS table
+# ## for dashbord to read.
+# enable_stats = true
+#
+# ## When publishing or subscribing, prefix all topics with a mountpoint string.
+# mountpoint = ""
+#
+# frame {
+# max_headers = 10
+# max_headers_length = 1024
+# max_body_length = 8192
+# }
+#
+# clientinfo_override {
+# username = "${Packet.headers.login}"
+# password = "${Packet.headers.passcode}"
+# }
+#
+# authentication: {
+# mechanism = password-based
+# backend = built-in-database
+# user_id_type = clientid
+# }
+#
+# listeners.tcp.default {
+# bind = 61613
+# acceptors = 16
+# max_connections = 1024000
+# max_conn_rate = 1000
+#
+# access_rules = [
+# "allow all"
+# ]
+#
+# authentication: {
+# mechanism = password-based
+# backend = built-in-database
+# user_id_type = username
+# }
+#
+# ## TCP options
+# ## See ${example_common_tcp_options} for more information
+# tcp.active_n = 100
+# tcp.backlog = 1024
+# tcp.buffer = 4KB
+# }
+#
+# listeners.ssl.default {
+# bind = 61614
+# acceptors = 16
+# max_connections = 1024000
+# max_conn_rate = 1000
+#
+# ## TCP options
+# ## See ${example_common_tcp_options} for more information
+# tcp.active_n = 100
+# tcp.backlog = 1024
+# tcp.buffer = 4KB
+#
+# ## SSL options
+# ## See ${example_common_ssl_options} for more information
+# ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+# ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
+# ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
+# ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
+# #ssl.verify = verify_none
+# #ssl.fail_if_no_peer_cert = false
+# #ssl.server_name_indication = disable
+# #ssl.secure_renegotiate = false
+# #ssl.reuse_sessions = false
+# #ssl.honor_cipher_order = false
+# #ssl.handshake_timeout = 15s
+# #ssl.depth = 10
+# #ssl.password = foo
+# #ssl.dhfile = path-to-your-file
+# }
+#}
+#
+#gateway.coap {
+#
+# ## How long time the connection will be disconnected if the
+# ## connection is established but no bytes received
+# idle_timeout = 30s
+#
+# ## To control whether write statistics data into ETS table
+# ## for dashbord to read.
+# enable_stats = true
+#
+# ## When publishing or subscribing, prefix all topics with a mountpoint string.
+# mountpoint = ""
+#
+# ## Enable or disable connection mode
+# ## If true, you need to establish a connection before send any publish/subscribe
+# ## requests
+# ##
+# ## Default: false
+# #connection_required = false
+#
+# ## The Notification Message Type.
+# ## The notification message will be delivered to the CoAP client if a new
+# ## message received on an observed topic.
+# ## The type of delivered coap message can be set to:
+# ## - non: Non-confirmable
+# ## - con: Confirmable
+# ## - qos: Mapping from QoS type of the recevied message.
+# ## QoS0 -> non, QoS1,2 -> con.
+# ##
+# ## Enum: non | con | qos
+# ## Default: qos
+# #notify_type = qos
+#
+# ## The *Default QoS Level* indicator for subscribe request.
+# ## This option specifies the QoS level for the CoAP Client when establishing
+# ## a subscription membership, if the subscribe request is not carried `qos`
+# ## option.
+# ## The indicator can be set to:
+# ## - qos0, qos1, qos2: Fixed default QoS level
+# ## - coap: Dynamic QoS level by the message type of subscribe request
+# ## * qos0: If the subscribe request is non-confirmable
+# ## * qos1: If the subscribe request is confirmable
+# ##
+# ## Enum: qos0 | qos1 | qos2 | coap
+# ## Default: coap
+# #subscribe_qos = coap
+#
+# ## The *Default QoS Level* indicator for publish request.
+# ## This option specifies the QoS level for the CoAP Client when publishing a
+# ## message to EMQ X PUB/SUB system, if the publish request is not carried `qos`
+# ## option.
+# ## The indicator can be set to:
+# ## - qos0, qos1, qos2: Fixed default QoS level
+# ## - coap: Dynamic QoS level by the message type of publish request
+# ## * qos0: If the publish request is non-confirmable
+# ## * qos1: If the publish request is confirmable
+# ##
+# ## Enum: qos0 | qos1 | qos2 | coap
+# #publish_qos = coap
+#
+# listeners.udp.default {
+# bind = 5683
+# max_connections = 102400
+# max_conn_rate = 1000
+#
+# ## UDP Options
+# ## See ${example_common_udp_options} for more information
+# udp.active_n = 100
+# udp.buffer = 16KB
+# }
+# listeners.dtls.default {
+# bind = 5684
+# acceptors = 4
+# max_connections = 102400
+# max_conn_rate = 1000
+#
+# ## UDP Options
+# ## See ${example_common_udp_options} for more information
+# udp.active_n = 100
+# udp.buffer = 16KB
+#
+# ## DTLS Options
+# ## See #{example_common_dtls_options} for more information
+# dtls.versions = ["dtlsv1.2", "dtlsv1"]
+# dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
+# dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
+# dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
+# dtls.handshake_timeout = 15s
+# }
+#}
+#
+#gateway.mqttsn {
+#
+# ## How long time the connection will be disconnected if the
+# ## connection is established but no bytes received
+# idle_timeout = 30s
+#
+# ## To control whether write statistics data into ETS table
+# ## for dashbord to read.
+# enable_stats = true
+#
+# ## When publishing or subscribing, prefix all topics with a mountpoint string.
+# mountpoint = ""
+#
+# ## The MQTT-SN Gateway ID in ADVERTISE message.
+# gateway_id = 1
+#
+# ## Enable broadcast this gateway to WLAN
+# broadcast = true
+#
+# ## To control whether accept and process the received
+# ## publish message with qos=-1.
+# enable_qos3 = true
+#
+# ## The pre-defined topic name corresponding to the pre-defined topic
+# ## id of N.
+# ## Note that the pre-defined topic id of 0 is reserved.
+# predefined = [
+# { id = 1
+# topic = "/predefined/topic/name/hello"
+# },
+# { id = 2
+# topic = "/predefined/topic/name/nice"
+# }
+# ]
+#
+# ### ClientInfo override
+# clientinfo_override {
+# username = "mqtt_sn_user"
+# password = "abc"
+# }
+#
+# listeners.udp.default {
+# bind = 1884
+# max_connections = 10240000
+# max_conn_rate = 1000
+# }
+#
+# listeners.dtls.default {
+# bind = 1885
+# acceptors = 4
+# max_connections = 102400
+# max_conn_rate = 1000
+#
+# ## UDP Options
+# ## See ${example_common_udp_options} for more information
+# udp.active_n = 100
+# udp.buffer = 16KB
+#
+# ## DTLS Options
+# ## See #{example_common_dtls_options} for more information
+# dtls.versions = ["dtlsv1.2", "dtlsv1"]
+# dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem"
+# dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem"
+# dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
+# }
+#
+#}
+#
+#gateway.lwm2m {
+#
+# ## How long time the connection will be disconnected if the
+# ## connection is established but no bytes received
+# idle_timeout = 30s
+#
+# ## To control whether write statistics data into ETS table
+# ## for dashbord to read.
+# enable_stats = true
+#
+# ## When publishing or subscribing, prefix all topics with a mountpoint string.
+# mountpoint = "lwm2m/%u"
+#
+# xml_dir = "{{ platform_etc_dir }}/lwm2m_xml"
+#
+# ##
+# ##
+# lifetime_min = 1s
+#
+# lifetime_max = 86400s
+#
+# qmode_time_window = 22
+#
+# auto_observe = false
+#
+# ## always | contains_object_list
+# update_msg_publish_condition = contains_object_list
+#
+#
+# translators {
+# command {
+# topic = "/dn/#"
+# qos = 0
+# }
+#
+# response {
+# topic = "/up/resp"
+# qos = 0
+# }
+#
+# notify {
+# topic = "/up/notify"
+# qos = 0
+# }
+#
+# register {
+# topic = "/up/resp"
+# qos = 0
+# }
+#
+# update {
+# topic = "/up/resp"
+# qos = 0
+# }
+# }
+#
+# listeners.udp.default {
+# bind = 5783
+# }
+#}
+#
+#gateway.exproto {
+#
+# ## How long time the connection will be disconnected if the
+# ## connection is established but no bytes received
+# idle_timeout = 30s
+#
+# ## To control whether write statistics data into ETS table
+# ## for dashbord to read.
+# enable_stats = true
+#
+# ## When publishing or subscribing, prefix all topics with a mountpoint string.
+# mountpoint = ""
+#
+# ## The gRPC server to accept requests
+# server {
+# bind = 9100
+# #ssl.keyfile:
+# #ssl.certfile:
+# #ssl.cacertfile:
+# }
+#
+# handler {
+# address = "http://127.0.0.1:9001"
+# #ssl.keyfile:
+# #ssl.certfile:
+# #ssl.cacertfile:
+# }
+#
+# listeners.tcp.default {
+# bind = 7993
+# acceptors = 8
+# max_connections = 10240
+# max_conn_rate = 1000
+# }
+# #listeners.ssl.default: {}
+# #listeners.udp.default: {}
+# #listeners.dtls.default: {}
+#}
diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl
index 5c0893cb2..8b2081a90 100644
--- a/apps/emqx_gateway/include/emqx_gateway.hrl
+++ b/apps/emqx_gateway/include/emqx_gateway.hrl
@@ -19,8 +19,6 @@
-type gateway_name() :: atom().
--type listener() :: #{}.
-
%% @doc The Gateway defination
-type gateway() ::
#{ name := gateway_name()
diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
index 543b2e169..af34a1754 100644
--- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
+++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
@@ -81,10 +81,13 @@
%% Frame Module
frame_mod :: atom(),
%% Channel Module
- chann_mod :: atom()
+ chann_mod :: atom(),
+ %% Listener Tag
+ listener :: listener() | undefined
}).
--type(state() :: #state{}).
+-type listener() :: {GwName :: atom(), LisType :: atom(), LisName :: atom()}.
+-type state() :: #state{}.
-define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]).
-define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]).
@@ -279,7 +282,8 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) ->
idle_timer = IdleTimer,
oom_policy = OomPolicy,
frame_mod = FrameMod,
- chann_mod = ChannMod
+ chann_mod = ChannMod,
+ listener = maps:get(listener, Options, undefined)
}.
run_loop(Parent, State = #state{socket = Socket,
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl
index 4d0e8aff8..3eed9b802 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl
@@ -52,8 +52,8 @@ request(post, #{body := Body, bindings := Bindings}) ->
CT = maps:get(<<"content_type">>, Body, <<"text/plain">>),
Token = maps:get(<<"token">>, Body, <<>>),
Payload = maps:get(<<"payload">>, Body, <<>>),
- WaitTime = maps:get(<<"timeout">>, Body, ?DEF_WAIT_TIME),
-
+ BinWaitTime = maps:get(<<"timeout">>, Body, <<"10s">>),
+ {ok, WaitTime} = emqx_schema:to_duration_ms(BinWaitTime),
Payload2 = parse_payload(CT, Payload),
ReqType = erlang:binary_to_atom(Method),
@@ -83,7 +83,7 @@ request_parameters() ->
request_properties() ->
properties([ {token, string, "message token, can be empty"}
, {method, string, "request method type", ["get", "put", "post", "delete"]}
- , {timeout, integer, "timespan for response"}
+ , {timeout, string, "timespan for response", "10s"}
, {content_type, string, "payload type",
[<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]}
, {payload, string, "payload"}]).
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
index 86b1f023f..8ec1d568a 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
@@ -103,9 +103,15 @@ init(ConnInfo = #{peername := {PeerHost, _},
#{ctx := Ctx} = Config) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Mountpoint = maps:get(mountpoint, Config, <<>>),
+ ListenerId = case maps:get(listener, Config, undefined) of
+ undefined -> undefined;
+ {GwName, Type, LisName} ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName)
+ end,
ClientInfo = set_peercert_infos(
Peercert,
#{ zone => default
+ , listener => ListenerId
, protocol => 'coap'
, peerhost => PeerHost
, sockport => SockPort
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
index 055eab759..5fd557def 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl
@@ -100,8 +100,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
- NCfg = Cfg#{
- ctx => Ctx,
+ NCfg = Cfg#{ctx => Ctx,
+ listener => {GwName, Type, LisName},
frame_mod => emqx_coap_frame,
chann_mod => emqx_coap_channel
},
diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl
index 596b47547..23e9ce19c 100644
--- a/apps/emqx_gateway/src/emqx_gateway.erl
+++ b/apps/emqx_gateway/src/emqx_gateway.erl
@@ -20,11 +20,6 @@
-include("include/emqx_gateway.hrl").
-%% callbacks for emqx_config_handler
--export([ pre_config_update/2
- , post_config_update/4
- ]).
-
%% Gateway APIs
-export([ registered_gateway/0
, load/2
@@ -36,8 +31,6 @@
, list/0
]).
--export([update_rawconf/2]).
-
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@@ -84,37 +77,6 @@ start(Name) ->
stop(Name) ->
emqx_gateway_sup:stop_gateway_insta(Name).
--spec update_rawconf(binary(), emqx_config:raw_config())
- -> ok
- | {error, any()}.
-update_rawconf(RawName, RawConfDiff) ->
- case emqx:update_config([gateway], {RawName, RawConfDiff}) of
- {ok, _Result} -> ok;
- {error, Reason} -> {error, Reason}
- end.
-
-%%--------------------------------------------------------------------
-%% Config Handler
-
--spec pre_config_update(emqx_config:update_request(),
- emqx_config:raw_config()) ->
- {ok, emqx_config:update_request()} | {error, term()}.
-pre_config_update({RawName, RawConfDiff}, RawConf) ->
- {ok, emqx_map_lib:deep_merge(RawConf, #{RawName => RawConfDiff})}.
-
--spec post_config_update(emqx_config:update_request(), emqx_config:config(),
- emqx_config:config(), emqx_config:app_envs())
- -> ok | {ok, Result::any()} | {error, Reason::term()}.
-post_config_update({RawName, _}, NewConfig, OldConfig, _AppEnvs) ->
- GwName = binary_to_existing_atom(RawName),
- SubConf = maps:get(GwName, NewConfig),
- case maps:get(GwName, OldConfig, undefined) of
- undefined ->
- emqx_gateway:load(GwName, SubConf);
- _ ->
- emqx_gateway:update(GwName, SubConf)
- end.
-
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index 9037518c5..78233e0b8 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -48,6 +48,7 @@ apis() ->
, {"/gateway/:name", gateway_insta}
, {"/gateway/:name/stats", gateway_insta_stats}
].
+
%%--------------------------------------------------------------------
%% http handlers
@@ -57,7 +58,29 @@ gateway(get, Request) ->
undefined -> all;
S0 -> binary_to_existing_atom(S0, utf8)
end,
- {200, emqx_gateway_http:gateways(Status)}.
+ {200, emqx_gateway_http:gateways(Status)};
+gateway(post, Request) ->
+ Body = maps:get(body, Request, #{}),
+ try
+ Name0 = maps:get(<<"name">>, Body),
+ GwName = binary_to_existing_atom(Name0),
+ case emqx_gateway_registry:lookup(GwName) of
+ undefined -> error(badarg);
+ _ ->
+ GwConf = maps:without([<<"name">>], Body),
+ case emqx_gateway_conf:load_gateway(GwName, GwConf) of
+ ok ->
+ {204};
+ {error, Reason} ->
+ return_http_error(500, Reason)
+ end
+ end
+ catch
+ error : {badkey, K} ->
+ return_http_error(400, [K, " is required"]);
+ error : badarg ->
+ return_http_error(404, "Bad gateway name")
+ end.
gateway_insta(delete, #{bindings := #{name := Name0}}) ->
with_gateway(Name0, fun(GwName, _) ->
@@ -69,18 +92,16 @@ gateway_insta(get, #{bindings := #{name := Name0}}) ->
GwConf = filled_raw_confs([<<"gateway">>, Name0]),
LisConf = maps:get(<<"listeners">>, GwConf, #{}),
NLisConf = emqx_gateway_http:mapping_listener_m2l(Name0, LisConf),
- {200, GwConf#{<<"listeners">> => NLisConf}}
+ {200, GwConf#{<<"name">> => Name0, <<"listeners">> => NLisConf}}
end);
gateway_insta(put, #{body := GwConf0,
bindings := #{name := Name0}
}) ->
- with_gateway(Name0, fun(_, _) ->
+ with_gateway(Name0, fun(GwName, _) ->
GwConf = maps:without([<<"authentication">>, <<"listeners">>], GwConf0),
- case emqx_gateway:update_rawconf(Name0, GwConf) of
+ case emqx_gateway_conf:update_gateway(GwName, GwConf) of
ok ->
{200};
- {error, not_found} ->
- return_http_error(404, "Gateway not found");
{error, Reason} ->
return_http_error(500, Reason)
end
@@ -122,6 +143,16 @@ swagger("/gateway", get) ->
, responses =>
#{ <<"200">> => schema_gateway_overview_list() }
};
+swagger("/gateway", post) ->
+ #{ description => <<"Load a gateway">>
+ , requestBody => schema_gateway_conf()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ };
swagger("/gateway/:name", get) ->
#{ description => <<"Get the gateway configurations">>
, parameters => params_gateway_name_in_path()
@@ -189,7 +220,7 @@ schema_gateway_overview_list() ->
#{ type => object
, properties => properties_gateway_overview()
},
- <<"Gateway Overview list">>
+ <<"Gateway list">>
).
%% XXX: This is whole confs for all type gateways. It is used to fill the
@@ -202,6 +233,7 @@ schema_gateway_overview_list() ->
<<"name">> => <<"authenticator1">>,
<<"server_type">> => <<"built-in-database">>,
<<"user_id_type">> => <<"clientid">>},
+ <<"name">> => <<"coap">>,
<<"enable">> => true,
<<"enable_stats">> => true,<<"heartbeat">> => <<"30s">>,
<<"idle_timeout">> => <<"30s">>,
@@ -219,6 +251,7 @@ schema_gateway_overview_list() ->
-define(EXPROTO_GATEWAY_CONFS,
#{<<"enable">> => true,
+ <<"name">> => <<"exproto">>,
<<"enable_stats">> => true,
<<"handler">> =>
#{<<"address">> => <<"http://127.0.0.1:9001">>},
@@ -236,6 +269,7 @@ schema_gateway_overview_list() ->
-define(LWM2M_GATEWAY_CONFS,
#{<<"auto_observe">> => false,
+ <<"name">> => <<"lwm2m">>,
<<"enable">> => true,
<<"enable_stats">> => true,
<<"idle_timeout">> => <<"30s">>,
@@ -264,6 +298,7 @@ schema_gateway_overview_list() ->
#{<<"password">> => <<"abc">>,
<<"username">> => <<"mqtt_sn_user">>},
<<"enable">> => true,
+ <<"name">> => <<"mqtt-sn">>,
<<"enable_qos3">> => true,<<"enable_stats">> => true,
<<"gateway_id">> => 1,<<"idle_timeout">> => <<"30s">>,
<<"listeners">> => [
@@ -290,6 +325,7 @@ schema_gateway_overview_list() ->
#{<<"password">> => <<"${Packet.headers.passcode}">>,
<<"username">> => <<"${Packet.headers.login}">>},
<<"enable">> => true,
+ <<"name">> => <<"stomp">>,
<<"enable_stats">> => true,
<<"frame">> =>
#{<<"max_body_length">> => 8192,<<"max_headers">> => 10,
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
index 85eb4ddc7..ac5ee17a5 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
@@ -18,8 +18,144 @@
-behaviour(minirest_api).
+-import(emqx_gateway_http,
+ [ return_http_error/2
+ , schema_bad_request/0
+ , schema_not_found/0
+ , schema_internal_error/0
+ , schema_no_content/0
+ , with_gateway/2
+ , checks/2
+ ]).
+
%% minirest behaviour callbacks
-export([api_spec/0]).
+%% http handlers
+-export([authn/2]).
+
+%% internal export for emqx_gateway_api_listeners module
+-export([schema_authn/0]).
+
+%%--------------------------------------------------------------------
+%% minirest behaviour callbacks
+%%--------------------------------------------------------------------
+
api_spec() ->
- {[], []}.
+ {metadata(apis()), []}.
+
+apis() ->
+ [ {"/gateway/:name/authentication", authn}
+ ].
+
+%%--------------------------------------------------------------------
+%% http handlers
+
+authn(get, #{bindings := #{name := Name0}}) ->
+ with_gateway(Name0, fun(GwName, _) ->
+ {200, emqx_gateway_http:authn(GwName)}
+ end);
+
+authn(put, #{bindings := #{name := Name0},
+ body := Body}) ->
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:update_authn(GwName, Body),
+ {204}
+ end);
+
+authn(post, #{bindings := #{name := Name0},
+ body := Body}) ->
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:add_authn(GwName, Body),
+ {204}
+ end);
+
+authn(delete, #{bindings := #{name := Name0}}) ->
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:remove_authn(GwName),
+ {204}
+ end).
+
+%%--------------------------------------------------------------------
+%% Swagger defines
+%%--------------------------------------------------------------------
+
+metadata(APIs) ->
+ metadata(APIs, []).
+metadata([], APIAcc) ->
+ lists:reverse(APIAcc);
+metadata([{Path, Fun}|More], APIAcc) ->
+ Methods = [get, post, put, delete, patch],
+ Mds = lists:foldl(fun(M, Acc) ->
+ try
+ Acc#{M => swagger(Path, M)}
+ catch
+ error : function_clause ->
+ Acc
+ end
+ end, #{}, Methods),
+ metadata(More, [{Path, Mds, Fun} | APIAcc]).
+
+swagger("/gateway/:name/authentication", get) ->
+ #{ description => <<"Get the gateway authentication">>
+ , parameters => params_gateway_name_in_path()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"200">> => schema_authn()
+ }
+ };
+swagger("/gateway/:name/authentication", put) ->
+ #{ description => <<"Create the gateway authentication">>
+ , parameters => params_gateway_name_in_path()
+ , requestBody => schema_authn()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ };
+swagger("/gateway/:name/authentication", post) ->
+ #{ description => <<"Add authentication for the gateway">>
+ , parameters => params_gateway_name_in_path()
+ , requestBody => schema_authn()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ };
+swagger("/gateway/:name/authentication", delete) ->
+ #{ description => <<"Remove the gateway authentication">>
+ , parameters => params_gateway_name_in_path()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ }.
+
+%%--------------------------------------------------------------------
+%% params defines
+
+params_gateway_name_in_path() ->
+ [#{ name => name
+ , in => path
+ , schema => #{type => string}
+ , required => true
+ }].
+
+%%--------------------------------------------------------------------
+%% schemas
+
+schema_authn() ->
+ #{ description => <<"OK">>
+ , content => #{
+ 'application/json' => #{
+ schema => minirest:ref(<<"AuthenticatorInstance">>)
+ }}
+ }.
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index 374f2841d..a9e16704e 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -20,20 +20,23 @@
-import(emqx_gateway_http,
[ return_http_error/2
- , with_gateway/2
- , checks/2
, schema_bad_request/0
, schema_not_found/0
, schema_internal_error/0
, schema_no_content/0
+ , with_gateway/2
+ , checks/2
]).
+-import(emqx_gateway_api_authn, [schema_authn/0]).
+
%% minirest behaviour callbacks
-export([api_spec/0]).
%% http handlers
-export([ listeners/2
, listeners_insta/2
+ , listeners_insta_authn/2
]).
%%--------------------------------------------------------------------
@@ -46,7 +49,9 @@ api_spec() ->
apis() ->
[ {"/gateway/:name/listeners", listeners}
, {"/gateway/:name/listeners/:id", listeners_insta}
+ , {"/gateway/:name/listeners/:id/authentication", listeners_insta_authn}
].
+
%%--------------------------------------------------------------------
%% http handlers
@@ -69,13 +74,8 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) ->
undefined ->
ListenerId = emqx_gateway_utils:listener_id(
GwName, Type, LName),
- case emqx_gateway_http:update_listener(
- ListenerId, LConf) of
- ok ->
- {204};
- {error, Reason} ->
- return_http_error(500, Reason)
- end;
+ ok = emqx_gateway_http:add_listener(ListenerId, LConf),
+ {204};
_ ->
return_http_error(400, "Listener name has occupied")
end
@@ -84,12 +84,8 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) ->
listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId0}}) ->
ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
with_gateway(Name0, fun(_GwName, _) ->
- case emqx_gateway_http:remove_listener(ListenerId) of
- ok -> {204};
- {error, not_found} -> {204};
- {error, Reason} ->
- return_http_error(500, Reason)
- end
+ ok = emqx_gateway_http:remove_listener(ListenerId),
+ {204}
end);
listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) ->
ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
@@ -108,12 +104,38 @@ listeners_insta(put, #{body := LConf,
}) ->
ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
with_gateway(Name0, fun(_GwName, _) ->
- case emqx_gateway_http:update_listener(ListenerId, LConf) of
- ok ->
- {204};
- {error, Reason} ->
- return_http_error(500, Reason)
- end
+ ok = emqx_gateway_http:update_listener(ListenerId, LConf),
+ {204}
+ end).
+
+listeners_insta_authn(get, #{bindings := #{name := Name0,
+ id := ListenerId0}}) ->
+ ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
+ with_gateway(Name0, fun(GwName, _) ->
+ {200, emqx_gateway_http:authn(GwName, ListenerId)}
+ end);
+listeners_insta_authn(post, #{body := Conf,
+ bindings := #{name := Name0,
+ id := ListenerId0}}) ->
+ ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:add_authn(GwName, ListenerId, Conf),
+ {204}
+ end);
+listeners_insta_authn(put, #{body := Conf,
+ bindings := #{name := Name0,
+ id := ListenerId0}}) ->
+ ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:update_authn(GwName, ListenerId, Conf),
+ {204}
+ end);
+listeners_insta_authn(delete, #{bindings := #{name := Name0,
+ id := ListenerId0}}) ->
+ ListenerId = emqx_mgmt_util:urldecode(ListenerId0),
+ with_gateway(Name0, fun(GwName, _) ->
+ ok = emqx_gateway_http:remove_authn(GwName, ListenerId),
+ {204}
end).
%%--------------------------------------------------------------------
@@ -190,6 +212,52 @@ swagger("/gateway/:name/listeners/:id", put) ->
, <<"500">> => schema_internal_error()
, <<"200">> => schema_no_content()
}
+ };
+swagger("/gateway/:name/listeners/:id/authentication", get) ->
+ #{ description => <<"Get the listener's authentication info">>
+ , parameters => params_gateway_name_in_path()
+ ++ params_listener_id_in_path()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"200">> => schema_authn()
+ }
+ };
+swagger("/gateway/:name/listeners/:id/authentication", post) ->
+ #{ description => <<"Add authentication for the listener">>
+ , parameters => params_gateway_name_in_path()
+ ++ params_listener_id_in_path()
+ , requestBody => schema_authn()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ };
+swagger("/gateway/:name/listeners/:id/authentication", put) ->
+ #{ description => <<"Update authentication for the listener">>
+ , parameters => params_gateway_name_in_path()
+ ++ params_listener_id_in_path()
+ , requestBody => schema_authn()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
+ };
+swagger("/gateway/:name/listeners/:id/authentication", delete) ->
+ #{ description => <<"Remove authentication for the listener">>
+ , parameters => params_gateway_name_in_path()
+ ++ params_listener_id_in_path()
+ , responses =>
+ #{ <<"400">> => schema_bad_request()
+ , <<"404">> => schema_not_found()
+ , <<"500">> => schema_internal_error()
+ , <<"204">> => schema_no_content()
+ }
}.
%%--------------------------------------------------------------------
@@ -301,7 +369,6 @@ raw_properties_common_listener() ->
<<"Listener type. Enum: tcp, udp, ssl, dtls">>,
[<<"tcp">>, <<"ssl">>, <<"udp">>, <<"dtls">>]}
, {running, boolean, <<"Listener running status">>}
- %% FIXME:
, {bind, string, <<"Listener bind address or port">>}
, {acceptors, integer, <<"Listener acceptors number">>}
, {access_rules, {array, string}, <<"Listener Access rules for client">>}
diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl
index d90942220..1f8d226e2 100644
--- a/apps/emqx_gateway/src/emqx_gateway_app.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_app.erl
@@ -22,20 +22,17 @@
-export([start/2, stop/1]).
--define(CONF_CALLBACK_MODULE, emqx_gateway).
-
start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_gateway_sup:start_link(),
emqx_gateway_cli:load(),
load_default_gateway_applications(),
load_gateway_by_default(),
- emqx_config_handler:add_handler([gateway], ?CONF_CALLBACK_MODULE),
+ emqx_gateway_conf:load(),
{ok, Sup}.
stop(_State) ->
+ emqx_gateway_conf:unload(),
emqx_gateway_cli:unload(),
- %% XXX: No api now
- %emqx_config_handler:remove_handler([gateway], ?MODULE),
ok.
%%--------------------------------------------------------------------
diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl
new file mode 100644
index 000000000..05cba46b0
--- /dev/null
+++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl
@@ -0,0 +1,270 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc The gateway configuration management module
+-module(emqx_gateway_conf).
+
+%% Load/Unload
+-export([ load/0
+ , unload/0
+ ]).
+
+%% APIs
+-export([ load_gateway/2
+ , update_gateway/2
+ , remove_gateway/1
+ , add_listener/3
+ , update_listener/3
+ , remove_listener/2
+ , add_authn/2
+ , add_authn/3
+ , update_authn/2
+ , update_authn/3
+ , remove_authn/1
+ , remove_authn/2
+ ]).
+
+%% callbacks for emqx_config_handler
+-export([ pre_config_update/2
+ , post_config_update/4
+ ]).
+
+-type atom_or_bin() :: atom() | binary().
+-type ok_or_err() :: ok_or_err().
+-type listener_ref() :: {ListenerType :: atom_or_bin(),
+ ListenerName :: atom_or_bin()}.
+
+%%--------------------------------------------------------------------
+%% Load/Unload
+%%--------------------------------------------------------------------
+
+-spec load() -> ok.
+load() ->
+ emqx_config_handler:add_handler([gateway], ?MODULE).
+
+-spec unload() -> ok.
+unload() ->
+ emqx_config_handler:remove_handler([gateway]).
+
+%%--------------------------------------------------------------------
+%% APIs
+
+-spec load_gateway(atom_or_bin(), map()) -> ok_or_err().
+load_gateway(GwName, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), Conf}).
+
+-spec update_gateway(atom_or_bin(), map()) -> ok_or_err().
+update_gateway(GwName, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), Conf}).
+
+-spec remove_gateway(atom_or_bin()) -> ok_or_err().
+remove_gateway(GwName) ->
+ update({?FUNCTION_NAME, bin(GwName)}).
+
+-spec add_listener(atom_or_bin(), listener_ref(), map()) -> ok_or_err().
+add_listener(GwName, ListenerRef, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}).
+
+-spec update_listener(atom_or_bin(), listener_ref(), map()) -> ok_or_err().
+update_listener(GwName, ListenerRef, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}).
+
+-spec remove_listener(atom_or_bin(), listener_ref()) -> ok_or_err().
+remove_listener(GwName, ListenerRef) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef)}).
+
+-spec add_authn(atom_or_bin(), map()) -> ok_or_err().
+add_authn(GwName, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), Conf}).
+
+-spec add_authn(atom_or_bin(), listener_ref(), map()) -> ok_or_err().
+add_authn(GwName, ListenerRef, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}).
+
+-spec update_authn(atom_or_bin(), map()) -> ok_or_err().
+update_authn(GwName, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), Conf}).
+
+-spec update_authn(atom_or_bin(), listener_ref(), map()) -> ok_or_err().
+update_authn(GwName, ListenerRef, Conf) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}).
+
+-spec remove_authn(atom_or_bin()) -> ok_or_err().
+remove_authn(GwName) ->
+ update({?FUNCTION_NAME, bin(GwName)}).
+
+-spec remove_authn(atom_or_bin(), listener_ref()) -> ok_or_err().
+remove_authn(GwName, ListenerRef) ->
+ update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef)}).
+
+%% @private
+update(Req) ->
+ res(emqx:update_config([gateway], Req)).
+
+res({ok, _Result}) -> ok;
+res({error, {pre_config_update,emqx_gateway_conf,Reason}}) -> {error, Reason};
+res({error, Reason}) -> {error, Reason}.
+
+bin({LType, LName}) ->
+ {bin(LType), bin(LName)};
+bin(A) when is_atom(A) ->
+ atom_to_binary(A);
+bin(B) when is_binary(B) ->
+ B.
+
+%%--------------------------------------------------------------------
+%% Config Handler
+%%--------------------------------------------------------------------
+
+-spec pre_config_update(emqx_config:update_request(),
+ emqx_config:raw_config()) ->
+ {ok, emqx_config:update_request()} | {error, term()}.
+pre_config_update({load_gateway, GwName, Conf}, RawConf) ->
+ case maps:get(GwName, RawConf, undefined) of
+ undefined ->
+ {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => Conf})};
+ _ ->
+ {error, already_exist}
+ end;
+pre_config_update({update_gateway, GwName, Conf}, RawConf) ->
+ case maps:get(GwName, RawConf, undefined) of
+ undefined ->
+ {error, not_found};
+ _ ->
+ NConf = maps:without([<<"listeners">>,
+ <<"authentication">>], Conf),
+ {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})}
+ end;
+pre_config_update({remove_gateway, GwName}, RawConf) ->
+ {ok, maps:remove(GwName, RawConf)};
+
+pre_config_update({add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"listeners">>, LType, LName], RawConf, undefined) of
+ undefined ->
+ NListener = #{LType => #{LName => Conf}},
+ {ok, emqx_map_lib:deep_merge(
+ RawConf,
+ #{GwName => #{<<"listeners">> => NListener}})};
+ _ ->
+ {error, already_exist}
+ end;
+pre_config_update({update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"listeners">>, LType, LName], RawConf, undefined) of
+ undefined ->
+ {error, not_found};
+ _OldConf ->
+ NListener = #{LType => #{LName => Conf}},
+ {ok, emqx_map_lib:deep_merge(
+ RawConf,
+ #{GwName => #{<<"listeners">> => NListener}})}
+
+ end;
+pre_config_update({remove_listener, GwName, {LType, LName}}, RawConf) ->
+ {ok, emqx_map_lib:deep_remove(
+ [GwName, <<"listeners">>, LType, LName], RawConf)};
+
+pre_config_update({add_authn, GwName, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"authentication">>], RawConf, undefined) of
+ undefined ->
+ {ok, emqx_map_lib:deep_merge(
+ RawConf,
+ #{GwName => #{<<"authentication">> => Conf}})};
+ _ ->
+ {error, already_exist}
+ end;
+pre_config_update({add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"listeners">>, LType, LName],
+ RawConf, undefined) of
+ undefined ->
+ {error, not_found};
+ Listener ->
+ case maps:get(<<"authentication">>, Listener, undefined) of
+ undefined ->
+ NListener = maps:put(<<"authentication">>, Conf, Listener),
+ NGateway = #{GwName =>
+ #{<<"listeners">> =>
+ #{LType => #{LName => NListener}}}},
+ {ok, emqx_map_lib:deep_merge(RawConf, NGateway)};
+ _ ->
+ {error, already_exist}
+ end
+ end;
+pre_config_update({update_authn, GwName, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"authentication">>], RawConf, undefined) of
+ undefined ->
+ {error, not_found};
+ _ ->
+ {ok, emqx_map_lib:deep_merge(
+ RawConf,
+ #{GwName => #{<<"authentication">> => Conf}})}
+ end;
+pre_config_update({update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+ case emqx_map_lib:deep_get(
+ [GwName, <<"listeners">>, LType, LName],
+ RawConf, undefined) of
+ undefined ->
+ {error, not_found};
+ Listener ->
+ case maps:get(<<"authentication">>, Listener, undefined) of
+ undefined ->
+ {error, not_found};
+ Auth ->
+ NListener = maps:put(
+ <<"authentication">>,
+ emqx_map_lib:deep_merge(Auth, Conf),
+ Listener
+ ),
+ NGateway = #{GwName =>
+ #{<<"listeners">> =>
+ #{LType => #{LName => NListener}}}},
+ {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}
+ end
+ end;
+pre_config_update({remove_authn, GwName}, RawConf) ->
+ {ok, emqx_map_lib:deep_remove(
+ [GwName, <<"authentication">>], RawConf)};
+pre_config_update({remove_authn, GwName, {LType, LName}}, RawConf) ->
+ Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>],
+ {ok, emqx_map_lib:deep_remove(Path, RawConf)};
+
+pre_config_update(UnknownReq, _RawConf) ->
+ logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
+ {error, badreq}.
+
+-spec post_config_update(emqx_config:update_request(), emqx_config:config(),
+ emqx_config:config(), emqx_config:app_envs())
+ -> ok | {ok, Result::any()} | {error, Reason::term()}.
+
+post_config_update(Req, NewConfig, OldConfig, _AppEnvs) ->
+ [_Tag, GwName0|_] = tuple_to_list(Req),
+ GwName = binary_to_existing_atom(GwName0),
+
+ case {maps:get(GwName, NewConfig, undefined),
+ maps:get(GwName, OldConfig, undefined)} of
+ {undefined, undefined} ->
+ ok; %% nothing to change
+ {undefined, Old} when is_map(Old) ->
+ emqx_gateway:unload(GwName);
+ {New, undefined} when is_map(New) ->
+ emqx_gateway:load(GwName, New);
+ {New, Old} when is_map(New), is_map(Old) ->
+ emqx_gateway:update(GwName, New)
+ end.
diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl
index 8022c3797..54714974a 100644
--- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl
@@ -29,8 +29,8 @@
-type context() ::
#{ %% Gateway Name
gwname := gateway_name()
- %% Autenticator
- , auth := emqx_authn:chain_id() | undefined
+ %% Authentication chains
+ , auth := [emqx_authentication:chain_name()] | undefined
%% The ConnectionManager PID
, cm := pid()
}.
@@ -66,12 +66,8 @@
| {error, any()}.
authenticate(_Ctx = #{auth := undefined}, ClientInfo) ->
{ok, mountpoint(ClientInfo)};
-authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) ->
- ClientInfo = ClientInfo0#{
- zone => default,
- listener => {tcp, default},
- chain_id => ChainId
- },
+authenticate(_Ctx = #{auth := _ChainName}, ClientInfo0) ->
+ ClientInfo = ClientInfo0#{zone => default},
case emqx_access_control:authenticate(ClientInfo) of
{ok, _} ->
{ok, mountpoint(ClientInfo)};
diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl
index ed8e511c7..fc3ded3c9 100644
--- a/apps/emqx_gateway/src/emqx_gateway_http.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_http.erl
@@ -27,11 +27,22 @@
%% Mgmt APIs - listeners
-export([ listeners/1
, listener/1
+ , add_listener/2
, remove_listener/1
, update_listener/2
, mapping_listener_m2l/2
]).
+-export([ authn/1
+ , authn/2
+ , add_authn/2
+ , add_authn/3
+ , update_authn/2
+ , update_authn/3
+ , remove_authn/1
+ , remove_authn/2
+ ]).
+
%% Mgmt APIs - clients
-export([ lookup_client/3
, lookup_client/4
@@ -171,12 +182,13 @@ listener(GwName, Type, Conf) ->
[begin
ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LName),
Running = is_running(ListenerId, LConf),
- LConf#{
- id => ListenerId,
- type => Type,
- name => LName,
- running => Running
- }
+ bind2str(
+ LConf#{
+ id => ListenerId,
+ type => Type,
+ name => LName,
+ running => Running
+ })
end || {LName, LConf} <- Conf, is_map(LConf)].
is_running(ListenerId, #{<<"bind">> := ListenOn0}) ->
@@ -188,27 +200,78 @@ is_running(ListenerId, #{<<"bind">> := ListenOn0}) ->
false
end.
--spec remove_listener(binary()) -> ok | {error, not_found} | {error, any()}.
-remove_listener(ListenerId) ->
- {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
- LConf = emqx:get_raw_config(
- [<<"gateway">>, GwName, <<"listeners">>, Type]
- ),
- NLConf = maps:remove(Name, LConf),
- emqx_gateway:update_rawconf(
- GwName,
- #{<<"listeners">> => #{Type => NLConf}}
- ).
+bind2str(LConf = #{bind := Bind}) when is_integer(Bind) ->
+ maps:put(bind, integer_to_binary(Bind), LConf);
+bind2str(LConf = #{<<"bind">> := Bind}) when is_integer(Bind) ->
+ maps:put(<<"bind">>, integer_to_binary(Bind), LConf);
+bind2str(LConf = #{bind := Bind}) when is_binary(Bind) ->
+ LConf;
+bind2str(LConf = #{<<"bind">> := Bind}) when is_binary(Bind) ->
+ LConf.
--spec update_listener(atom() | binary(), map()) -> ok | {error, any()}.
-update_listener(ListenerId, NewConf0) ->
+-spec add_listener(atom() | binary(), map()) -> ok.
+add_listener(ListenerId, NewConf0) ->
{GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
NewConf = maps:without([<<"id">>, <<"name">>,
<<"type">>, <<"running">>], NewConf0),
- emqx_gateway:update_rawconf(
- GwName,
- #{<<"listeners">> => #{Type => #{Name => NewConf}}
- }).
+ confexp(emqx_gateway_conf:add_listener(GwName, {Type, Name}, NewConf)).
+
+-spec update_listener(atom() | binary(), map()) -> ok.
+update_listener(ListenerId, NewConf0) ->
+ {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+
+ NewConf = maps:without([<<"id">>, <<"name">>,
+ <<"type">>, <<"running">>], NewConf0),
+ confexp(emqx_gateway_conf:update_listener(GwName, {Type, Name}, NewConf)).
+
+-spec remove_listener(binary()) -> ok.
+remove_listener(ListenerId) ->
+ {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+ confexp(emqx_gateway_conf:remove_listener(GwName, {Type, Name})).
+
+-spec authn(gateway_name()) -> map().
+authn(GwName) ->
+ Path = [gateway, GwName, authentication],
+ emqx_map_lib:jsonable_map(emqx:get_config(Path)).
+
+-spec authn(gateway_name(), binary()) -> map().
+authn(GwName, ListenerId) ->
+ {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+ Path = [gateway, GwName, listeners, Type, Name, authentication],
+ emqx_map_lib:jsonable_map(emqx:get_config(Path)).
+
+-spec add_authn(gateway_name(), map()) -> ok.
+add_authn(GwName, AuthConf) ->
+ confexp(emqx_gateway_conf:add_authn(GwName, AuthConf)).
+
+-spec add_authn(gateway_name(), binary(), map()) -> ok.
+add_authn(GwName, ListenerId, AuthConf) ->
+ {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+ confexp(emqx_gateway_conf:add_authn(GwName, {Type, Name}, AuthConf)).
+
+-spec update_authn(gateway_name(), map()) -> ok.
+update_authn(GwName, AuthConf) ->
+ confexp(emqx_gateway_conf:update_authn(GwName, AuthConf)).
+
+-spec update_authn(gateway_name(), binary(), map()) -> ok.
+update_authn(GwName, ListenerId, AuthConf) ->
+ {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+ confexp(emqx_gateway_conf:update_authn(GwName, {Type, Name}, AuthConf)).
+
+-spec remove_authn(gateway_name()) -> ok.
+remove_authn(GwName) ->
+ confexp(emqx_gateway_conf:remove_authn(GwName)).
+
+-spec remove_authn(gateway_name(), binary()) -> ok.
+remove_authn(GwName, ListenerId) ->
+ {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
+ confexp(emqx_gateway_conf:remove_authn(GwName, {Type, Name})).
+
+confexp(ok) -> ok;
+confexp({error, not_found}) ->
+ error({update_conf_error, not_found});
+confexp({error, already_exist}) ->
+ error({update_conf_error, already_exist}).
%%--------------------------------------------------------------------
%% Mgmt APIs - clients
@@ -328,10 +391,22 @@ with_gateway(GwName0, Fun) ->
catch
error : badname ->
return_http_error(404, "Bad gateway name");
+ %% Exceptions from: checks/2
error : {miss_param, K} ->
return_http_error(400, [K, " is required"]);
+ %% Exceptions from emqx_gateway_utils:parse_listener_id/1
error : {invalid_listener_id, Id} ->
return_http_error(400, ["invalid listener id: ", Id]);
+ %% Exceptions from: emqx:get_config/1
+ error : {config_not_found, Path0} ->
+ Path = lists:concat(
+ lists:join(".", lists:map(fun to_list/1, Path0))),
+ return_http_error(404, "Resource not found. path: " ++ Path);
+ %% Exceptions from: confexp/1
+ error : {update_conf_error, not_found} ->
+ return_http_error(404, "Resource not found");
+ error : {update_conf_error, already_exist} ->
+ return_http_error(400, "Resource already exist");
Class : Reason : Stk ->
?LOG(error, "Uncatched error: {~p, ~p}, stacktrace: ~0p",
[Class, Reason, Stk]),
@@ -348,6 +423,11 @@ checks([K|Ks], Map) ->
error({miss_param, K})
end.
+to_list(A) when is_atom(A) ->
+ atom_to_list(A);
+to_list(B) when is_binary(B) ->
+ binary_to_list(B).
+
%%--------------------------------------------------------------------
%% common schemas
diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
index 39115f114..5f78a84b5 100644
--- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
@@ -43,6 +43,7 @@
name :: gateway_name(),
config :: emqx_config:config(),
ctx :: emqx_gateway_ctx:context(),
+ authns :: [emqx_authentication:chain_name()],
status :: stopped | running,
child_pids :: [pid()],
gw_state :: emqx_gateway_impl:state() | undefined,
@@ -94,16 +95,23 @@ init([Gateway, Ctx, _GwDscrptr]) ->
State = #state{
ctx = Ctx,
name = GwName,
+ authns = [],
config = Config,
child_pids = [],
status = stopped,
created_at = erlang:system_time(millisecond)
},
- case cb_gateway_load(State) of
- {error, Reason} ->
- {stop, {load_gateway_failure, Reason}};
- {ok, NState} ->
- {ok, NState}
+ case maps:get(enable, Config, true) of
+ false ->
+ ?LOG(info, "Skipp to start ~s gateway due to disabled", [GwName]),
+ {ok, State};
+ true ->
+ case cb_gateway_load(State) of
+ {error, Reason} ->
+ {stop, {load_gateway_failure, Reason}};
+ {ok, NState} ->
+ {ok, NState}
+ end
end.
handle_call(info, _From, State) ->
@@ -174,9 +182,9 @@ handle_info(Info, State) ->
?LOG(warning, "Unexcepted info: ~p", [Info]),
{noreply, State}.
-terminate(_Reason, State = #state{ctx = Ctx, child_pids = Pids}) ->
+terminate(_Reason, State = #state{child_pids = Pids}) ->
Pids /= [] andalso (_ = cb_gateway_unload(State)),
- _ = do_deinit_authn(maps:get(auth, Ctx, undefined)),
+ _ = do_deinit_authn(State#state.authns),
ok.
code_change(_OldVsn, State, _Extra) ->
@@ -197,52 +205,102 @@ detailed_gateway_info(State) ->
%% Internal funcs
%%--------------------------------------------------------------------
-do_init_authn(GwName, Config) ->
- case maps:get(authentication, Config, #{enable => false}) of
- #{enable := false} -> undefined;
- AuthCfg when is_map(AuthCfg) ->
- case maps:get(enable, AuthCfg, true) of
- false ->
- undefined;
- _ ->
- %% TODO: Implement Authentication
- GwName
- %case emqx_authn:create_chain(#{id => ChainId}) of
- % {ok, _ChainInfo} ->
- % case emqx_authn:create_authenticator(ChainId, AuthCfg) of
- % {ok, _} -> ChainId;
- % {error, Reason} ->
- % ?LOG(error, "Failed to create authentication ~p", [Reason]),
- % throw({bad_authentication, Reason})
- % end;
- % {error, Reason} ->
- % ?LOG(error, "Failed to create authentication chain: ~p", [Reason]),
- % throw({bad_chain, {ChainId, Reason}})
- %end.
- end;
- _ ->
- undefined
+%% same with emqx_authentication:global_chain/1
+global_chain(mqtt) ->
+ 'mqtt:global';
+global_chain('mqtt-sn') ->
+ 'mqtt-sn:global';
+global_chain(coap) ->
+ 'coap:global';
+global_chain(lwm2m) ->
+ 'lwm2m:global';
+global_chain(stomp) ->
+ 'stomp:global';
+global_chain(_) ->
+ 'unknown:global'.
+
+listener_chain(GwName, Type, LisName) ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName).
+
+%% There are two layer authentication configs
+%% stomp.authn
+%% / \
+%% listeners.tcp.defautl.authn *.ssl.default.authn
+%%
+
+init_authn(GwName, Config) ->
+ Authns = authns(GwName, Config),
+ try
+ do_init_authn(Authns, [])
+ catch
+ throw : Reason = {badauth, _} ->
+ do_deinit_authn(proplists:get_keys(Authns)),
+ throw(Reason)
end.
-do_deinit_authn(undefined) ->
- ok;
-do_deinit_authn(AuthnRef) ->
- %% TODO:
- ?LOG(warning, "Failed to clean authn ~p, not suppported now", [AuthnRef]).
- %case emqx_authn:delete_chain(AuthnRef) of
- % ok -> ok;
- % {error, {not_found, _}} ->
- % ?LOG(warning, "Failed to clean authentication chain: ~s, "
- % "reason: not_found", [AuthnRef]);
- % {error, Reason} ->
- % ?LOG(error, "Failed to clean authentication chain: ~s, "
- % "reason: ~p", [AuthnRef, Reason])
- %end.
+do_init_authn([], Names) ->
+ Names;
+do_init_authn([{_ChainName, _AuthConf = #{enable := false}}|More], Names) ->
+ do_init_authn(More, Names);
+do_init_authn([{ChainName, AuthConf}|More], Names) when is_map(AuthConf) ->
+ _ = application:ensure_all_started(emqx_authn),
+ do_create_authn_chain(ChainName, AuthConf),
+ do_init_authn(More, [ChainName|Names]);
+do_init_authn([_BadConf|More], Names) ->
+ do_init_authn(More, Names).
+
+authns(GwName, Config) ->
+ Listeners = maps:to_list(maps:get(listeners, Config, #{})),
+ lists:append(
+ [ [{listener_chain(GwName, LisType, LisName), authn_conf(Opts)}
+ || {LisName, Opts} <- maps:to_list(LisNames) ]
+ || {LisType, LisNames} <- Listeners])
+ ++ [{global_chain(GwName), authn_conf(Config)}].
+
+authn_conf(Conf) ->
+ maps:get(authentication, Conf, #{enable => false}).
+
+do_create_authn_chain(ChainName, AuthConf) ->
+ case ensure_chain(ChainName) of
+ ok ->
+ case emqx_authentication:create_authenticator(ChainName, AuthConf) of
+ {ok, _} -> ok;
+ {error, Reason} ->
+ ?LOG(error, "Failed to create authenticator chain ~s, "
+ "reason: ~p, config: ~p",
+ [ChainName, Reason, AuthConf]),
+ throw({badauth, Reason})
+ end;
+ {error, Reason} ->
+ ?LOG(error, "Falied to create authn chain ~s, reason ~p",
+ [ChainName, Reason]),
+ throw({badauth, Reason})
+ end.
+
+ensure_chain(ChainName) ->
+ case emqx_authentication:create_chain(ChainName) of
+ {ok, _ChainInfo} ->
+ ok;
+ {error, {already_exists, _}} ->
+ ok;
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+do_deinit_authn(Names) ->
+ lists:foreach(fun(ChainName) ->
+ case emqx_authentication:delete_chain(ChainName) of
+ ok -> ok;
+ {error, {not_found, _}} -> ok;
+ {error, Reason} ->
+ ?LOG(error, "Failed to clean authentication chain: ~s, "
+ "reason: ~p", [ChainName, Reason])
+ end
+ end, Names).
do_update_one_by_one(NCfg0, State = #state{
- ctx = Ctx,
- config = OCfg,
- status = Status}) ->
+ config = OCfg,
+ status = Status}) ->
NCfg = emqx_map_lib:deep_merge(OCfg, NCfg0),
@@ -263,14 +321,9 @@ do_update_one_by_one(NCfg0, State = #state{
true -> State;
false ->
%% Reset Authentication first
- _ = do_deinit_authn(maps:get(auth, Ctx, undefined)),
- NCtx = Ctx#{
- auth => do_init_authn(
- State#state.name,
- NCfg
- )
- },
- State#state{ctx = NCtx}
+ _ = do_deinit_authn(State#state.authns),
+ AuthnNames = init_authn(State#state.name, NCfg),
+ State#state{authns = AuthnNames}
end,
cb_gateway_update(NCfg, NState);
Status == running, NEnable == false ->
@@ -289,6 +342,7 @@ cb_gateway_unload(State = #state{name = GwName,
#{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName),
CbMod:on_gateway_unload(Gateway, GwState),
{ok, State#state{child_pids = [],
+ authns = [],
status = stopped,
gw_state = undefined,
started_at = undefined,
@@ -300,6 +354,8 @@ cb_gateway_unload(State = #state{name = GwName,
[GwName, GwState,
Class, Reason, Stk]),
{error, {Class, Reason, Stk}}
+ after
+ _ = do_deinit_authn(State#state.authns)
end.
%% @doc 1. Create Authentcation Context
@@ -311,38 +367,33 @@ cb_gateway_load(State = #state{name = GwName,
ctx = Ctx}) ->
Gateway = detailed_gateway_info(State),
-
- case maps:get(enable, Config, true) of
- false ->
- ?LOG(info, "Skipp to start ~s gateway due to disabled", [GwName]);
- true ->
- try
- AuthnRef = do_init_authn(GwName, Config),
- NCtx = Ctx#{auth => AuthnRef},
- #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName),
- case CbMod:on_gateway_load(Gateway, NCtx) of
- {error, Reason} ->
- do_deinit_authn(AuthnRef),
- throw({callback_return_error, Reason});
- {ok, ChildPidOrSpecs, GwState} ->
- ChildPids = start_child_process(ChildPidOrSpecs),
- {ok, State#state{
- ctx = NCtx,
- status = running,
- child_pids = ChildPids,
- gw_state = GwState,
- stopped_at = undefined,
- started_at = erlang:system_time(millisecond)
- }}
- end
- catch
- Class : Reason1 : Stk ->
- ?LOG(error, "Failed to load ~s gateway (~0p, ~0p) "
- "crashed: {~p, ~p}, stacktrace: ~0p",
- [GwName, Gateway, Ctx,
- Class, Reason1, Stk]),
- {error, {Class, Reason1, Stk}}
- end
+ try
+ AuthnNames = init_authn(GwName, Config),
+ NCtx = Ctx#{auth => AuthnNames},
+ #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName),
+ case CbMod:on_gateway_load(Gateway, NCtx) of
+ {error, Reason} ->
+ do_deinit_authn(AuthnNames),
+ throw({callback_return_error, Reason});
+ {ok, ChildPidOrSpecs, GwState} ->
+ ChildPids = start_child_process(ChildPidOrSpecs),
+ {ok, State#state{
+ ctx = NCtx,
+ authns = AuthnNames,
+ status = running,
+ child_pids = ChildPids,
+ gw_state = GwState,
+ stopped_at = undefined,
+ started_at = erlang:system_time(millisecond)
+ }}
+ end
+ catch
+ Class : Reason1 : Stk ->
+ ?LOG(error, "Failed to load ~s gateway (~0p, ~0p) "
+ "crashed: {~p, ~p}, stacktrace: ~0p",
+ [GwName, Gateway, Ctx,
+ Class, Reason1, Stk]),
+ {error, {Class, Reason1, Stk}}
end.
cb_gateway_update(Config,
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index abef053cb..bb0bf9dbe 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -50,11 +50,11 @@ namespace() -> gateway.
roots() -> [gateway].
fields(gateway) ->
- [{stomp, sc(ref(stomp))},
- {mqttsn, sc(ref(mqttsn))},
- {coap, sc(ref(coap))},
- {lwm2m, sc(ref(lwm2m))},
- {exproto, sc(ref(exproto))}
+ [{stomp, sc_meta(ref(stomp) , #{nullable => {true, recursively}})},
+ {mqttsn, sc_meta(ref(mqttsn) , #{nullable => {true, recursively}})},
+ {coap, sc_meta(ref(coap) , #{nullable => {true, recursively}})},
+ {lwm2m, sc_meta(ref(lwm2m) , #{nullable => {true, recursively}})},
+ {exproto, sc_meta(ref(exproto), #{nullable => {true, recursively}})}
];
fields(stomp) ->
@@ -92,10 +92,10 @@ fields(coap) ->
fields(lwm2m) ->
[ {xml_dir, sc(binary())}
- , {lifetime_min, sc(duration())}
- , {lifetime_max, sc(duration())}
- , {qmode_time_windonw, sc(integer())}
- , {auto_observe, sc(boolean())}
+ , {lifetime_min, sc(duration(), "1s")}
+ , {lifetime_max, sc(duration(), "86400s")}
+ , {qmode_time_window, sc(integer(), 22)}
+ , {auto_observe, sc(boolean(), false)}
, {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))}
, {translators, sc(ref(translators))}
, {listeners, sc(ref(udp_listeners))}
@@ -154,8 +154,8 @@ fields(udp_tcp_listeners) ->
];
fields(tcp_listener) ->
- [
- %% some special confs for tcp listener
+ [ %% some special confs for tcp listener
+ {acceptors, sc(integer(), 16)}
] ++
tcp_opts() ++
proxy_protocol_opts() ++
@@ -175,6 +175,8 @@ fields(udp_listener) ->
common_listener_opts();
fields(dtls_listener) ->
+ [ {acceptors, sc(integer(), 16)}
+ ] ++
fields(udp_listener) ++
[{dtls, sc_meta(ref(dtls_opts),
#{desc => "DTLS listener options"})}];
@@ -191,29 +193,33 @@ fields(dtls_opts) ->
emqx_schema:server_ssl_opts_schema(
#{ depth => 10
, reuse_sessions => true
- , versions => dtls
- , ciphers => dtls
+ , versions => dtls_all_available
+ , ciphers => dtls_all_available
}, false).
-% authentication() ->
-% hoconsc:union(
-% [ undefined
-% , hoconsc:ref(emqx_authn_mnesia, config)
-% , hoconsc:ref(emqx_authn_mysql, config)
-% , hoconsc:ref(emqx_authn_pgsql, config)
-% , hoconsc:ref(emqx_authn_mongodb, standalone)
-% , hoconsc:ref(emqx_authn_mongodb, 'replica-set')
-% , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster')
-% , hoconsc:ref(emqx_authn_redis, standalone)
-% , hoconsc:ref(emqx_authn_redis, cluster)
-% , hoconsc:ref(emqx_authn_redis, sentinel)
-% , hoconsc:ref(emqx_authn_http, get)
-% , hoconsc:ref(emqx_authn_http, post)
-% , hoconsc:ref(emqx_authn_jwt, 'hmac-based')
-% , hoconsc:ref(emqx_authn_jwt, 'public-key')
-% , hoconsc:ref(emqx_authn_jwt, 'jwks')
-% , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config)
-% ]).
+authentication() ->
+ sc_meta(hoconsc:union(
+ [ hoconsc:ref(emqx_authn_mnesia, config)
+ , hoconsc:ref(emqx_authn_mysql, config)
+ , hoconsc:ref(emqx_authn_pgsql, config)
+ , hoconsc:ref(emqx_authn_mongodb, standalone)
+ , hoconsc:ref(emqx_authn_mongodb, 'replica-set')
+ , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster')
+ , hoconsc:ref(emqx_authn_redis, standalone)
+ , hoconsc:ref(emqx_authn_redis, cluster)
+ , hoconsc:ref(emqx_authn_redis, sentinel)
+ , hoconsc:ref(emqx_authn_http, get)
+ , hoconsc:ref(emqx_authn_http, post)
+ , hoconsc:ref(emqx_authn_jwt, 'hmac-based')
+ , hoconsc:ref(emqx_authn_jwt, 'public-key')
+ , hoconsc:ref(emqx_authn_jwt, 'jwks')
+ , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config)
+ ]),
+ #{nullable => {true, recursively},
+ desc =>
+"""Default authentication configs for all of the gateway listeners.
+For per-listener overrides see authentication
+in listener configs"""}).
gateway_common_options() ->
[ {enable, sc(boolean(), true)}
@@ -221,16 +227,15 @@ gateway_common_options() ->
, {idle_timeout, sc(duration(), <<"30s">>)}
, {mountpoint, sc(binary(), <<>>)}
, {clientinfo_override, sc(ref(clientinfo_override))}
- , {authentication, sc(hoconsc:lazy(map()))}
+ , {authentication, authentication()}
].
common_listener_opts() ->
[ {enable, sc(boolean(), true)}
, {bind, sc(union(ip_port(), integer()))}
- , {acceptors, sc(integer(), 16)}
, {max_connections, sc(integer(), 1024)}
, {max_conn_rate, sc(integer())}
- %, {rate_limit, sc(comma_separated_list())}
+ , {authentication, authentication()}
, {mountpoint, sc(binary(), undefined)}
, {access_rules, sc(hoconsc:array(string()), [])}
].
@@ -242,8 +247,8 @@ udp_opts() ->
[{udp, sc_meta(ref(udp_opts), #{})}].
proxy_protocol_opts() ->
- [ {proxy_protocol, sc(boolean())}
- , {proxy_protocol_timeout, sc(duration())}
+ [ {proxy_protocol, sc(boolean(), false)}
+ , {proxy_protocol_timeout, sc(duration(), "15s")}
].
sc(Type) ->
diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl
index 6d19cbbcf..8120b5d48 100644
--- a/apps/emqx_gateway/src/emqx_gateway_utils.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl
@@ -117,13 +117,18 @@ format_listenon({Addr, Port}) when is_tuple(Addr) ->
parse_listenon(Port) when is_integer(Port) ->
Port;
+parse_listenon(IpPort) when is_tuple(IpPort) ->
+ IpPort;
parse_listenon(Str) when is_binary(Str) ->
parse_listenon(binary_to_list(Str));
parse_listenon(Str) when is_list(Str) ->
- case emqx_schema:to_ip_port(Str) of
- {ok, R} -> R;
- {error, _} ->
- error({invalid_listenon_name, Str})
+ try list_to_integer(Str)
+ catch _ : _ ->
+ case emqx_schema:to_ip_port(Str) of
+ {ok, R} -> R;
+ {error, _} ->
+ error({invalid_listenon_name, Str})
+ end
end.
listener_id(GwName, Type, LisName) ->
@@ -226,11 +231,7 @@ sock_opts(Name, Opts) ->
%% Envs
active_n(Options) ->
- maps:get(
- active_n,
- maps:get(listener, Options, #{active_n => ?ACTIVE_N}),
- ?ACTIVE_N
- ).
+ maps:get(active_n, Options, ?ACTIVE_N).
-spec idle_timeout(map()) -> pos_integer().
idle_timeout(Options) ->
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
index 3de231958..1cc48e574 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
@@ -139,7 +139,12 @@ init(ConnInfo = #{socktype := Socktype,
GRpcChann = maps:get(handler, Options),
PoolName = maps:get(pool_name, Options),
NConnInfo = default_conninfo(ConnInfo),
- ClientInfo = default_clientinfo(ConnInfo),
+ ListenerId = case maps:get(listener, Options, undefined) of
+ undefined -> undefined;
+ {GwName, Type, LisName} ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName)
+ end,
+ ClientInfo = maps:put(listener, ListenerId, default_clientinfo(ConnInfo)),
Channel = #channel{
ctx = Ctx,
gcli = #{channel => GRpcChann, pool_name => PoolName},
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
index 3e142f3dc..87170fff2 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
@@ -156,6 +156,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
+ listener => {GwName, Type, LisName},
frame_mod => emqx_exproto_frame,
chann_mod => emqx_exproto_channel
},
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl
index 6ad78742f..db610749c 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl
@@ -89,9 +89,15 @@ init(ConnInfo = #{peername := {PeerHost, _},
#{ctx := Ctx} = Config) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Mountpoint = maps:get(mountpoint, Config, undefined),
+ ListenerId = case maps:get(listener, Config, undefined) of
+ undefined -> undefined;
+ {GwName, Type, LisName} ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName)
+ end,
ClientInfo = set_peercert_infos(
Peercert,
#{ zone => default
+ , listener => ListenerId
, protocol => lwm2m
, peerhost => PeerHost
, sockport => SockPort
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
index 649a14643..b5cce573f 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
@@ -102,6 +102,7 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ ctx => Ctx
+ , listener => {GwName, Type, LisName}
, frame_mod => emqx_coap_frame
, chann_mod => emqx_lwm2m_channel
},
diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
index 34e6ec8d6..85921ce79 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
@@ -116,9 +116,15 @@ init(ConnInfo = #{peername := {PeerHost, _},
Registry = maps:get(registry, Option),
GwId = maps:get(gateway_id, Option),
EnableQoS3 = maps:get(enable_qos3, Option, true),
+ ListenerId = case maps:get(listener, Option, undefined) of
+ undefined -> undefined;
+ {GwName, Type, LisName} ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName)
+ end,
ClientInfo = set_peercert_infos(
Peercert,
#{ zone => default
+ , listener => ListenerId
, protocol => 'mqtt-sn'
, peerhost => PeerHost
, sockport => SockPort
diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
index a79173cff..f5660e0dc 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
@@ -121,6 +121,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
+ listene => {GwName, Type, LisName},
frame_mod => emqx_sn_frame,
chann_mod => emqx_sn_channel
},
@@ -138,13 +139,13 @@ merge_default(Options) ->
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
- StopRet = stop_listener(GwName, LisName, Type, ListenOn, SocketOpts, Cfg),
+ StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
- ?ELOG("Failed to stop gatewat ~s:~s:~s on ~s: ~0p~n",
+ ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
index e55e0a580..9a1ca029d 100644
--- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
+++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
@@ -109,10 +109,15 @@ init(ConnInfo = #{peername := {PeerHost, _},
sockname := {_, SockPort}}, Option) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Mountpoint = maps:get(mountpoint, Option, undefined),
+ ListenerId = case maps:get(listener, Option, undefined) of
+ undefined -> undefined;
+ {GwName, Type, LisName} ->
+ emqx_gateway_utils:listener_id(GwName, Type, LisName)
+ end,
ClientInfo = setting_peercert_infos(
Peercert,
#{ zone => default
- , listener => {tcp, default}
+ , listener => ListenerId
, protocol => stomp
, peerhost => PeerHost
, sockport => SockPort
diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
index 9599ef6e3..a93240207 100644
--- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
+++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
@@ -106,6 +106,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
+ listener => {GwName, Type, LisName}, %% Used for authn
frame_mod => emqx_stomp_frame,
chann_mod => emqx_stomp_channel
},
diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_SUITE.erl
index 8e7352a74..f55fdf88c 100644
--- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_coap_SUITE.erl
@@ -34,7 +34,6 @@ gateway.coap
connection_required = true
subscribe_qos = qos1
publish_qos = qos1
- authentication = undefined
listeners.udp.default
{bind = 5683}
@@ -113,24 +112,24 @@ t_publish(_Config) ->
with_connection(Action).
-t_publish_authz_deny(_Config) ->
- Action = fun(Channel, Token) ->
- Topic = <<"/abc">>,
- Payload = <<"123">>,
- InvalidToken = lists:reverse(Token),
-
- TopicStr = binary_to_list(Topic),
- URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ InvalidToken,
-
- %% Sub topic first
- emqx:subscribe(Topic),
-
- Req = make_req(post, Payload),
- Result = do_request(Channel, URI, Req),
- ?assertEqual({error, reset}, Result)
- end,
-
- with_connection(Action).
+%t_publish_authz_deny(_Config) ->
+% Action = fun(Channel, Token) ->
+% Topic = <<"/abc">>,
+% Payload = <<"123">>,
+% InvalidToken = lists:reverse(Token),
+%
+% TopicStr = binary_to_list(Topic),
+% URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ InvalidToken,
+%
+% %% Sub topic first
+% emqx:subscribe(Topic),
+%
+% Req = make_req(post, Payload),
+% Result = do_request(Channel, URI, Req),
+% ?assertEqual({error, reset}, Result)
+% end,
+%
+% with_connection(Action).
t_subscribe(_Config) ->
Topic = <<"/abc">>,
diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl
index 83521f5cd..935a4ec53 100644
--- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl
@@ -25,20 +25,18 @@
-define(CONF_DEFAULT, <<"
gateway.coap {
- idle_timeout = 30s
- enable_stats = false
- mountpoint = \"\"
- notify_type = qos
- connection_required = true
- subscribe_qos = qos1
- publish_qos = qos1
- authentication = undefined
-
- listeners.udp.default {
- bind = 5683
- }
- }
- ">>).
+ idle_timeout = 30s
+ enable_stats = false
+ mountpoint = \"\"
+ notify_type = qos
+ connection_required = true
+ subscribe_qos = qos1
+ publish_qos = qos1
+ listeners.udp.default {
+ bind = 5683
+ }
+}
+">>).
-define(HOST, "127.0.0.1").
-define(PORT, 5683).
@@ -73,7 +71,7 @@ t_send_request_api(_) ->
Payload = <<"simple echo this">>,
Req = #{token => Token,
payload => Payload,
- timeout => 10,
+ timeout => <<"10s">>,
content_type => <<"text/plain">>,
method => <<"get">>},
Auth = emqx_mgmt_api_test_util:auth_header_(),
diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
new file mode 100644
index 000000000..5ffbdb4dd
--- /dev/null
+++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
@@ -0,0 +1,250 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_gateway_conf_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+
+all() ->
+ emqx_ct:all(?MODULE).
+
+init_per_suite(Conf) ->
+ %% FIXME: Magic line. for saving gateway schema name for emqx_config
+ emqx_config:init_load(emqx_gateway_schema, <<"gateway {}">>),
+ emqx_ct_helpers:start_apps([emqx_gateway]),
+ Conf.
+
+end_per_suite(_Conf) ->
+ emqx_ct_helpers:stop_apps([emqx_gateway]).
+
+init_per_testcase(_CaseName, Conf) ->
+ _ = emqx_gateway_conf:remove_gateway(stomp),
+ Conf.
+
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+-define(CONF_STOMP_BAISC_1,
+ #{ <<"idle_timeout">> => <<"10s">>,
+ <<"mountpoint">> => <<"t/">>,
+ <<"frame">> =>
+ #{ <<"max_headers">> => 20,
+ <<"max_headers_length">> => 2000,
+ <<"max_body_length">> => 2000
+ }
+ }).
+-define(CONF_STOMP_BAISC_2,
+ #{ <<"idle_timeout">> => <<"20s">>,
+ <<"mountpoint">> => <<"t2/">>,
+ <<"frame">> =>
+ #{ <<"max_headers">> => 30,
+ <<"max_headers_length">> => 3000,
+ <<"max_body_length">> => 3000
+ }
+ }).
+-define(CONF_STOMP_LISTENER_1,
+ #{ <<"bind">> => <<"61613">>
+ }).
+-define(CONF_STOMP_LISTENER_2,
+ #{ <<"bind">> => <<"61614">>
+ }).
+-define(CONF_STOMP_AUTHN_1,
+ #{ <<"mechanism">> => <<"password-based">>,
+ <<"backend">> => <<"built-in-database">>,
+ <<"user_id_type">> => <<"clientid">>
+ }).
+-define(CONF_STOMP_AUTHN_2,
+ #{ <<"mechanism">> => <<"password-based">>,
+ <<"backend">> => <<"built-in-database">>,
+ <<"user_id_type">> => <<"username">>
+ }).
+
+t_load_remove_gateway(_) ->
+ StompConf1 = compose(?CONF_STOMP_BAISC_1,
+ ?CONF_STOMP_AUTHN_1,
+ ?CONF_STOMP_LISTENER_1
+ ),
+ StompConf2 = compose(?CONF_STOMP_BAISC_2,
+ ?CONF_STOMP_AUTHN_1,
+ ?CONF_STOMP_LISTENER_1),
+
+ ok = emqx_gateway_conf:load_gateway(stomp, StompConf1),
+ {error, already_exist} =
+ emqx_gateway_conf:load_gateway(stomp, StompConf1),
+ assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:update_gateway(stomp, StompConf2),
+ assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:remove_gateway(stomp),
+ ok = emqx_gateway_conf:remove_gateway(stomp),
+
+ {error, not_found} =
+ emqx_gateway_conf:update_gateway(stomp, StompConf2),
+
+ ?assertException(error, {config_not_found, [gateway, stomp]},
+ emqx:get_raw_config([gateway, stomp])),
+ ok.
+
+t_load_remove_authn(_) ->
+ StompConf = compose_listener(?CONF_STOMP_BAISC_1, ?CONF_STOMP_LISTENER_1),
+
+ ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf),
+ assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:add_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_1),
+ assert_confs(
+ maps:put(<<"authentication">>, ?CONF_STOMP_AUTHN_1, StompConf),
+ emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2),
+ assert_confs(
+ maps:put(<<"authentication">>, ?CONF_STOMP_AUTHN_2, StompConf),
+ emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:remove_authn(<<"stomp">>),
+
+ {error, not_found} =
+ emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2),
+
+ ?assertException(
+ error, {config_not_found, [gateway, stomp, authentication]},
+ emqx:get_raw_config([gateway, stomp, authentication])
+ ),
+ ok.
+
+t_load_remove_listeners(_) ->
+ StompConf = compose_authn(?CONF_STOMP_BAISC_1, ?CONF_STOMP_AUTHN_1),
+
+ ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf),
+ assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:add_listener(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_1),
+ assert_confs(
+ maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
+ emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:update_listener(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2),
+ assert_confs(
+ maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_2)),
+ emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:remove_listener(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}),
+
+ {error, not_found} =
+ emqx_gateway_conf:update_listener(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2),
+
+ ?assertException(
+ error, {config_not_found, [gateway, stomp, listeners, tcp, default]},
+ emqx:get_raw_config([gateway, stomp, listeners, tcp, default])
+ ),
+ ok.
+
+t_load_remove_listener_authn(_) ->
+ StompConf = compose_listener(
+ ?CONF_STOMP_BAISC_1,
+ ?CONF_STOMP_LISTENER_1
+ ),
+ StompConf1 = compose_listener_authn(
+ ?CONF_STOMP_BAISC_1,
+ ?CONF_STOMP_LISTENER_1,
+ ?CONF_STOMP_AUTHN_1
+ ),
+ StompConf2 = compose_listener_authn(
+ ?CONF_STOMP_BAISC_1,
+ ?CONF_STOMP_LISTENER_1,
+ ?CONF_STOMP_AUTHN_2
+ ),
+
+ ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf),
+ assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:add_authn(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_1),
+ assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:update_authn(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2),
+ assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])),
+
+ ok = emqx_gateway_conf:remove_authn(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}),
+
+ {error, not_found} =
+ emqx_gateway_conf:update_authn(
+ <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2),
+
+ Path = [gateway, stomp, listeners, tcp, default, authentication],
+ ?assertException(
+ error, {config_not_found, Path},
+ emqx:get_raw_config(Path)
+ ),
+ ok.
+
+%%--------------------------------------------------------------------
+%% Utils
+
+compose(Basic, Authn, Listener) ->
+ maps:merge(
+ maps:merge(Basic, #{<<"authentication">> => Authn}),
+ listener(Listener)).
+
+compose_listener(Basic, Listener) ->
+ maps:merge(Basic, listener(Listener)).
+
+compose_authn(Basic, Authn) ->
+ maps:merge(Basic, #{<<"authentication">> => Authn}).
+
+compose_listener_authn(Basic, Listener, Authn) ->
+ maps:merge(
+ Basic,
+ listener(maps:put(<<"authentication">>, Authn, Listener))).
+
+listener(L) ->
+ #{<<"listeners">> => #{<<"tcp">> => #{<<"default">> => L}}}.
+
+assert_confs(Expected, Effected) ->
+ case do_assert_confs(Expected, Effected) of
+ false ->
+ io:format(standard_error, "Expected config: ~p,\n"
+ "Effected config: ~p",
+ [Expected, Effected]),
+ exit(conf_not_match);
+ true ->
+ ok
+ end.
+
+do_assert_confs(Expected, Effected) when is_map(Expected),
+ is_map(Effected) ->
+ Ks1 = maps:keys(Expected),
+ lists:all(fun(K) ->
+ do_assert_confs(maps:get(K, Expected),
+ maps:get(K, Effected, undefined))
+ end, Ks1);
+do_assert_confs(Expected, Effected) ->
+ Expected =:= Effected.
diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
index 2ee2312df..f36c1e816 100644
--- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
@@ -33,7 +33,7 @@ gateway.lwm2m {
xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"
lifetime_min = 1s
lifetime_max = 86400s
- qmode_time_windonw = 22
+ qmode_time_window = 22
auto_observe = false
mountpoint = \"lwm2m/%u\"
update_msg_publish_condition = contains_object_list
diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl
index 081f11005..a875aceb6 100644
--- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl
@@ -33,7 +33,7 @@ gateway.lwm2m {
xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"
lifetime_min = 1s
lifetime_max = 86400s
- qmode_time_windonw = 22
+ qmode_time_window = 22
auto_observe = false
mountpoint = \"lwm2m/%u\"
update_msg_publish_condition = contains_object_list
diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl
index 97125d79f..5a717c5e7 100644
--- a/apps/emqx_machine/src/emqx_machine.erl
+++ b/apps/emqx_machine/src/emqx_machine.erl
@@ -35,8 +35,12 @@
%% @doc EMQ X boot entrypoint.
start() ->
- os:set_signal(sighup, ignore),
- os:set_signal(sigterm, handle), %% default is handle
+ case os:type() of
+ {win32, nt} -> ok;
+ _nix ->
+ os:set_signal(sighup, ignore),
+ os:set_signal(sigterm, handle) %% default is handle
+ end,
ok = set_backtrace_depth(),
ok = print_otp_version_warning(),
@@ -146,7 +150,6 @@ reboot_apps() ->
, emqx_management
, emqx_retainer
, emqx_exhook
- , emqx_rule_actions
, emqx_authn
, emqx_authz
].
diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl
index 1637ca0e6..1a4b4a975 100644
--- a/apps/emqx_machine/src/emqx_machine_schema.erl
+++ b/apps/emqx_machine/src/emqx_machine_schema.erl
@@ -102,7 +102,7 @@ fields("cluster") ->
, default => emqxcl
})}
, {"discovery_strategy",
- sc(union([manual, static, mcast, dns, etcd, k8s]),
+ sc(hoconsc:enum([manual, static, mcast, dns, etcd, k8s]),
#{ default => manual
})}
, {"autoclean",
@@ -122,7 +122,7 @@ fields("cluster") ->
sc(ref(cluster_mcast),
#{})}
, {"proto_dist",
- sc(union([inet_tcp, inet6_tcp, inet_tls]),
+ sc(hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]),
#{ mapping => "ekka.proto_dist"
, default => inet_tcp
})}
@@ -136,7 +136,7 @@ fields("cluster") ->
sc(ref(cluster_k8s),
#{})}
, {"db_backend",
- sc(union([mnesia, rlog]),
+ sc(hoconsc:enum([mnesia, rlog]),
#{ mapping => "ekka.db_backend"
, default => mnesia
})}
@@ -224,7 +224,7 @@ fields(cluster_k8s) ->
#{ default => "emqx"
})}
, {"address_type",
- sc(union([ip, dns, hostname]),
+ sc(hoconsc:enum([ip, dns, hostname]),
#{})}
, {"app_name",
sc(string(),
@@ -242,7 +242,7 @@ fields(cluster_k8s) ->
fields("rlog") ->
[ {"role",
- sc(union([core, replicant]),
+ sc(hoconsc:enum([core, replicant]),
#{ mapping => "ekka.node_role"
, default => core
})}
@@ -334,7 +334,7 @@ fields("cluster_call") ->
fields("rpc") ->
[ {"mode",
- sc(union(sync, async),
+ sc(hoconsc:enum([sync, async]),
#{ default => async
})}
, {"async_batch_size",
@@ -343,7 +343,7 @@ fields("rpc") ->
, default => 256
})}
, {"port_discovery",
- sc(union(manual, stateless),
+ sc(hoconsc:enum([manual, stateless]),
#{ mapping => "gen_rpc.port_discovery"
, default => stateless
})}
@@ -434,7 +434,7 @@ fields("log_file_handler") ->
sc(ref("log_rotation"),
#{})}
, {"max_size",
- sc(union([infinity, emqx_schema:bytesize()]),
+ sc(hoconsc:union([infinity, emqx_schema:bytesize()]),
#{ default => "10MB"
})}
] ++ log_handler_common_confs();
@@ -464,7 +464,7 @@ fields("log_overload_kill") ->
#{ default => 20000
})}
, {"restart_after",
- sc(union(emqx_schema:duration(), infinity),
+ sc(hoconsc:union([emqx_schema:duration(), infinity]),
#{ default => "5s"
})}
];
@@ -582,7 +582,7 @@ log_handler_common_confs() ->
#{ default => unlimited
})}
, {"formatter",
- sc(union([text, json]),
+ sc(hoconsc:enum([text, json]),
#{ default => text
})}
, {"single_line",
@@ -608,11 +608,11 @@ log_handler_common_confs() ->
sc(ref("log_burst_limit"),
#{})}
, {"supervisor_reports",
- sc(union([error, progress]),
+ sc(hoconsc:enum([error, progress]),
#{ default => error
})}
, {"max_depth",
- sc(union([unlimited, integer()]),
+ sc(hoconsc:union([unlimited, integer()]),
#{ default => 100
})}
].
diff --git a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl
index d39dc3c6e..b1530afd2 100644
--- a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl
+++ b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl
@@ -124,7 +124,7 @@ t_catch_up_status_handle_next_commit(_Config) ->
t_commit_ok_apply_fail_on_other_node_then_recover(_Config) ->
emqx_cluster_rpc:reset(),
{atomic, []} = emqx_cluster_rpc:status(),
- Now = erlang:system_time(second),
+ Now = erlang:system_time(millisecond),
{M, F, A} = {?MODULE, failed_on_other_recover_after_5_second, [erlang:whereis(?NODE1), Now]},
{ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
{ok, _, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000),
@@ -132,10 +132,10 @@ t_commit_ok_apply_fail_on_other_node_then_recover(_Config) ->
?assertEqual([], L),
?assertEqual({io, format, ["test"]}, maps:get(mfa, Status)),
?assertEqual(node(), maps:get(node, Status)),
- sleep(3000),
+ sleep(2300),
{atomic, [Status1]} = emqx_cluster_rpc:status(),
?assertEqual(Status, Status1),
- sleep(2600),
+ sleep(3600),
{atomic, NewStatus} = emqx_cluster_rpc:status(),
?assertEqual(3, length(NewStatus)),
Pid = self(),
@@ -243,11 +243,11 @@ failed_on_node_by_odd(Pid) ->
end.
failed_on_other_recover_after_5_second(Pid, CreatedAt) ->
- Now = erlang:system_time(second),
+ Now = erlang:system_time(millisecond),
case Pid =:= self() of
true -> ok;
false ->
- case Now < CreatedAt + 5 of
+ case Now < CreatedAt + 5001 of
true -> "MFA return not ok";
false -> ok
end
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 8cf2fa1cb..e13f52691 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -18,7 +18,9 @@
-include_lib("stdlib/include/qlc.hrl").
--export([paginate/3]).
+-export([ paginate/3
+ , paginate/4
+ ]).
%% first_next query APIs
-export([ params2qs/2
@@ -47,6 +49,23 @@ paginate(Tables, Params, RowFun) ->
#{meta => #{page => Page, limit => Limit, count => Count},
data => [RowFun(Row) || Row <- Rows]}.
+paginate(Tables, MatchSpec, Params, RowFun) ->
+ Qh = query_handle(Tables, MatchSpec),
+ Count = count(Tables, MatchSpec),
+ Page = b2i(page(Params)),
+ Limit = b2i(limit(Params)),
+ Cursor = qlc:cursor(Qh),
+ case Page > 1 of
+ true ->
+ _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
+ ok;
+ false -> ok
+ end,
+ Rows = qlc:next_answers(Cursor, Limit),
+ qlc:delete_cursor(Cursor),
+ #{meta => #{page => Page, limit => Limit, count => Count},
+ data => [RowFun(Row) || Row <- Rows]}.
+
query_handle(Table) when is_atom(Table) ->
qlc:q([R|| R <- ets:table(Table)]);
query_handle([Table]) when is_atom(Table) ->
@@ -54,6 +73,16 @@ query_handle([Table]) when is_atom(Table) ->
query_handle(Tables) ->
qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]).
+query_handle(Table, MatchSpec) when is_atom(Table) ->
+ Options = {traverse, {select, MatchSpec}},
+ qlc:q([R|| R <- ets:table(Table, Options)]);
+query_handle([Table], MatchSpec) when is_atom(Table) ->
+ Options = {traverse, {select, MatchSpec}},
+ qlc:q([R|| R <- ets:table(Table, Options)]);
+query_handle(Tables, MatchSpec) ->
+ Options = {traverse, {select, MatchSpec}},
+ qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
+
count(Table) when is_atom(Table) ->
ets:info(Table, size);
count([Table]) when is_atom(Table) ->
@@ -61,8 +90,16 @@ count([Table]) when is_atom(Table) ->
count(Tables) ->
lists:sum([count(T) || T <- Tables]).
-count(Table, Nodes) ->
- lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]).
+count(Table, MatchSpec) when is_atom(Table) ->
+ [{MatchPattern, Where, _Re}] = MatchSpec,
+ NMatchSpec = [{MatchPattern, Where, [true]}],
+ ets:select_count(Table, NMatchSpec);
+count([Table], MatchSpec) when is_atom(Table) ->
+ [{MatchPattern, Where, _Re}] = MatchSpec,
+ NMatchSpec = [{MatchPattern, Where, [true]}],
+ ets:select_count(Table, NMatchSpec);
+count(Tables, MatchSpec) ->
+ lists:sum([count(T, MatchSpec) || T <- Tables]).
page(Params) when is_map(Params) ->
maps:get(<<"page">>, Params, 1);
@@ -122,7 +159,7 @@ cluster_query(Params, Tab, QsSchema, QueryFun) ->
Rows = do_cluster_query(Nodes, Tab, Qs, QueryFun, Start, Limit+1, []),
Meta = #{page => Page, limit => Limit},
NMeta = case CodCnt =:= 0 of
- true -> Meta#{count => count(Tab, Nodes)};
+ true -> Meta#{count => lists:sum([rpc_call(Node, ets, info, [Tab, size], 5000) || Node <- Nodes])};
_ -> Meta#{count => length(Rows)}
end,
#{meta => NMeta, data => lists:sublist(Rows, Limit)}.
diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl
index 5975dfdd4..91ac29474 100644
--- a/apps/emqx_modules/src/emqx_delayed_api.erl
+++ b/apps/emqx_modules/src/emqx_delayed_api.erl
@@ -18,22 +18,19 @@
-behavior(minirest_api).
--import(emqx_mgmt_util, [ page_params/0
- , schema/1
- , schema/2
- , object_schema/2
- , error_schema/2
- , page_object_schema/1
- , properties/1
- ]).
+-include_lib("typerefl/include/types.hrl").
+
+-import(hoconsc, [mk/2, ref/1, ref/2]).
-define(MAX_PAYLOAD_LENGTH, 2048).
-define(PAYLOAD_TOO_LARGE, 'PAYLOAD_TOO_LARGE').
--export([ status/2
- , delayed_messages/2
- , delayed_message/2
- ]).
+-export([status/2
+ , delayed_messages/2
+ , delayed_message/2
+]).
+
+-export([paths/0, fields/1, schema/1]).
%% for rpc
-export([update_config_/1]).
@@ -49,91 +46,94 @@
-define(MESSAGE_ID_SCHEMA_ERROR, 'MESSAGE_ID_SCHEMA_ERROR').
api_spec() ->
- {
- [status_api(), delayed_messages_api(), delayed_message_api()],
- []
- }.
+ emqx_dashboard_swagger:spec(?MODULE).
-conf_schema() ->
- emqx_mgmt_api_configs:gen_schema(emqx:get_raw_config([delayed])).
-properties() ->
- PayloadDesc = io_lib:format("Payload, base64 encode. Payload will be ~p if length large than ~p",
- [?PAYLOAD_TOO_LARGE, ?MAX_PAYLOAD_LENGTH]),
- properties([
- {msgid, integer, <<"Message Id">>},
- {publish_at, string, <<"Client publish message time, rfc 3339">>},
- {delayed_interval, integer, <<"Delayed interval, second">>},
- {delayed_remaining, integer, <<"Delayed remaining, second">>},
- {expected_at, string, <<"Expect publish time, rfc 3339">>},
- {topic, string, <<"Topic">>},
- {qos, string, <<"QoS">>},
- {payload, string, iolist_to_binary(PayloadDesc)},
- {from_clientid, string, <<"From ClientId">>},
- {from_username, string, <<"From Username">>}
- ]).
+paths() -> ["/mqtt/delayed", "/mqtt/delayed/messages", "/mqtt/delayed/messages/:msgid"].
-parameters() ->
- [#{
- name => msgid,
- in => path,
- schema => #{type => string},
- required => true
- }].
-
-status_api() ->
- Metadata = #{
+schema("/mqtt/delayed") ->
+ #{
+ operationId => status,
get => #{
+ tags => [<<"mqtt">>],
description => <<"Get delayed status">>,
+ summary => <<"Get delayed status">>,
responses => #{
- <<"200">> => schema(conf_schema())}
- },
+ 200 => ref(emqx_modules_schema, "delayed")
+ }
+ },
put => #{
+ tags => [<<"mqtt">>],
description => <<"Enable or disable delayed, set max delayed messages">>,
- 'requestBody' => schema(conf_schema()),
+ requestBody => ref(emqx_modules_schema, "delayed"),
responses => #{
- <<"200">> =>
- schema(conf_schema(), <<"Enable or disable delayed successfully">>),
- <<"400">> =>
- error_schema(<<"Max limit illegality">>, [?BAD_REQUEST])
+ 200 => mk(ref(emqx_modules_schema, "delayed"),
+ #{desc => <<"Enable or disable delayed successfully">>}),
+ 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Max limit illegality">>)
}
}
- },
- {"/mqtt/delayed", Metadata, status}.
+ };
-delayed_messages_api() ->
- Metadata = #{
- get => #{
- description => "List delayed messages",
- parameters => page_params(),
- responses => #{
- <<"200">> => page_object_schema(properties())
- }
- }
- },
- {"/mqtt/delayed/messages", Metadata, delayed_messages}.
-
-delayed_message_api() ->
- Metadata = #{
+schema("/mqtt/delayed/messages/:msgid") ->
+ #{operationId => delayed_message,
get => #{
+ tags => [<<"mqtt">>],
description => <<"Get delayed message">>,
- parameters => parameters(),
+ parameters => [{msgid, mk(binary(), #{in => path, desc => <<"delay message ID">>})}],
responses => #{
- <<"400">> => error_schema(<<"Message ID Schema error">>, [?MESSAGE_ID_SCHEMA_ERROR]),
- <<"404">> => error_schema(<<"Message ID not found">>, [?MESSAGE_ID_NOT_FOUND]),
- <<"200">> => object_schema(maps:without([payload], properties()), <<"Get delayed message success">>)
+ 200 => ref("message_without_payload"),
+ 400 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_SCHEMA_ERROR], <<"Bad MsgId format">>),
+ 404 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_NOT_FOUND], <<"MsgId not found">>)
}
},
delete => #{
+ tags => [<<"mqtt">>],
description => <<"Delete delayed message">>,
- parameters => parameters(),
+ parameters => [{msgid, mk(binary(), #{in => path, desc => <<"delay message ID">>})}],
responses => #{
- <<"400">> => error_schema(<<"Message ID Schema error">>, [?MESSAGE_ID_SCHEMA_ERROR]),
- <<"404">> => error_schema(<<"Message ID not found">>, [?MESSAGE_ID_NOT_FOUND]),
- <<"200">> => schema(<<"Delete delayed message success">>)
+ 200 => <<"Delete delayed message success">>,
+ 400 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_SCHEMA_ERROR], <<"Bad MsgId format">>),
+ 404 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_NOT_FOUND], <<"MsgId not found">>)
}
}
- },
- {"/mqtt/delayed/messages/:msgid", Metadata, delayed_message}.
+ };
+schema("/mqtt/delayed/messages") ->
+ #{
+ operationId => delayed_messages,
+ get => #{
+ tags => [<<"mqtt">>],
+ description => <<"List delayed messages">>,
+ parameters => [ref(emqx_dashboard_swagger, page), ref(emqx_dashboard_swagger, limit)],
+ responses => #{
+ 200 =>
+ [
+ {data, mk(hoconsc:array(ref("message")), #{})},
+ {meta, [
+ {page, mk(integer(), #{})},
+ {limit, mk(integer(), #{})},
+ {count, mk(integer(), #{})}
+ ]}
+ ]
+ }
+ }
+ }.
+
+fields("message_without_payload") ->
+ [
+ {msgid, mk(integer(), #{desc => <<"Message Id (MQTT message id hash)">>})},
+ {publish_at, mk(binary(), #{desc => <<"Client publish message time, rfc 3339">>})},
+ {delayed_interval, mk(integer(), #{desc => <<"Delayed interval, second">>})},
+ {delayed_remaining, mk(integer(), #{desc => <<"Delayed remaining, second">>})},
+ {expected_at, mk(binary(), #{desc => <<"Expect publish time, rfc 3339">>})},
+ {topic, mk(binary(), #{desc => <<"Topic">>, example => <<"/sys/#">>})},
+ {qos, mk(binary(), #{desc => <<"QoS">>})},
+ {from_clientid, mk(binary(), #{desc => <<"From ClientId">>})},
+ {from_username, mk(binary(), #{desc => <<"From Username">>})}
+ ];
+fields("message") ->
+ PayloadDesc = io_lib:format("Payload, base64 encode. Payload will be ~p if length large than ~p",
+ [?PAYLOAD_TOO_LARGE, ?MAX_PAYLOAD_LENGTH]),
+ fields("message_without_payload") ++
+ [{payload, mk(binary(), #{desc => iolist_to_binary(PayloadDesc)})}].
%%--------------------------------------------------------------------
%% HTTP API
@@ -210,7 +210,7 @@ generate_max_delayed_messages(Config) ->
update_config_(Config) ->
lists:foreach(fun(Node) ->
update_config_(Node, Config)
- end, ekka_mnesia:running_nodes()).
+ end, ekka_mnesia:running_nodes()).
update_config_(Node, Config) when Node =:= node() ->
_ = emqx_delayed:update_config(Config),
diff --git a/apps/emqx_rule_actions/README.md b/apps/emqx_rule_actions/README.md
deleted file mode 100644
index c17e1a34a..000000000
--- a/apps/emqx_rule_actions/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# emqx_rule_actions
-
-This project contains a collection of rule actions/resources. It is mainly for
- making unit test easier. Also it's easier for us to create utils that many
- modules depends on it.
-
-## Build
------
-
- $ rebar3 compile
-
diff --git a/apps/emqx_rule_actions/rebar.config b/apps/emqx_rule_actions/rebar.config
deleted file mode 100644
index 097c18a3d..000000000
--- a/apps/emqx_rule_actions/rebar.config
+++ /dev/null
@@ -1,25 +0,0 @@
-{deps, []}.
-
-{erl_opts, [warn_unused_vars,
- warn_shadow_vars,
- warn_unused_import,
- warn_obsolete_guard,
- no_debug_info,
- compressed, %% for edge
- {parse_transform}
- ]}.
-
-{overrides, [{add, [{erl_opts, [no_debug_info, compressed]}]}]}.
-
-{edoc_opts, [{preprocess, true}]}.
-
-{xref_checks, [undefined_function_calls, undefined_functions,
- locals_not_used, deprecated_function_calls,
- warnings_as_errors, deprecated_functions
- ]}.
-
-{cover_enabled, true}.
-{cover_opts, [verbose]}.
-{cover_export_enabled, true}.
-
-{plugins, [rebar3_proper]}.
diff --git a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl
deleted file mode 100644
index ce1192579..000000000
--- a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl
+++ /dev/null
@@ -1,576 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol
-
--module(emqx_bridge_mqtt_actions).
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx_rule_engine/include/rule_actions.hrl").
-
--import(emqx_plugin_libs_rule, [str/1]).
-
--export([ on_resource_create/2
- , on_get_resource_status/2
- , on_resource_destroy/2
- ]).
-
-%% Callbacks of ecpool Worker
--export([connect/1]).
-
--export([subscriptions/1]).
-
--export([ on_action_create_data_to_mqtt_broker/2
- , on_action_data_to_mqtt_broker/2
- ]).
-
--define(RESOURCE_TYPE_MQTT, 'bridge_mqtt').
--define(RESOURCE_TYPE_RPC, 'bridge_rpc').
-
--define(RESOURCE_CONFIG_SPEC_MQTT, #{
- address => #{
- order => 1,
- type => string,
- required => true,
- default => <<"127.0.0.1:1883">>,
- title => #{en => <<" Broker Address">>,
- zh => <<"远程 broker 地址"/utf8>>},
- description => #{en => <<"The MQTT Remote Address">>,
- zh => <<"远程 MQTT Broker 的地址"/utf8>>}
- },
- pool_size => #{
- order => 2,
- type => number,
- required => true,
- default => 8,
- title => #{en => <<"Pool Size">>,
- zh => <<"连接池大小"/utf8>>},
- description => #{en => <<"MQTT Connection Pool Size">>,
- zh => <<"连接池大小"/utf8>>}
- },
- clientid => #{
- order => 3,
- type => string,
- required => true,
- default => <<"client">>,
- title => #{en => <<"ClientId">>,
- zh => <<"客户端 Id"/utf8>>},
- description => #{en => <<"ClientId for connecting to remote MQTT broker">>,
- zh => <<"连接远程 Broker 的 ClientId"/utf8>>}
- },
- append => #{
- order => 4,
- type => boolean,
- required => false,
- default => true,
- title => #{en => <<"Append GUID">>,
- zh => <<"附加 GUID"/utf8>>},
- description => #{en => <<"Append GUID to MQTT ClientId?">>,
- zh => <<"是否将GUID附加到 MQTT ClientId 后"/utf8>>}
- },
- username => #{
- order => 5,
- type => string,
- required => false,
- default => <<"">>,
- title => #{en => <<"Username">>, zh => <<"用户名"/utf8>>},
- description => #{en => <<"Username for connecting to remote MQTT Broker">>,
- zh => <<"连接远程 Broker 的用户名"/utf8>>}
- },
- password => #{
- order => 6,
- type => password,
- required => false,
- default => <<"">>,
- title => #{en => <<"Password">>,
- zh => <<"密码"/utf8>>},
- description => #{en => <<"Password for connecting to remote MQTT Broker">>,
- zh => <<"连接远程 Broker 的密码"/utf8>>}
- },
- mountpoint => #{
- order => 7,
- type => string,
- required => false,
- default => <<"bridge/aws/${node}/">>,
- title => #{en => <<"Bridge MountPoint">>,
- zh => <<"桥接挂载点"/utf8>>},
- description => #{
- en => <<"MountPoint for bridge topic:
"
- "Example: The topic of messages sent to `topic1` on local node "
- "will be transformed to `bridge/aws/${node}/topic1`">>,
- zh => <<"桥接主题的挂载点:
"
- "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题"
- "会变换为 `bridge/aws/${node}/topic1`"/utf8>>
- }
- },
- disk_cache => #{
- order => 8,
- type => boolean,
- required => false,
- default => false,
- title => #{en => <<"Disk Cache">>,
- zh => <<"磁盘缓存"/utf8>>},
- description => #{en => <<"The flag which determines whether messages "
- "can be cached on local disk when bridge is "
- "disconnected">>,
- zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁"
- "盘队列上"/utf8>>}
- },
- proto_ver => #{
- order => 9,
- type => string,
- required => false,
- default => <<"mqttv4">>,
- enum => [<<"mqttv3">>, <<"mqttv4">>, <<"mqttv5">>],
- title => #{en => <<"Protocol Version">>,
- zh => <<"协议版本"/utf8>>},
- description => #{en => <<"MQTTT Protocol version">>,
- zh => <<"MQTT 协议版本"/utf8>>}
- },
- keepalive => #{
- order => 10,
- type => string,
- required => false,
- default => <<"60s">> ,
- title => #{en => <<"Keepalive">>,
- zh => <<"心跳间隔"/utf8>>},
- description => #{en => <<"Keepalive">>,
- zh => <<"心跳间隔"/utf8>>}
- },
- reconnect_interval => #{
- order => 11,
- type => string,
- required => false,
- default => <<"30s">>,
- title => #{en => <<"Reconnect Interval">>,
- zh => <<"重连间隔"/utf8>>},
- description => #{en => <<"Reconnect interval of bridge:
">>,
- zh => <<"重连间隔"/utf8>>}
- },
- retry_interval => #{
- order => 12,
- type => string,
- required => false,
- default => <<"20s">>,
- title => #{en => <<"Retry interval">>,
- zh => <<"重传间隔"/utf8>>},
- description => #{en => <<"Retry interval for bridge QoS1 message delivering">>,
- zh => <<"消息重传间隔"/utf8>>}
- },
- bridge_mode => #{
- order => 13,
- type => boolean,
- required => false,
- default => false,
- title => #{en => <<"Bridge Mode">>,
- zh => <<"桥接模式"/utf8>>},
- description => #{en => <<"Bridge mode for MQTT bridge connection">>,
- zh => <<"MQTT 连接是否为桥接模式"/utf8>>}
- },
- ssl => #{
- order => 14,
- type => boolean,
- default => false,
- title => #{en => <<"Enable SSL">>,
- zh => <<"开启SSL链接"/utf8>>},
- description => #{en => <<"Enable SSL or not">>,
- zh => <<"是否开启 SSL"/utf8>>}
- },
- cacertfile => #{
- order => 15,
- type => file,
- required => false,
- default => <<"etc/certs/cacert.pem">>,
- title => #{en => <<"CA certificates">>,
- zh => <<"CA 证书"/utf8>>},
- description => #{en => <<"The file path of the CA certificates">>,
- zh => <<"CA 证书路径"/utf8>>}
- },
- certfile => #{
- order => 16,
- type => file,
- required => false,
- default => <<"etc/certs/client-cert.pem">>,
- title => #{en => <<"SSL Certfile">>,
- zh => <<"SSL 客户端证书"/utf8>>},
- description => #{en => <<"The file path of the client certfile">>,
- zh => <<"客户端证书路径"/utf8>>}
- },
- keyfile => #{
- order => 17,
- type => file,
- required => false,
- default => <<"etc/certs/client-key.pem">>,
- title => #{en => <<"SSL Keyfile">>,
- zh => <<"SSL 密钥文件"/utf8>>},
- description => #{en => <<"The file path of the client keyfile">>,
- zh => <<"客户端密钥路径"/utf8>>}
- },
- ciphers => #{
- order => 18,
- type => string,
- required => false,
- default => <<"ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,",
- "ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,",
- "ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,",
- "ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,",
- "AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,",
- "ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,",
- "ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,",
- "DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,",
- "ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,",
- "ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,",
- "DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA">>,
- title => #{en => <<"SSL Ciphers">>,
- zh => <<"SSL 加密算法"/utf8>>},
- description => #{en => <<"SSL Ciphers">>,
- zh => <<"SSL 加密算法"/utf8>>}
- }
- }).
-
--define(RESOURCE_CONFIG_SPEC_RPC, #{
- address => #{
- order => 1,
- type => string,
- required => true,
- default => <<"emqx2@127.0.0.1">>,
- title => #{en => <<"EMQ X Node Name">>,
- zh => <<"EMQ X 节点名称"/utf8>>},
- description => #{en => <<"EMQ X Remote Node Name">>,
- zh => <<"远程 EMQ X 节点名称 "/utf8>>}
- },
- mountpoint => #{
- order => 2,
- type => string,
- required => false,
- default => <<"bridge/emqx/${node}/">>,
- title => #{en => <<"Bridge MountPoint">>,
- zh => <<"桥接挂载点"/utf8>>},
- description => #{en => <<"MountPoint for bridge topic
"
- "Example: The topic of messages sent to `topic1` on local node "
- "will be transformed to `bridge/aws/${node}/topic1`">>,
- zh => <<"桥接主题的挂载点
"
- "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题"
- "会变换为 `bridge/aws/${node}/topic1`"/utf8>>}
- },
- pool_size => #{
- order => 3,
- type => number,
- required => true,
- default => 8,
- title => #{en => <<"Pool Size">>,
- zh => <<"连接池大小"/utf8>>},
- description => #{en => <<"MQTT/RPC Connection Pool Size">>,
- zh => <<"连接池大小"/utf8>>}
- },
- reconnect_interval => #{
- order => 4,
- type => string,
- required => false,
- default => <<"30s">>,
- title => #{en => <<"Reconnect Interval">>,
- zh => <<"重连间隔"/utf8>>},
- description => #{en => <<"Reconnect Interval of bridge">>,
- zh => <<"重连间隔"/utf8>>}
- },
- batch_size => #{
- order => 5,
- type => number,
- required => false,
- default => 32,
- title => #{en => <<"Batch Size">>,
- zh => <<"批处理大小"/utf8>>},
- description => #{en => <<"Batch Size">>,
- zh => <<"批处理大小"/utf8>>}
- },
- disk_cache => #{
- order => 6,
- type => boolean,
- required => false,
- default => false,
- title => #{en => <<"Disk Cache">>,
- zh => <<"磁盘缓存"/utf8>>},
- description => #{en => <<"The flag which determines whether messages "
- "can be cached on local disk when bridge is "
- "disconnected">>,
- zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁"
- "盘队列上"/utf8>>}
- }
- }).
-
--define(ACTION_PARAM_RESOURCE, #{
- type => string,
- required => true,
- title => #{en => <<"Resource ID">>, zh => <<"资源 ID"/utf8>>},
- description => #{en => <<"Bind a resource to this action">>,
- zh => <<"给动作绑定一个资源"/utf8>>}
- }).
-
--resource_type(#{
- name => ?RESOURCE_TYPE_MQTT,
- create => on_resource_create,
- status => on_get_resource_status,
- destroy => on_resource_destroy,
- params => ?RESOURCE_CONFIG_SPEC_MQTT,
- title => #{en => <<"MQTT Bridge">>, zh => <<"MQTT Bridge"/utf8>>},
- description => #{en => <<"MQTT Message Bridge">>, zh => <<"MQTT 消息桥接"/utf8>>}
- }).
-
-
--resource_type(#{
- name => ?RESOURCE_TYPE_RPC,
- create => on_resource_create,
- status => on_get_resource_status,
- destroy => on_resource_destroy,
- params => ?RESOURCE_CONFIG_SPEC_RPC,
- title => #{en => <<"EMQX Bridge">>, zh => <<"EMQX Bridge"/utf8>>},
- description => #{en => <<"EMQ X RPC Bridge">>, zh => <<"EMQ X RPC 消息桥接"/utf8>>}
- }).
-
--rule_action(#{
- name => data_to_mqtt_broker,
- category => data_forward,
- for => 'message.publish',
- types => [?RESOURCE_TYPE_MQTT, ?RESOURCE_TYPE_RPC],
- create => on_action_create_data_to_mqtt_broker,
- params => #{'$resource' => ?ACTION_PARAM_RESOURCE,
- forward_topic => #{
- order => 1,
- type => string,
- required => false,
- default => <<"">>,
- title => #{en => <<"Forward Topic">>,
- zh => <<"转发消息主题"/utf8>>},
- description => #{en => <<"The topic used when forwarding the message. "
- "Defaults to the topic of the bridge message if not provided.">>,
- zh => <<"转发消息时使用的主题。如果未提供，则默认为桥接消息的主题。"/utf8>>}
- },
- payload_tmpl => #{
- order => 2,
- type => string,
- input => textarea,
- required => false,
- default => <<"">>,
- title => #{en => <<"Payload Template">>,
- zh => <<"消息内容模板"/utf8>>},
- description => #{en => <<"The payload template, variable interpolation is supported. "
- "If using empty template (default), then the payload will be "
- "all the available vars in JSON format">>,
- zh => <<"消息内容模板，支持变量。"
- "若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>}
- }
- },
- title => #{en => <<"Data bridge to MQTT Broker">>,
- zh => <<"桥接数据到 MQTT Broker"/utf8>>},
- description => #{en => <<"Bridge Data to MQTT Broker">>,
- zh => <<"桥接数据到 MQTT Broker"/utf8>>}
- }).
-
-on_resource_create(ResId, Params) ->
- ?LOG(info, "Initiating Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]),
- {ok, _} = application:ensure_all_started(ecpool),
- PoolName = pool_name(ResId),
- Options = options(Params, PoolName, ResId),
- start_resource(ResId, PoolName, Options),
- case test_resource_status(PoolName) of
- true -> ok;
- false ->
- on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
- error({{?RESOURCE_TYPE_MQTT, ResId}, connection_failed})
- end,
- #{<<"pool">> => PoolName}.
-
-start_resource(ResId, PoolName, Options) ->
- case ecpool:start_sup_pool(PoolName, ?MODULE, Options) of
- {ok, _} ->
- ?LOG(info, "Initiated Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]);
- {error, {already_started, _Pid}} ->
- on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
- start_resource(ResId, PoolName, Options);
- {error, Reason} ->
- ?LOG(error, "Initiate Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]),
- on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
- error({{?RESOURCE_TYPE_MQTT, ResId}, create_failed})
- end.
-
-test_resource_status(PoolName) ->
- IsConnected = fun(Worker) ->
- case ecpool_worker:client(Worker) of
- {ok, Bridge} ->
- try emqx_connector_mqtt_worker:status(Bridge) of
- connected -> true;
- _ -> false
- catch _Error:_Reason ->
- false
- end;
- {error, _} ->
- false
- end
- end,
- Status = [IsConnected(Worker) || {_WorkerName, Worker} <- ecpool:workers(PoolName)],
- lists:any(fun(St) -> St =:= true end, Status).
-
--spec(on_get_resource_status(ResId::binary(), Params::map()) -> Status::map()).
-on_get_resource_status(_ResId, #{<<"pool">> := PoolName}) ->
- IsAlive = test_resource_status(PoolName),
- #{is_alive => IsAlive}.
-
-on_resource_destroy(ResId, #{<<"pool">> := PoolName}) ->
- ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]),
- case ecpool:stop_sup_pool(PoolName) of
- ok ->
- ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]);
- {error, Reason} ->
- ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]),
- error({{?RESOURCE_TYPE_MQTT, ResId}, destroy_failed})
- end.
-
-on_action_create_data_to_mqtt_broker(ActId, Opts = #{<<"pool">> := PoolName,
- <<"forward_topic">> := ForwardTopic,
- <<"payload_tmpl">> := PayloadTmpl}) ->
- ?LOG(info, "Initiating Action ~p.", [?FUNCTION_NAME]),
- PayloadTks = emqx_plugin_libs_rule:preproc_tmpl(PayloadTmpl),
- TopicTks = case ForwardTopic == <<"">> of
- true -> undefined;
- false -> emqx_plugin_libs_rule:preproc_tmpl(ForwardTopic)
- end,
- Opts.
-
-on_action_data_to_mqtt_broker(Msg, _Env =
- #{id := Id, clientid := From, flags := Flags,
- topic := Topic, timestamp := TimeStamp, qos := QoS,
- ?BINDING_KEYS := #{
- 'ActId' := ActId,
- 'PoolName' := PoolName,
- 'TopicTks' := TopicTks,
- 'PayloadTks' := PayloadTks
- }}) ->
- Topic1 = case TopicTks =:= undefined of
- true -> Topic;
- false -> emqx_plugin_libs_rule:proc_tmpl(TopicTks, Msg)
- end,
- BrokerMsg = #message{id = Id,
- qos = QoS,
- from = From,
- flags = Flags,
- topic = Topic1,
- payload = format_data(PayloadTks, Msg),
- timestamp = TimeStamp},
- ecpool:with_client(PoolName,
- fun(BridgePid) ->
- BridgePid ! {deliver, rule_engine, BrokerMsg}
- end),
- emqx_rule_metrics:inc_actions_success(ActId).
-
-format_data([], Msg) ->
- emqx_json:encode(Msg);
-
-format_data(Tokens, Msg) ->
- emqx_plugin_libs_rule:proc_tmpl(Tokens, Msg).
-
-subscriptions(Subscriptions) ->
- scan_binary(<<"[", Subscriptions/binary, "].">>).
-
-is_node_addr(Addr0) ->
- Addr = binary_to_list(Addr0),
- case string:tokens(Addr, "@") of
- [_NodeName, _Hostname] -> true;
- _ -> false
- end.
-
-scan_binary(Bin) ->
- TermString = binary_to_list(Bin),
- scan_string(TermString).
-
-scan_string(TermString) ->
- {ok, Tokens, _} = erl_scan:string(TermString),
- {ok, Term} = erl_parse:parse_term(Tokens),
- Term.
-
-connect(Options) when is_list(Options) ->
- connect(maps:from_list(Options));
-connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name := Pool}) ->
- Options0 = case DiskCache of
- true ->
- DataDir = filename:join([emqx:get_config([node, data_dir]), replayq, Pool, integer_to_list(Id)]),
- QueueOption = #{replayq_dir => DataDir},
- Options#{queue => QueueOption};
- false ->
- Options
- end,
- Options1 = case maps:is_key(append, Options0) of
- false -> Options0;
- true ->
- case maps:get(append, Options0, false) of
- true ->
- ClientId = lists:concat([str(maps:get(clientid, Options0)), "_", str(emqx_guid:to_hexstr(emqx_guid:gen()))]),
- Options0#{clientid => ClientId};
- false ->
- Options0
- end
- end,
- Options2 = maps:without([ecpool_worker_id, pool_name, append], Options1),
- emqx_connector_mqtt_worker:start_link(Options2#{name => name(Pool, Id)}).
-name(Pool, Id) ->
- list_to_atom(atom_to_list(Pool) ++ ":" ++ integer_to_list(Id)).
-pool_name(ResId) ->
- list_to_atom("bridge_mqtt:" ++ str(ResId)).
-
-options(Options, PoolName, ResId) ->
- GetD = fun(Key, Default) -> maps:get(Key, Options, Default) end,
- Get = fun(Key) -> GetD(Key, undefined) end,
- Address = Get(<<"address">>),
- [{max_inflight_batches, 32},
- {forward_mountpoint, str(Get(<<"mountpoint">>))},
- {disk_cache, GetD(<<"disk_cache">>, false)},
- {start_type, auto},
- {reconnect_delay_ms, hocon_postprocess:duration(str(Get(<<"reconnect_interval">>)))},
- {if_record_metrics, false},
- {pool_size, GetD(<<"pool_size">>, 1)},
- {pool_name, PoolName}
- ] ++ case is_node_addr(Address) of
- true ->
- [{address, binary_to_atom(Get(<<"address">>), utf8)},
- {connect_module, emqx_bridge_rpc},
- {batch_size, Get(<<"batch_size">>)}];
- false ->
- [{address, binary_to_list(Address)},
- {bridge_mode, GetD(<<"bridge_mode">>, true)},
- {clean_start, true},
- {clientid, str(Get(<<"clientid">>))},
- {append, Get(<<"append">>)},
- {connect_module, emqx_bridge_mqtt},
- {keepalive, hocon_postprocess:duration(str(Get(<<"keepalive">>))) div 1000},
- {username, str(Get(<<"username">>))},
- {password, str(Get(<<"password">>))},
- {proto_ver, mqtt_ver(Get(<<"proto_ver">>))},
- {retry_interval, hocon_postprocess:duration(str(GetD(<<"retry_interval">>, "30s"))) div 1000}
- | maybe_ssl(Options, Get(<<"ssl">>), ResId)]
- end.
-
-maybe_ssl(_Options, false, _ResId) ->
- [];
-maybe_ssl(Options, true, ResId) ->
- [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, "rules", ResId)}].
-
-mqtt_ver(ProtoVer) ->
- case ProtoVer of
- <<"mqttv3">> -> v3;
- <<"mqttv4">> -> v4;
- <<"mqttv5">> -> v5;
- _ -> v4
- end.
diff --git a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src
deleted file mode 100644
index 8c2b8d247..000000000
--- a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src
+++ /dev/null
@@ -1,12 +0,0 @@
-%% -*- mode: erlang -*-
-{application, emqx_rule_actions,
- [{description, "Rule actions"},
- {vsn, "5.0.0"},
- {registered, []},
- {applications,
- [kernel,stdlib,emqx]},
- {env,[]},
- {modules, []},
- {licenses, ["Apache 2.0"]},
- {links, []}
- ]}.
diff --git a/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl b/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl
deleted file mode 100644
index 1b68ad5b0..000000000
--- a/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl
+++ /dev/null
@@ -1,379 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% Define the default actions.
--module(emqx_web_hook_actions).
-
--export([ on_resource_create/2
- , on_get_resource_status/2
- , on_resource_destroy/2
- ]).
-
--export([ on_action_create_data_to_webserver/2
- , on_action_data_to_webserver/2
- ]).
-
--export_type([action_fun/0]).
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx_rule_engine/include/rule_actions.hrl").
-
--type(action_fun() :: fun((Data :: map(), Envs :: map()) -> Result :: any())).
-
--type(url() :: binary()).
-
--define(RESOURCE_TYPE_WEBHOOK, 'web_hook').
--define(RESOURCE_CONFIG_SPEC, #{
- url => #{order => 1,
- type => string,
- format => url,
- required => true,
- title => #{en => <<"Request URL">>,
- zh => <<"请求 URL"/utf8>>},
- description => #{en => <<"The URL of the server that will receive the Webhook requests.">>,
- zh => <<"用于接收 Webhook 请求的服务器的 URL。"/utf8>>}},
- connect_timeout => #{order => 2,
- type => string,
- default => <<"5s">>,
- title => #{en => <<"Connect Timeout">>,
- zh => <<"连接超时时间"/utf8>>},
- description => #{en => <<"Connect Timeout In Seconds">>,
- zh => <<"连接超时时间"/utf8>>}},
- request_timeout => #{order => 3,
- type => string,
- default => <<"5s">>,
- title => #{en => <<"Request Timeout">>,
- zh => <<"请求超时时间时间"/utf8>>},
- description => #{en => <<"Request Timeout In Seconds">>,
- zh => <<"请求超时时间"/utf8>>}},
- pool_size => #{order => 4,
- type => number,
- default => 8,
- title => #{en => <<"Pool Size">>, zh => <<"连接池大小"/utf8>>},
- description => #{en => <<"Connection Pool">>,
- zh => <<"连接池大小"/utf8>>}
- },
- cacertfile => #{order => 5,
- type => file,
- default => <<"">>,
- title => #{en => <<"CA Certificate File">>,
- zh => <<"CA 证书文件"/utf8>>},
- description => #{en => <<"CA Certificate file">>,
- zh => <<"CA 证书文件"/utf8>>}},
- keyfile => #{order => 6,
- type => file,
- default => <<"">>,
- title =>#{en => <<"SSL Key">>,
- zh => <<"SSL Key"/utf8>>},
- description => #{en => <<"Your ssl keyfile">>,
- zh => <<"SSL 私钥"/utf8>>}},
- certfile => #{order => 7,
- type => file,
- default => <<"">>,
- title => #{en => <<"SSL Cert">>,
- zh => <<"SSL Cert"/utf8>>},
- description => #{en => <<"Your ssl certfile">>,
- zh => <<"SSL 证书"/utf8>>}},
- verify => #{order => 8,
- type => boolean,
- default => false,
- title => #{en => <<"Verify Server Certfile">>,
- zh => <<"校验服务器证书"/utf8>>},
- description => #{en => <<"Whether to verify the server certificate. By default, the client will not verify the server's certificate. If verification is required, please set it to true.">>,
- zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>}},
- server_name_indication => #{order => 9,
- type => string,
- title => #{en => <<"Server Name Indication">>,
- zh => <<"服务器名称指示"/utf8>>},
- description => #{en => <<"Specify the hostname used for peer certificate verification, or set to disable to turn off this verification.">>,
- zh => <<"指定用于对端证书验证时使用的主机名，或者设置为 disable 以关闭此项验证。"/utf8>>}}
-}).
-
--define(ACTION_PARAM_RESOURCE, #{
- order => 0,
- type => string,
- required => true,
- title => #{en => <<"Resource ID">>,
- zh => <<"资源 ID"/utf8>>},
- description => #{en => <<"Bind a resource to this action">>,
- zh => <<"给动作绑定一个资源"/utf8>>}
-}).
-
--define(ACTION_DATA_SPEC, #{
- '$resource' => ?ACTION_PARAM_RESOURCE,
- method => #{
- order => 1,
- type => string,
- enum => [<<"POST">>, <<"DELETE">>, <<"PUT">>, <<"GET">>],
- default => <<"POST">>,
- title => #{en => <<"Method">>,
- zh => <<"Method"/utf8>>},
- description => #{en => <<"HTTP Method.\n"
- "Note that: the Body option in the Action will be discarded in case of GET or DELETE method.">>,
- zh => <<"HTTP Method。\n"
- "注意：当方法为 GET 或 DELETE 时，动作中的 Body 选项会被忽略。"/utf8>>}},
- path => #{
- order => 2,
- type => string,
- required => false,
- default => <<"">>,
- title => #{en => <<"Path">>,
- zh => <<"Path"/utf8>>},
- description => #{en => <<"The path part of the URL, support using ${Var} to get the field value output by the rule.">>,
- zh => <<"URL 的路径部分，支持使用 ${Var} 获取规则输出的字段值。\n"/utf8>>}
- },
- headers => #{
- order => 3,
- type => object,
- schema => #{},
- default => #{<<"content-type">> => <<"application/json">>},
- title => #{en => <<"Headers">>,
- zh => <<"Headers"/utf8>>},
- description => #{en => <<"HTTP headers.">>,
- zh => <<"HTTP headers。"/utf8>>}},
- body => #{
- order => 4,
- type => string,
- input => textarea,
- required => false,
- default => <<"">>,
- title => #{en => <<"Body">>,
- zh => <<"Body"/utf8>>},
- description => #{en => <<"The HTTP body supports the use of ${Var} to obtain the field value output by the rule.\n"
- "The content of the default HTTP request body is a JSON string composed of the keys and values of all fields output by the rule.">>,
- zh => <<"HTTP 请求体，支持使用 ${Var} 获取规则输出的字段值\n"
- "默认 HTTP 请求体的内容为规则输出的所有字段的键和值构成的 JSON 字符串。"/utf8>>}}
- }).
-
--resource_type(
- #{name => ?RESOURCE_TYPE_WEBHOOK,
- create => on_resource_create,
- status => on_get_resource_status,
- destroy => on_resource_destroy,
- params => ?RESOURCE_CONFIG_SPEC,
- title => #{en => <<"WebHook">>,
- zh => <<"WebHook"/utf8>>},
- description => #{en => <<"WebHook">>,
- zh => <<"WebHook"/utf8>>}
-}).
-
--rule_action(#{name => data_to_webserver,
- category => data_forward,
- for => '$any',
- create => on_action_create_data_to_webserver,
- params => ?ACTION_DATA_SPEC,
- types => [?RESOURCE_TYPE_WEBHOOK],
- title => #{en => <<"Data to Web Server">>,
- zh => <<"发送数据到 Web 服务"/utf8>>},
- description => #{en => <<"Forward Messages to Web Server">>,
- zh => <<"将数据转发给 Web 服务"/utf8>>}
-}).
-
-%%------------------------------------------------------------------------------
-%% Actions for web hook
-%%------------------------------------------------------------------------------
-
--spec(on_resource_create(binary(), map()) -> map()).
-on_resource_create(ResId, Conf) ->
- {ok, _} = application:ensure_all_started(ehttpc),
- Options = pool_opts(Conf, ResId),
- PoolName = pool_name(ResId),
- case test_http_connect(Conf) of
- true -> ok;
- false -> error({error, check_http_connectivity_failed})
- end,
- start_resource(ResId, PoolName, Options),
- Conf#{<<"pool">> => PoolName, options => Options}.
-
-start_resource(ResId, PoolName, Options) ->
- case ehttpc_pool:start_pool(PoolName, Options) of
- {ok, _} ->
- ?LOG(info, "Initiated Resource ~p Successfully, ResId: ~p",
- [?RESOURCE_TYPE_WEBHOOK, ResId]);
- {error, {already_started, _Pid}} ->
- on_resource_destroy(ResId, #{<<"pool">> => PoolName}),
- start_resource(ResId, PoolName, Options);
- {error, Reason} ->
- ?LOG(error, "Initiate Resource ~p failed, ResId: ~p, ~0p",
- [?RESOURCE_TYPE_WEBHOOK, ResId, Reason]),
- error({{?RESOURCE_TYPE_WEBHOOK, ResId}, create_failed})
- end.
-
--spec(on_get_resource_status(binary(), map()) -> map()).
-on_get_resource_status(_ResId, Conf) ->
- #{is_alive => test_http_connect(Conf)}.
-
--spec(on_resource_destroy(binary(), map()) -> ok | {error, Reason::term()}).
-on_resource_destroy(ResId, #{<<"pool">> := PoolName}) ->
- ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]),
- case ehttpc_pool:stop_pool(PoolName) of
- ok ->
- ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]);
- {error, Reason} ->
- ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_WEBHOOK, ResId, Reason]),
- error({{?RESOURCE_TYPE_WEBHOOK, ResId}, destroy_failed})
- end.
-
-%% An action that forwards publish messages to a remote web server.
--spec(on_action_create_data_to_webserver(Id::binary(), #{url() := string()}) -> {bindings(), NewParams :: map()}).
-on_action_create_data_to_webserver(Id, Params) ->
- #{method := Method,
- path := Path,
- headers := Headers,
- body := Body,
- pool := Pool,
- request_timeout := RequestTimeout} = parse_action_params(Params),
- BodyTokens = emqx_plugin_libs_rule:preproc_tmpl(Body),
- PathTokens = emqx_plugin_libs_rule:preproc_tmpl(Path),
- Params.
-
-on_action_data_to_webserver(Selected, _Envs =
- #{?BINDING_KEYS := #{
- 'Id' := Id,
- 'Method' := Method,
- 'Headers' := Headers,
- 'PathTokens' := PathTokens,
- 'BodyTokens' := BodyTokens,
- 'Pool' := Pool,
- 'RequestTimeout' := RequestTimeout},
- clientid := ClientID}) ->
- NBody = format_msg(BodyTokens, Selected),
- NPath = emqx_plugin_libs_rule:proc_tmpl(PathTokens, Selected),
- Req = create_req(Method, NPath, Headers, NBody),
- case ehttpc:request(ehttpc_pool:pick_worker(Pool, ClientID), Method, Req, RequestTimeout) of
- {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
- emqx_rule_metrics:inc_actions_success(Id);
- {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
- emqx_rule_metrics:inc_actions_success(Id);
- {ok, StatusCode, _} ->
- ?LOG(warning, "[WebHook Action] HTTP request failed with status code: ~p", [StatusCode]),
- emqx_rule_metrics:inc_actions_error(Id);
- {ok, StatusCode, _, _} ->
- ?LOG(warning, "[WebHook Action] HTTP request failed with status code: ~p", [StatusCode]),
- emqx_rule_metrics:inc_actions_error(Id);
- {error, Reason} ->
- ?LOG(error, "[WebHook Action] HTTP request error: ~p", [Reason]),
- emqx_rule_metrics:inc_actions_error(Id)
- end.
-
-format_msg([], Data) ->
- emqx_json:encode(Data);
-format_msg(Tokens, Data) ->
- emqx_plugin_libs_rule:proc_tmpl(Tokens, Data).
-
-%%------------------------------------------------------------------------------
-%% Internal functions
-%%------------------------------------------------------------------------------
-
-create_req(Method, Path, Headers, _Body)
- when Method =:= get orelse Method =:= delete ->
- {Path, Headers};
-create_req(_, Path, Headers, Body) ->
- {Path, Headers, Body}.
-
-parse_action_params(Params = #{<<"url">> := URL}) ->
- try
- {ok, #{path := CommonPath}} = emqx_http_lib:uri_parse(URL),
- Method = method(maps:get(<<"method">>, Params, <<"POST">>)),
- Headers = headers(maps:get(<<"headers">>, Params, undefined)),
- NHeaders = ensure_content_type_header(Headers, Method),
- #{method => Method,
- path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)),
- headers => NHeaders,
- body => maps:get(<<"body">>, Params, <<>>),
- request_timeout => hocon_postprocess:duration(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))),
- pool => maps:get(<<"pool">>, Params)}
- catch _:_ ->
- throw({invalid_params, Params})
- end.
-
-ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put ->
- Headers;
-ensure_content_type_header(Headers, _Method) ->
- lists:keydelete("content-type", 1, Headers).
-
-merge_path(CommonPath, <<>>) ->
- CommonPath;
-merge_path(CommonPath, Path0) ->
- case emqx_http_lib:uri_parse(Path0) of
- {ok, #{path := Path1, 'query' := Query}} ->
- Path2 = filename:join(CommonPath, Path1),
- <>;
- {ok, #{path := Path1}} ->
- filename:join(CommonPath, Path1)
- end.
-
-method(GET) when GET == <<"GET">>; GET == <<"get">> -> get;
-method(POST) when POST == <<"POST">>; POST == <<"post">> -> post;
-method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put;
-method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete.
-
-headers(undefined) -> [];
-headers(Headers) when is_map(Headers) ->
- headers(maps:to_list(Headers));
-headers(Headers) when is_list(Headers) ->
- [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers].
-
-str(Str) when is_list(Str) -> Str;
-str(Atom) when is_atom(Atom) -> atom_to_list(Atom);
-str(Bin) when is_binary(Bin) -> binary_to_list(Bin).
-
-pool_opts(Params = #{<<"url">> := URL}, ResId) ->
- {ok, #{host := Host,
- port := Port,
- scheme := Scheme}} = emqx_http_lib:uri_parse(URL),
- PoolSize = maps:get(<<"pool_size">>, Params, 32),
- ConnectTimeout =
- hocon_postprocess:duration(str(maps:get(<<"connect_timeout">>, Params, <<"5s">>))),
- TransportOpts0 =
- case Scheme =:= https of
- true -> [get_ssl_opts(Params, ResId)];
- false -> []
- end,
- TransportOpts = emqx_misc:ipv6_probe(TransportOpts0),
- Opts = case Scheme =:= https of
- true -> [{transport_opts, TransportOpts}, {transport, ssl}];
- false -> [{transport_opts, TransportOpts}]
- end,
- [{host, Host},
- {port, Port},
- {pool_size, PoolSize},
- {pool_type, hash},
- {connect_timeout, ConnectTimeout},
- {retry, 5},
- {retry_timeout, 1000} | Opts].
-
-pool_name(ResId) ->
- list_to_atom("webhook:" ++ str(ResId)).
-
-get_ssl_opts(Opts, ResId) ->
- emqx_plugin_libs_ssl:save_files_return_opts(Opts, "rules", ResId).
-
-test_http_connect(Conf) ->
- Url = fun() -> maps:get(<<"url">>, Conf) end,
- try
- emqx_plugin_libs_rule:http_connectivity(Url())
- of
- ok -> true;
- {error, _Reason} ->
- ?LOG(error, "check http_connectivity failed: ~p", [Url()]),
- false
- catch
- Err:Reason:ST ->
- ?LOG(error, "check http_connectivity failed: ~p, ~0p", [Conf, {Err, Reason, ST}]),
- false
- end.
diff --git a/apps/emqx_rule_engine/docs/api_examples.md b/apps/emqx_rule_engine/docs/api_examples.md
deleted file mode 100644
index f546a3a57..000000000
--- a/apps/emqx_rule_engine/docs/api_examples.md
+++ /dev/null
@@ -1,197 +0,0 @@
-#Rule-Engine-APIs
-
-## ENVs
-
-APPSECRET="88ebdd6569afc:Mjg3MzUyNTI2Mjk2NTcyOTEwMDEwMDMzMTE2NTM1MTkzNjA"
-
-## Rules
-
-### test sql
-
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules?test' -d \
-'{"rawsql":"select * from \"message.publish\" where topic=\"t/a\"","ctx":{}}'
-
-
-
-### create
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \
-'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule"}'
-
-{"code":0,"data":{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""}}
-
-## with a resource id in the action args
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \
-'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1"}}],"description":"test-rule"}'
-
-{"code":0,"data":{"actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1","a":1}}],"description":"test-rule","enabled":true,"id":"rule:6fce0ca9","rawsql":"select * from \"t/a\""}}
-```
-
-### modify
-
-```shell
-## modify all of the params
-$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \
-'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule"}'
-
-## modify some of the params: disable it
-$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \
-'{"enabled": false}'
-
-## modify some of the params: add fallback actions
-$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \
-'{"actions":[{"name":"inspect","params":{"a":1}, "fallbacks": [{"name":"donothing"}]}]}'
-```
-
-### show
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915'
-
-{"code":0,"data":{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""}}
-```
-
-### list
-
-```shell
-$ curl -v --basic -u $APPSECRET -k http://localhost:8081/api/v4/rules
-
-{"code":0,"data":[{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""},{"actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1","a":1}}],"description":"test-rule","enabled":true,"id":"rule:6fce0ca9","rawsql":"select * from \"t/a\""}]}
-```
-
-### delete
-
-```shell
-$ curl -XDELETE -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915'
-
-{"code":0}
-```
-
-## Actions
-
-### list
-
-```shell
-$ curl -v --basic -u $APPSECRET -k http://localhost:8081/api/v4/actions
-
-{"code":0,"data":[{"app":"emqx_rule_engine","description":"Republish a MQTT message to a another topic","name":"republish","params":{...},"types":[]},{"app":"emqx_rule_engine","description":"Inspect the details of action params for debug purpose","name":"inspect","params":{},"types":[]},{"app":"emqx_web_hook","description":"Forward Messages to Web Server","name":"data_to_webserver","params":{...},"types":["web_hook"]}]}
-```
-
-### show
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/actions/inspect'
-
-{"code":0,"data":{"app":"emqx_rule_engine","description":"Debug Action","name":"inspect","params":{"$resource":"built_in"}}}
-```
-
-## Resource Types
-
-### list
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types'
-
-{"code":0,"data":[{"description":"Debug resource type","name":"built_in","params":{},"provider":"emqx_rule_engine"}]}
-```
-
-### list all resources of a type
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types/built_in/resources'
-
-{"code":0,"data":[{"attrs":"undefined","config":{"a":1},"description":"test-rule","id":"resource:71df3086","type":"built_in"}]}
-```
-
-### show
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types/built_in'
-
-{"code":0,"data":{"description":"Debug resource type","name":"built_in","params":{},"provider":"emqx_rule_engine"}}
-```
-
-
-
-## Resources
-
-### create
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources' -d \
-'{"type": "built_in", "config": {"a":1}, "description": "test-resource"}'
-
-{"code":0,"data":{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}}
-```
-
-### start
-
-```shell
-$ curl -XPOST -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086'
-
-{"code":0}
-```
-
-### list
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources'
-
-{"code":0,"data":[{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}]}
-```
-
-### show
-
-```shell
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086'
-
-{"code":0,"data":{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}}
-```
-
-### get resource status
-
-```shell
-curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_status/resource:71df3086'
-
-{"code":0,"data":{"is_alive":true}}
-```
-
-### delete
-
-```shell
-$ curl -XDELETE -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086'
-
-{"code":0}
-```
-
-## Rule example using webhook
-
-``` shell
-
-$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources' -d \
-'{"type": "web_hook", "config": {"url": "http://127.0.0.1:9910", "headers": {"token":"axfw34y235wrq234t4ersgw4t"}, "method": "POST"}, "description": "web hook resource-1"}'
-
-{"code":0,"data":{"attrs":"undefined","config":{"headers":{"token":"axfw34y235wrq234t4ersgw4t"},"method":"POST","url":"http://127.0.0.1:9910"},"description":"web hook resource-1","id":"resource:8531a11f","type":"web_hook"}}
-
-curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \
-'{"rawsql":"SELECT clientid as c, username as u.name FROM \"#\"","actions":[{"name":"data_to_webserver","params":{"$resource": "resource:8531a11f"}}],"description":"Forward connected events to webhook"}'
-
-{"code":0,"data":{"actions":[{"name":"data_to_webserver","params":{"$resource":"resource:8531a11f","headers":{"token":"axfw34y235wrq234t4ersgw4t"},"method":"POST","url":"http://127.0.0.1:9910"}}],"description":"Forward connected events to webhook","enabled":true,"id":"rule:4fe05936","rawsql":"select * from \"#\""}}
-```
-
-Start a `web server` using `nc`, and then connect to emqx broker using a mqtt client with username = 'Shawn':
-
-```shell
-$ echo -e "HTTP/1.1 200 OK\n\n $(date)" | nc -l 127.0.0.1 9910
-
-POST / HTTP/1.1
-content-type: application/json
-content-length: 48
-te:
-host: 127.0.0.1:9910
-connection: keep-alive
-token: axfw34y235wrq234t4ersgw4t
-
-{"c":"clientId-bP70ymeIyo","u":{"name":"Shawn"}
-```
diff --git a/apps/emqx_rule_engine/docs/cli_examples.md b/apps/emqx_rule_engine/docs/cli_examples.md
deleted file mode 100644
index 3d854129c..000000000
--- a/apps/emqx_rule_engine/docs/cli_examples.md
+++ /dev/null
@@ -1,164 +0,0 @@
-#Rule-Engine-CLIs
-
-## Rules
-
-### create
-
-```shell
- $ ./bin/emqx_ctl rules create 'SELECT payload FROM "t/#" username="Steven"' '[{"name":"data_to_webserver", "params": {"$resource": "resource:9093f1cb"}}]' --descr="Msg From Steven to WebServer"
-
-Rule rule:98a75239 created
-```
-
-### modify
-
-
-```shell
- ## update sql, action, description
- $ ./bin/emqx_ctl rules update 'rule:98a75239' \
- -s "select * from \"t/a\" " \
- -a '[{"name":"do_nothing", "fallbacks": []' -g continue \
- -d 'Rule for debug2' \
-
- ## update sql only
- $ ./bin/emqx_ctl rules update 'rule:98a75239' -s 'SELECT * FROM "t/a"'
-
- ## disable the rule
- $ ./bin/emqx_ctl rules update 'rule:98a75239' -e false
-
-```
-
-### show
-
-```shell
-$ ./bin/emqx_ctl rules show rule:98a75239
-
-rule(id='rule:98a75239', rawsql='SELECT payload FROM "t/#" username="Steven"', actions=[{"name":"data_to_webserver","params":{"$resource":"resource:9093f1cb","url":"http://host-name/chats"}}], enabled='true', description='Msg From Steven to WebServer')
-```
-
-### list
-
-```shell
-$ ./bin/emqx_ctl rules list
-
-rule(id='rule:98a75239', rawsql='SELECT payload FROM "t/#" username="Steven"', actions=[{"name":"data_to_webserver","params":{"$resource":"resource:9093f1cb","url":"http://host-name/chats"}}], enabled='true', description='Msg From Steven to WebServer')
-
-```
-
-### delete
-
-```shell
-$ ./bin/emqx_ctl rules delete 'rule:98a75239'
-
-ok
-```
-
-## Actions
-
-### list
-
-```shell
-$ ./bin/emqx_ctl rule-actions list
-
-action(name='republish', app='emqx_rule_engine', types=[], params=#{...}, description='Republish a MQTT message to a another topic')
-action(name='inspect', app='emqx_rule_engine', types=[], params=#{...}, description='Inspect the details of action params for debug purpose')
-action(name='data_to_webserver', app='emqx_web_hook', types=[], params=#{...}, description='Forward Messages to Web Server')
-```
-
-### show
-
-```shell
-$ ./bin/emqx_ctl rule-actions show 'data_to_webserver'
-
-action(name='data_to_webserver', app='emqx_web_hook', types=['web_hook'], params=#{...}, description='Forward Messages to Web Server')
-```
-
-## Resource
-
-### create
-
-```shell
-$ ./bin/emqx_ctl resources create 'web_hook' -c '{"url": "http://host-name/chats"}' --descr 'Resource towards http://host-name/chats'
-
-Resource resource:19addfef created
-```
-
-### list
-
-```shell
-$ ./bin/emqx_ctl resources list
-
-resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats')
-
-```
-
-### list all resources of a type
-
-```shell
-$ ./bin/emqx_ctl resources list -t 'web_hook'
-
-resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats')
-```
-
-### show
-
-```shell
-$ ./bin/emqx_ctl resources show 'resource:19addfef'
-
-resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats')
-```
-
-### delete
-
-```shell
-$ ./bin/emqx_ctl resources delete 'resource:19addfef'
-
-ok
-```
-
-## Resources Types
-
-### list
-
-```shell
-$ ./bin/emqx_ctl resource-types list
-
-resource_type(name='built_in', provider='emqx_rule_engine', params=#{...}, on_create={emqx_rule_actions,on_resource_create}, description='The built in resource type for debug purpose')
-resource_type(name='web_hook', provider='emqx_web_hook', params=#{...}, on_create={emqx_web_hook_actions,on_resource_create}, description='WebHook Resource')
-```
-
-### show
-
-```shell
-$ ./bin/emqx_ctl resource-types show built_in
-
-resource_type(name='built_in', provider='emqx_rule_engine', params=#{}, description='The built in resource type for debug purpose')
-```
-
-## Rule example using webhook
-
-``` shell
-1. Create a webhook resource to URL http://127.0.0.1:9910.
-./bin/emqx_ctl resources create 'web_hook' --config '{"url": "http://127.0.0.1:9910", "headers": {"token":"axfw34y235wrq234t4ersgw4t"}, "method": "POST"}'
-Resource resource:3128243e created
-
-2. Create a rule using action data_to_webserver, and bind above resource to that action.
-./bin/emqx_ctl rules create 'client.connected' 'SELECT clientid as c, username as u.name FROM "#"' '[{"name":"data_to_webserver", "params": {"$resource": "resource:3128243e"}}]' --descr "Forward Connected Events to WebServer"
-Rule rule:222b59f7 created
-```
-
-Start a simple `Web Server` using `nc`, and then connect to emqx broker using a mqtt client with username = 'Shawn':
-
-```shell
-$ echo -e "HTTP/1.1 200 OK\n\n $(date)" | nc -l 127.0.0.1 9910
-
-POST / HTTP/1.1
-content-type: application/json
-content-length: 48
-te:
-host: 127.0.0.1:9910
-connection: keep-alive
-token: axfw34y235wrq234t4ersgw4t
-
-{"c":"clientId-bP70ymeIyo","u":{"name":"Shawn"}
-```
diff --git a/apps/emqx_rule_engine/docs/design.md b/apps/emqx_rule_engine/docs/design.md
deleted file mode 100644
index 3e2c60c41..000000000
--- a/apps/emqx_rule_engine/docs/design.md
+++ /dev/null
@@ -1,188 +0,0 @@
-
-# EMQ X Rule Engine
-
-This is the design guide of message routing rule engine for the EMQ X Broker.
-
-## Concept
-
-A rule is:
-
-```
-when
- Match |
-then
- Select and Take ;
-```
-
-or:
-
-```
-rule "Rule Name"
- when
- rule match
- select
- para1 = val1
- para2 = val2
- then
- action(#{para2 => val1, #para2 => val2})
-```
-
-## Architecture
-
-```
- |-----------------|
- P ---->| Message Routing |----> S
- |-----------------|
- | /|\
- \|/ |
- |-----------------|
- | Rule Engine |
- |-----------------|
- | |
- Backends Services Bridges
-```
-
-## Design
-
-```
-Event | Message -> Rules -> Actions -> Resources
-```
-
-```
- P -> |--------------------| |---------------------------------------|
- | Messages (Routing) | -> | Rules (Select Data, Match Conditions) |
- S <- |--------------------| |---------------------------------------|
- |---------| |-----------| |-------------------------------|
- ->| Actions | -> | Resources | -> | (Backends, Bridges, WebHooks) |
- |---------| |-----------| |-------------------------------|
-```
-
-
-
-## Rule
-
-A rule consists of a SELECT statement, a topic filter, and a rule action
-
-Rules consist of the following:
-
-- Id
-- Name
-- Topic
-- Description
-- Action
-- Enabled
-
-The operations on a rule:
-
-- Create
-- Enable
-- Disable
-- Delete
-
-
-
-## Action
-
-Actions consist of the following:
-
-- Id
-- Name
-- For
-- App
-- Module
-- Func
-- Args
-- Descr
-
-Define a rule action in ADT:
-
-```
-action :: Application -> Resource -> Params -> IO ()
-```
-
-A rule action:
-
-Module:function(Args)
-
-
-
-## Resource
-
-### Resource Name
-
-```
-backend:mysql:localhost:port:db
-backend:redis:localhost:
-webhook:url
-bridge:kafka:
-bridge:rabbit:localhost
-```
-
-### Resource Properties
-
-- Name
-- Descr or Description
-- Config #{}
-- Instances
-- State: Running | Stopped
-
-### Resource Management
-
-1. Create Resource
-2. List Resources
-3. Lookup Resource
-4. Delete Resource
-5. Test Resource
-
-### Resource State (Lifecircle)
-
-0. Create Resource and Validate a Resource
-1. Start/Connect Resource
-2. Bind resource name to instance
-3. Stop/Disconnect Resource
-4. Unbind resource name with instance
-5. Is Resource Alive?
-
-### Resource Type
-
-The properties and behaviors of resources is defined by resource types. A resoure type is provided(contributed) by a plugin.
-
-### Resource Type Provider
-
-Provider of resource type is a EMQ X Plugin.
-
-### Resource Manager
-
-```
- Supervisor
- |
- \|/
-Action ----> Proxy(Batch|Write) ----> Connection -----> ExternalResource
- | /|\
- |------------------Fetch----------------|
-```
-
-
-
-## REST API
-
-Rules API
-Actions API
-Resources API
-
-## CLI
-
-```
-rules list
-rules show
-
-rule-actions list
-rule-actions show
-
-resources list
-resources show
-
-resource_templates list
-resource_templates show
-```
-
diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf
index 22543a977..a4344cda8 100644
--- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf
+++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf
@@ -1,6 +1,6 @@
##====================================================================
## Rule Engine for EMQ X R5.0
##====================================================================
-emqx_rule_engine {
+rule_engine {
ignore_sys_message = true
}
diff --git a/apps/emqx_rule_engine/include/rule_actions.hrl b/apps/emqx_rule_engine/include/rule_actions.hrl
deleted file mode 100644
index e432c4399..000000000
--- a/apps/emqx_rule_engine/include/rule_actions.hrl
+++ /dev/null
@@ -1,11 +0,0 @@
--compile({parse_transform, emqx_rule_actions_trans}).
-
--type selected_data() :: map().
--type env_vars() :: map().
--type bindings() :: list({atom(), term()}).
-
--define(BINDING_KEYS, '__bindings__').
-
--define(bound_v(Key, ENVS0),
- maps:get(Key,
- maps:get(?BINDING_KEYS, ENVS0, #{}))).
diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl
index 760495f6b..2908051fe 100644
--- a/apps/emqx_rule_engine/include/rule_engine.hrl
+++ b/apps/emqx_rule_engine/include/rule_engine.hrl
@@ -18,109 +18,46 @@
-define(KV_TAB, '@rule_engine_db').
--type(maybe(T) :: T | undefined).
+-type maybe(T) :: T | undefined.
--type(rule_id() :: binary()).
--type(rule_name() :: binary()).
+-type rule_id() :: binary().
+-type rule_name() :: binary().
--type(resource_id() :: binary()).
--type(action_instance_id() :: binary()).
+-type mf() :: {Module::atom(), Fun::atom()}.
--type(action_name() :: atom()).
--type(resource_type_name() :: atom()).
+-type hook() :: atom() | 'any'.
--type(category() :: data_persist| data_forward | offline_msgs | debug | other).
+-type topic() :: binary().
+-type bridge_channel_id() :: binary().
+-type selected_data() :: map().
+-type envs() :: map().
+-type output_type() :: bridge | builtin | func.
+-type output_target() :: bridge_channel_id() | atom() | output_fun().
+-type output_fun_args() :: map().
+-type output() :: #{
+ type := output_type(),
+ target := output_target(),
+ args => output_fun_args()
+}.
+-type output_fun() :: fun((selected_data(), envs(), output_fun_args()) -> any()).
--type(descr() :: #{en := binary(), zh => binary()}).
-
--type(mf() :: {Module::atom(), Fun::atom()}).
-
--type(hook() :: atom() | 'any').
-
--type(topic() :: binary()).
-
--type(resource_status() :: #{ alive := boolean()
- , atom() => binary() | atom() | list(binary()|atom())
- }).
-
--define(descr, #{en => <<>>, zh => <<>>}).
-
--record(action,
- { name :: action_name()
- , category :: category()
- , for :: hook()
- , app :: atom()
- , types = [] :: list(resource_type_name())
- , module :: module()
- , on_create :: mf()
- , on_destroy :: maybe(mf())
- , hidden = false :: boolean()
- , params_spec :: #{atom() => term()} %% params specs
- , title = ?descr :: descr()
- , description = ?descr :: descr()
- }).
-
--record(action_instance,
- { id :: action_instance_id()
- , name :: action_name()
- , fallbacks :: list(#action_instance{})
- , args :: #{binary() => term()} %% the args got from API for initializing action_instance
- }).
+-type rule_info() ::
+ #{ from := list(topic())
+ , outputs := [output()]
+ , sql := binary()
+ , is_foreach := boolean()
+ , fields := list()
+ , doeach := term()
+ , incase := term()
+ , conditions := tuple()
+ , enabled := boolean()
+ , description => binary()
+ }.
-record(rule,
{ id :: rule_id()
- , for :: list(topic())
- , rawsql :: binary()
- , is_foreach :: boolean()
- , fields :: list()
- , doeach :: term()
- , incase :: list()
- , conditions :: tuple()
- , on_action_failed :: continue | stop
- , actions :: list(#action_instance{})
- , enabled :: boolean()
, created_at :: integer() %% epoch in millisecond precision
- , description :: binary()
- , state = normal :: atom()
- }).
-
--record(resource,
- { id :: resource_id()
- , type :: resource_type_name()
- , config :: #{} %% the configs got from API for initializing resource
- , created_at :: integer() | undefined %% epoch in millisecond precision
- , description :: binary()
- }).
-
--record(resource_type,
- { name :: resource_type_name()
- , provider :: atom()
- , params_spec :: #{atom() => term()} %% params specs
- , on_create :: mf()
- , on_status :: mf()
- , on_destroy :: mf()
- , title = ?descr :: descr()
- , description = ?descr :: descr()
- }).
-
--record(rule_hooks,
- { hook :: atom()
- , rule_id :: rule_id()
- }).
-
--record(resource_params,
- { id :: resource_id()
- , params :: #{} %% the params got after initializing the resource
- , status = #{is_alive => false} :: #{is_alive := boolean(), atom() => term()}
- }).
-
--record(action_instance_params,
- { id :: action_instance_id()
- %% the params got after initializing the action
- , params :: #{}
- %% the Func/Bindings got after initializing the action
- , apply :: fun((Data::map(), Envs::map()) -> any())
- | #{mod := module(), bindings := #{atom() => term()}}
+ , info :: rule_info()
}).
%% Arithmetic operators
@@ -157,9 +94,3 @@
%% Tables
-define(RULE_TAB, emqx_rule).
--define(ACTION_TAB, emqx_rule_action).
--define(ACTION_INST_PARAMS_TAB, emqx_action_instance_params).
--define(RES_TAB, emqx_resource).
--define(RES_PARAMS_TAB, emqx_resource_params).
--define(RULE_HOOKS, emqx_rule_hooks).
--define(RES_TYPE_TAB, emqx_resource_type).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
deleted file mode 100644
index 7ac45633c..000000000
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ /dev/null
@@ -1,208 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% Define the default actions.
--module(emqx_rule_actions).
-
--include("rule_engine.hrl").
--include("rule_actions.hrl").
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/logger.hrl").
-
--define(REPUBLISH_PARAMS_SPEC, #{
- target_topic => #{
- order => 1,
- type => string,
- required => true,
- default => <<"repub/to/${clientid}">>,
- title => #{en => <<"Target Topic">>,
- zh => <<"目的主题"/utf8>>},
- description => #{en => <<"To which topic the message will be republished">>,
- zh => <<"重新发布消息到哪个主题"/utf8>>}
- },
- target_qos => #{
- order => 2,
- type => number,
- enum => [-1, 0, 1, 2],
- required => true,
- default => 0,
- title => #{en => <<"Target QoS">>,
- zh => <<"目的 QoS"/utf8>>},
- description => #{en => <<"The QoS Level to be uses when republishing the message. Set to -1 to use the original QoS">>,
- zh => <<"重新发布消息时用的 QoS 级别, 设置为 -1 以使用原消息中的 QoS"/utf8>>}
- },
- payload_tmpl => #{
- order => 3,
- type => string,
- input => textarea,
- required => false,
- default => <<"${payload}">>,
- title => #{en => <<"Payload Template">>,
- zh => <<"消息内容模板"/utf8>>},
- description => #{en => <<"The payload template, variable interpolation is supported">>,
- zh => <<"消息内容模板，支持变量"/utf8>>}
- }
- }).
-
--rule_action(#{name => inspect,
- category => debug,
- for => '$any',
- types => [],
- create => on_action_create_inspect,
- params => #{},
- title => #{en => <<"Inspect (debug)">>,
- zh => <<"检查 (调试)"/utf8>>},
- description => #{en => <<"Inspect the details of action params for debug purpose">>,
- zh => <<"检查动作参数 (用以调试)"/utf8>>}
- }).
-
--rule_action(#{name => republish,
- category => data_forward,
- for => '$any',
- types => [],
- create => on_action_create_republish,
- params => ?REPUBLISH_PARAMS_SPEC,
- title => #{en => <<"Republish">>,
- zh => <<"消息重新发布"/utf8>>},
- description => #{en => <<"Republish a MQTT message to another topic">>,
- zh => <<"重新发布消息到另一个主题"/utf8>>}
- }).
-
--rule_action(#{name => do_nothing,
- category => debug,
- for => '$any',
- types => [],
- create => on_action_create_do_nothing,
- params => #{},
- title => #{en => <<"Do Nothing (debug)">>,
- zh => <<"空动作 (调试)"/utf8>>},
- description => #{en => <<"This action does nothing and never fails. It's for debug purpose">>,
- zh => <<"此动作什么都不做，并且不会失败 (用以调试)"/utf8>>}
- }).
-
--export([on_resource_create/2]).
-
-%% callbacks for rule engine
--export([ on_action_create_inspect/2
- , on_action_create_republish/2
- , on_action_create_do_nothing/2
- ]).
-
--export([ on_action_inspect/2
- , on_action_republish/2
- , on_action_do_nothing/2
- ]).
-
--spec(on_resource_create(binary(), map()) -> map()).
-on_resource_create(_Name, Conf) ->
- Conf.
-
-%%------------------------------------------------------------------------------
-%% Action 'inspect'
-%%------------------------------------------------------------------------------
--spec on_action_create_inspect(Id :: action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
-on_action_create_inspect(Id, Params) ->
- Params.
-
--spec on_action_inspect(selected_data(), env_vars()) -> any().
-on_action_inspect(Selected, Envs) ->
- ?ULOG("[inspect]~n"
- "\tSelected Data: ~p~n"
- "\tEnvs: ~p~n"
- "\tAction Init Params: ~p~n", [Selected, Envs, ?bound_v('Params', Envs)]),
- emqx_rule_metrics:inc_actions_success(?bound_v('Id', Envs)).
-
-
-%%------------------------------------------------------------------------------
-%% Action 'republish'
-%%------------------------------------------------------------------------------
--spec on_action_create_republish(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
-on_action_create_republish(Id, Params = #{
- <<"target_topic">> := TargetTopic,
- <<"target_qos">> := TargetQoS,
- <<"payload_tmpl">> := PayloadTmpl
- }) ->
- TopicTks = emqx_plugin_libs_rule:preproc_tmpl(TargetTopic),
- PayloadTks = emqx_plugin_libs_rule:preproc_tmpl(PayloadTmpl),
- Params.
-
--spec on_action_republish(selected_data(), env_vars()) -> any().
-on_action_republish(_Selected, Envs = #{
- topic := Topic,
- headers := #{republish_by := ActId},
- ?BINDING_KEYS := #{'Id' := ActId}
- }) ->
- ?LOG(error, "[republish] recursively republish detected, msg topic: ~p, target topic: ~p",
- [Topic, ?bound_v('TargetTopic', Envs)]),
- emqx_rule_metrics:inc_actions_error(?bound_v('Id', Envs));
-
-on_action_republish(Selected, _Envs = #{
- qos := QoS, flags := Flags, timestamp := Timestamp,
- ?BINDING_KEYS := #{
- 'Id' := ActId,
- 'TargetTopic' := TargetTopic,
- 'TargetQoS' := TargetQoS,
- 'TopicTks' := TopicTks,
- 'PayloadTks' := PayloadTks
- }}) ->
- ?LOG(debug, "[republish] republish to: ~p, Payload: ~p",
- [TargetTopic, Selected]),
- increase_and_publish(ActId,
- #message{
- id = emqx_guid:gen(),
- qos = if TargetQoS =:= -1 -> QoS; true -> TargetQoS end,
- from = ActId,
- flags = Flags,
- headers = #{republish_by => ActId},
- topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
- payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
- timestamp = Timestamp
- });
-
-%% in case this is not a "message.publish" request
-on_action_republish(Selected, _Envs = #{
- ?BINDING_KEYS := #{
- 'Id' := ActId,
- 'TargetTopic' := TargetTopic,
- 'TargetQoS' := TargetQoS,
- 'TopicTks' := TopicTks,
- 'PayloadTks' := PayloadTks
- }}) ->
- ?LOG(debug, "[republish] republish to: ~p, Payload: ~p",
- [TargetTopic, Selected]),
- increase_and_publish(ActId,
- #message{
- id = emqx_guid:gen(),
- qos = if TargetQoS =:= -1 -> 0; true -> TargetQoS end,
- from = ActId,
- flags = #{dup => false, retain => false},
- headers = #{republish_by => ActId},
- topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
- payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
- timestamp = erlang:system_time(millisecond)
- }).
-
-increase_and_publish(ActId, Msg) ->
- _ = emqx_broker:safe_publish(Msg),
- emqx_rule_metrics:inc_actions_success(ActId),
- emqx_metrics:inc_msg(Msg).
-
--spec on_action_create_do_nothing(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}.
-on_action_create_do_nothing(ActId, Params) when is_binary(ActId) ->
- Params.
-
-on_action_do_nothing(Selected, Envs) when is_map(Selected) ->
- emqx_rule_metrics:inc_actions_success(?bound_v('ActId', Envs)).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
new file mode 100644
index 000000000..c96e82ecb
--- /dev/null
+++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
@@ -0,0 +1,188 @@
+-module(emqx_rule_api_schema).
+
+-behaviour(hocon_schema).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-export([ check_params/2
+ ]).
+
+-export([roots/0, fields/1]).
+
+-type tag() :: rule_creation | rule_test.
+
+-spec check_params(map(), tag()) -> {ok, map()} | {error, term()}.
+check_params(Params, Tag) ->
+ BTag = atom_to_binary(Tag),
+ try hocon_schema:check_plain(?MODULE, #{BTag => Params},
+ #{atom_key => true, nullable => true}, [BTag]) of
+ #{Tag := Checked} -> {ok, Checked}
+ catch
+ Error:Reason:ST ->
+ ?SLOG(error, #{msg => "check_rule_params_failed",
+ exception => Error,
+ reason => Reason,
+ stacktrace => ST}),
+ {error, {Reason, ST}}
+ end.
+
+%%======================================================================================
+%% Hocon Schema Definitions
+
+roots() ->
+ [ {"rule_creation", sc(ref("rule_creation"), #{desc => "Schema for creating rules"})}
+ , {"rule_test", sc(ref("rule_test"), #{desc => "Schema for testing rules"})}
+ ].
+
+fields("rule_creation") ->
+ [ {"id", sc(binary(), #{desc => "The Id of the rule", nullable => false})}
+ , {"sql", sc(binary(), #{desc => "The SQL of the rule", nullable => false})}
+ , {"outputs", sc(hoconsc:array(hoconsc:union(
+ [ ref("bridge_output")
+ , ref("builtin_output")
+ ])),
+ #{desc => "The outputs of the rule",
+ default => []})}
+ , {"enable", sc(boolean(), #{desc => "Enable or disable the rule", default => true})}
+ , {"description", sc(binary(), #{desc => "The description of the rule", default => <<>>})}
+ ];
+
+fields("rule_test") ->
+ [ {"context", sc(hoconsc:union([ ref("ctx_pub")
+ , ref("ctx_sub")
+ , ref("ctx_delivered")
+ , ref("ctx_acked")
+ , ref("ctx_dropped")
+ , ref("ctx_connected")
+ , ref("ctx_disconnected")
+ ]),
+ #{desc => "The context of the event for testing",
+ default => #{}})}
+ , {"sql", sc(binary(), #{desc => "The SQL of the rule for testing", nullable => false})}
+ ];
+
+fields("bridge_output") ->
+ [ {type, bridge}
+ , {target, sc(binary(), #{desc => "The Channel ID of the bridge"})}
+ ];
+
+fields("builtin_output") ->
+ [ {type, builtin}
+ , {target, sc(binary(), #{desc => "The Name of the built-on output"})}
+ , {args, sc(map(), #{desc => "The arguments of the built-in output",
+ default => #{}})}
+ ];
+
+%% TODO: how to use this in "builtin_output".args ?
+fields("republish_args") ->
+ [ {topic, sc(binary(),
+ #{desc => "The target topic of the re-published message."
+ " Template with with variables is allowed.",
+ nullable => false})}
+ , {qos, sc(binary(),
+ #{desc => "The qos of the re-published message."
+ " Template with with variables is allowed. Defaults to ${qos}.",
+ default => <<"${qos}">> })}
+ , {retain, sc(binary(),
+ #{desc => "The retain of the re-published message."
+ " Template with with variables is allowed. Defaults to ${retain}.",
+ default => <<"${retain}">> })}
+ , {payload, sc(binary(),
+ #{desc => "The payload of the re-published message."
+ " Template with with variables is allowed. Defaults to ${payload}.",
+ default => <<"${payload}">>})}
+ ];
+
+fields("ctx_pub") ->
+ [ {"event_type", sc(message_publish, #{desc => "Event Type", nullable => false})}
+ , {"id", sc(binary(), #{desc => "Message ID"})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"payload", sc(binary(), #{desc => "The Message Payload"})}
+ , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})}
+ , {"topic", sc(binary(), #{desc => "Message Topic"})}
+ , {"publish_received_at", sc(integer(), #{
+ desc => "The Time that this Message is Received"})}
+ ] ++ [qos()];
+
+fields("ctx_sub") ->
+ [ {"event_type", sc(session_subscribed, #{desc => "Event Type", nullable => false})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"payload", sc(binary(), #{desc => "The Message Payload"})}
+ , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})}
+ , {"topic", sc(binary(), #{desc => "Message Topic"})}
+ , {"publish_received_at", sc(integer(), #{
+ desc => "The Time that this Message is Received"})}
+ ] ++ [qos()];
+
+fields("ctx_unsub") ->
+ [{"event_type", sc(session_unsubscribed, #{desc => "Event Type", nullable => false})}] ++
+ proplists:delete("event_type", fields("ctx_sub"));
+
+fields("ctx_delivered") ->
+ [ {"event_type", sc(message_delivered, #{desc => "Event Type", nullable => false})}
+ , {"id", sc(binary(), #{desc => "Message ID"})}
+ , {"from_clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"from_username", sc(binary(), #{desc => "The User Name"})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"payload", sc(binary(), #{desc => "The Message Payload"})}
+ , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})}
+ , {"topic", sc(binary(), #{desc => "Message Topic"})}
+ , {"publish_received_at", sc(integer(), #{
+ desc => "The Time that this Message is Received"})}
+ ] ++ [qos()];
+
+fields("ctx_acked") ->
+ [{"event_type", sc(message_acked, #{desc => "Event Type", nullable => false})}] ++
+ proplists:delete("event_type", fields("ctx_delivered"));
+
+fields("ctx_dropped") ->
+ [ {"event_type", sc(message_dropped, #{desc => "Event Type", nullable => false})}
+ , {"id", sc(binary(), #{desc => "Message ID"})}
+ , {"reason", sc(binary(), #{desc => "The Reason for Dropping"})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"payload", sc(binary(), #{desc => "The Message Payload"})}
+ , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})}
+ , {"topic", sc(binary(), #{desc => "Message Topic"})}
+ , {"publish_received_at", sc(integer(), #{
+ desc => "The Time that this Message is Received"})}
+ ] ++ [qos()];
+
+fields("ctx_connected") ->
+ [ {"event_type", sc(client_connected, #{desc => "Event Type", nullable => false})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"mountpoint", sc(binary(), #{desc => "The Mountpoint"})}
+ , {"peername", sc(binary(), #{desc => "The IP Address and Port of the Peer Client"})}
+ , {"sockname", sc(binary(), #{desc => "The IP Address and Port of the Local Listener"})}
+ , {"proto_name", sc(binary(), #{desc => "Protocol Name"})}
+ , {"proto_ver", sc(binary(), #{desc => "Protocol Version"})}
+ , {"keepalive", sc(integer(), #{desc => "KeepAlive"})}
+ , {"clean_start", sc(boolean(), #{desc => "Clean Start", default => true})}
+ , {"expiry_interval", sc(integer(), #{desc => "Expiry Interval"})}
+ , {"is_bridge", sc(boolean(), #{desc => "Is Bridge", default => false})}
+ , {"connected_at", sc(integer(), #{
+ desc => "The Time that this Client is Connected"})}
+ ];
+
+fields("ctx_disconnected") ->
+ [ {"event_type", sc(client_disconnected, #{desc => "Event Type", nullable => false})}
+ , {"clientid", sc(binary(), #{desc => "The Client ID"})}
+ , {"username", sc(binary(), #{desc => "The User Name"})}
+ , {"reason", sc(binary(), #{desc => "The Reason for Disconnect"})}
+ , {"peername", sc(binary(), #{desc => "The IP Address and Port of the Peer Client"})}
+ , {"sockname", sc(binary(), #{desc => "The IP Address and Port of the Local Listener"})}
+ , {"disconnected_at", sc(integer(), #{
+ desc => "The Time that this Client is Disconnected"})}
+ ].
+
+qos() ->
+ {"qos", sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]),
+ #{desc => "The Message QoS"})}.
+
+sc(Type, Meta) -> hoconsc:mk(Type, Meta).
+ref(Field) -> hoconsc:ref(?MODULE, Field).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
index bf0eb06e8..04d35931a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
@@ -19,627 +19,102 @@
-include("rule_engine.hrl").
-include_lib("emqx/include/logger.hrl").
--export([ load_providers/0
- , unload_providers/0
- , refresh_resources/0
- , refresh_resource/1
- , refresh_rule/1
- , refresh_rules/0
- , refresh_actions/1
- , refresh_actions/2
- , refresh_resource_status/0
- ]).
-
-export([ create_rule/1
, update_rule/1
, delete_rule/1
- , create_resource/1
- , test_resource/1
- , start_resource/1
- , get_resource_status/1
- , get_resource_params/1
- , delete_resource/1
- , update_resource/2
]).
--export([ init_resource/4
- , init_action/4
- , clear_resource/3
- , clear_rule/1
- , clear_actions/1
- , clear_action/3
- ]).
+-export_type([rule/0]).
--type(rule() :: #rule{}).
--type(action() :: #action{}).
--type(resource() :: #resource{}).
--type(resource_type() :: #resource_type{}).
--type(resource_params() :: #resource_params{}).
--type(action_instance_params() :: #action_instance_params{}).
-
--export_type([ rule/0
- , action/0
- , resource/0
- , resource_type/0
- , resource_params/0
- , action_instance_params/0
- ]).
+-type rule() :: #rule{}.
-define(T_RETRY, 60000).
-%%------------------------------------------------------------------------------
-%% Load resource/action providers from all available applications
-%%------------------------------------------------------------------------------
-
-%% Load all providers .
--spec(load_providers() -> ok).
-load_providers() ->
- lists:foreach(fun(App) ->
- load_provider(App)
- end, ignore_lib_apps(application:loaded_applications())).
-
--spec(load_provider(App :: atom()) -> ok).
-load_provider(App) when is_atom(App) ->
- ok = load_actions(App),
- ok = load_resource_types(App).
-
-%%------------------------------------------------------------------------------
-%% Unload providers
-%%------------------------------------------------------------------------------
-%% Load all providers .
--spec(unload_providers() -> ok).
-unload_providers() ->
- lists:foreach(fun(App) ->
- unload_provider(App)
- end, ignore_lib_apps(application:loaded_applications())).
-
-%% @doc Unload a provider.
--spec(unload_provider(App :: atom()) -> ok).
-unload_provider(App) ->
- ok = emqx_rule_registry:remove_actions_of(App),
- ok = emqx_rule_registry:unregister_resource_types_of(App).
-
-load_actions(App) ->
- Actions = find_actions(App),
- emqx_rule_registry:add_actions(Actions).
-
-load_resource_types(App) ->
- ResourceTypes = find_resource_types(App),
- emqx_rule_registry:register_resource_types(ResourceTypes).
-
--spec(find_actions(App :: atom()) -> list(action())).
-find_actions(App) ->
- lists:map(fun new_action/1, find_attrs(App, rule_action)).
-
--spec(find_resource_types(App :: atom()) -> list(resource_type())).
-find_resource_types(App) ->
- lists:map(fun new_resource_type/1, find_attrs(App, resource_type)).
-
-new_action({App, Mod, #{name := Name,
- for := Hook,
- types := Types,
- create := Create,
- params := ParamsSpec} = Params}) ->
- ok = emqx_rule_validator:validate_spec(ParamsSpec),
- #action{name = Name, for = Hook, app = App, types = Types,
- category = maps:get(category, Params, other),
- module = Mod, on_create = Create,
- hidden = maps:get(hidden, Params, false),
- on_destroy = maps:get(destroy, Params, undefined),
- params_spec = ParamsSpec,
- title = maps:get(title, Params, ?descr),
- description = maps:get(description, Params, ?descr)}.
-
-new_resource_type({App, Mod, #{name := Name,
- params := ParamsSpec,
- create := Create} = Params}) ->
- ok = emqx_rule_validator:validate_spec(ParamsSpec),
- #resource_type{name = Name, provider = App,
- params_spec = ParamsSpec,
- on_create = {Mod, Create},
- on_status = {Mod, maps:get(status, Params, undefined)},
- on_destroy = {Mod, maps:get(destroy, Params, undefined)},
- title = maps:get(title, Params, ?descr),
- description = maps:get(description, Params, ?descr)}.
-
-find_attrs(App, Def) ->
- [{App, Mod, Attr} || {ok, Modules} <- [application:get_key(App, modules)],
- Mod <- Modules,
- {Name, Attrs} <- module_attributes(Mod), Name =:= Def,
- Attr <- Attrs].
-
-module_attributes(Module) ->
- try Module:module_info(attributes)
- catch
- error:undef -> []
- end.
-
%%------------------------------------------------------------------------------
%% APIs for rules and resources
%%------------------------------------------------------------------------------
--dialyzer([{nowarn_function, [create_rule/1, rule_id/0]}]).
-spec create_rule(map()) -> {ok, rule()} | {error, term()}.
-create_rule(Params = #{rawsql := Sql, actions := ActArgs}) ->
- case emqx_rule_sqlparser:parse_select(Sql) of
- {ok, Select} ->
- RuleId = maps:get(id, Params, rule_id()),
- Enabled = maps:get(enabled, Params, true),
- try prepare_actions(ActArgs, Enabled) of
- Actions ->
- Rule = #rule{
- id = RuleId,
- rawsql = Sql,
- for = emqx_rule_sqlparser:select_from(Select),
- is_foreach = emqx_rule_sqlparser:select_is_foreach(Select),
- fields = emqx_rule_sqlparser:select_fields(Select),
- doeach = emqx_rule_sqlparser:select_doeach(Select),
- incase = emqx_rule_sqlparser:select_incase(Select),
- conditions = emqx_rule_sqlparser:select_where(Select),
- on_action_failed = maps:get(on_action_failed, Params, continue),
- actions = Actions,
- enabled = Enabled,
- created_at = erlang:system_time(millisecond),
- description = maps:get(description, Params, ""),
- state = normal
- },
- ok = emqx_rule_registry:add_rule(Rule),
- ok = emqx_rule_metrics:create_rule_metrics(RuleId),
- {ok, Rule}
- catch
- throw:{action_not_found, ActionName} ->
- {error, {action_not_found, ActionName}};
- throw:Reason ->
- {error, Reason}
- end;
- Reason -> {error, Reason}
- end.
-
--spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()}}).
-update_rule(Params = #{id := RuleId}) ->
+create_rule(Params = #{id := RuleId}) ->
case emqx_rule_registry:get_rule(RuleId) of
- {ok, Rule0} ->
- try may_update_rule_params(Rule0, Params) of
- Rule ->
- ok = emqx_rule_registry:add_rule(Rule),
- {ok, Rule}
- catch
- throw:Reason ->
- {error, Reason}
- end;
- not_found ->
- {error, {not_found, RuleId}}
+ not_found -> do_create_rule(Params);
+ {ok, _} -> {error, {already_exists, RuleId}}
end.
--spec(delete_rule(RuleId :: rule_id()) -> ok).
+-spec update_rule(map()) -> {ok, rule()} | {error, term()}.
+update_rule(Params = #{id := RuleId}) ->
+ case delete_rule(RuleId) of
+ ok -> do_create_rule(Params);
+ Error -> Error
+ end.
+
+-spec(delete_rule(RuleId :: rule_id()) -> ok | {error, term()}).
delete_rule(RuleId) ->
case emqx_rule_registry:get_rule(RuleId) of
- {ok, Rule = #rule{actions = Actions}} ->
- try
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_rule, [Rule]),
- ok = emqx_rule_registry:remove_rule(Rule)
- catch
- Error:Reason:ST ->
- ?LOG(error, "clear_rule ~p failed: ~p", [RuleId, {Error, Reason, ST}]),
- refresh_actions(Actions)
- end;
- not_found ->
- ok
- end.
-
--spec(create_resource(#{type := _, config := _, _ => _}) -> {ok, resource()} | {error, Reason :: term()}).
-create_resource(#{type := Type, config := Config0} = Params) ->
- case emqx_rule_registry:find_resource_type(Type) of
- {ok, #resource_type{on_create = {M, F}, params_spec = ParamSpec}} ->
- Config = emqx_rule_validator:validate_params(Config0, ParamSpec),
- ResId = maps:get(id, Params, resource_id()),
- Resource = #resource{id = ResId,
- type = Type,
- config = Config,
- description = iolist_to_binary(maps:get(description, Params, "")),
- created_at = erlang:system_time(millisecond)
- },
- ok = emqx_rule_registry:add_resource(Resource),
- %% Note that we will return OK in case of resource creation failure,
- %% A timer is started to re-start the resource later.
- catch _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [M, F, ResId, Config]),
- {ok, Resource};
- not_found ->
- {error, {resource_type_not_found, Type}}
- end.
-
--spec(update_resource(resource_id(), map()) -> ok | {error, Reason :: term()}).
-update_resource(ResId, NewParams) ->
- case emqx_rule_registry:find_enabled_rules_depends_on_resource(ResId) of
- [] -> check_and_update_resource(ResId, NewParams);
- Rules ->
- {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}}
- end.
-
-check_and_update_resource(Id, NewParams) ->
- case emqx_rule_registry:find_resource(Id) of
- {ok, #resource{id = Id, type = Type, config = OldConfig, description = OldDescr}} ->
- try
- Conifg = maps:get(<<"config">>, NewParams, OldConfig),
- Descr = maps:get(<<"description">>, NewParams, OldDescr),
- do_check_and_update_resource(#{id => Id, config => Conifg, type => Type,
- description => Descr})
- catch Error:Reason:ST ->
- ?LOG(error, "check_and_update_resource failed: ~0p", [{Error, Reason, ST}]),
- {error, Reason}
- end;
- _Other ->
- {error, not_found}
- end.
-
-do_check_and_update_resource(#{id := Id, type := Type, description := NewDescription,
- config := NewConfig}) ->
- case emqx_rule_registry:find_resource_type(Type) of
- {ok, #resource_type{on_create = {Module, Create},
- params_spec = ParamSpec}} ->
- Config = emqx_rule_validator:validate_params(NewConfig, ParamSpec),
- case test_resource(#{type => Type, config => NewConfig}) of
- ok ->
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [Module, Create, Id, Config]),
- emqx_rule_registry:add_resource(#resource{
- id = Id,
- type = Type,
- config = Config,
- description = NewDescription,
- created_at = erlang:system_time(millisecond)
- }),
- ok;
- {error, Reason} ->
- error({error, Reason})
- end
- end.
-
--spec(start_resource(resource_id()) -> ok | {error, Reason :: term()}).
-start_resource(ResId) ->
- case emqx_rule_registry:find_resource(ResId) of
- {ok, #resource{type = ResType, config = Config}} ->
- {ok, #resource_type{on_create = {Mod, Create}}}
- = emqx_rule_registry:find_resource_type(ResType),
- try
- init_resource(Mod, Create, ResId, Config),
- refresh_actions_of_a_resource(ResId)
- catch
- throw:Reason -> {error, Reason}
- end;
- not_found ->
- {error, {resource_not_found, ResId}}
- end.
-
--spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}).
-test_resource(#{type := Type, config := Config0}) ->
- case emqx_rule_registry:find_resource_type(Type) of
- {ok, #resource_type{on_create = {ModC, Create},
- on_destroy = {ModD, Destroy},
- params_spec = ParamSpec}} ->
- Config = emqx_rule_validator:validate_params(Config0, ParamSpec),
- ResId = resource_id(),
- try
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [ModC, Create, ResId, Config]),
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_resource, [ModD, Destroy, ResId]),
- ok
- catch
- throw:Reason -> {error, Reason}
- end;
- not_found ->
- {error, {resource_type_not_found, Type}}
- end.
-
--spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}).
-get_resource_status(ResId) ->
- case emqx_rule_registry:find_resource(ResId) of
- {ok, #resource{type = ResType}} ->
- {ok, #resource_type{on_status = {Mod, OnStatus}}}
- = emqx_rule_registry:find_resource_type(ResType),
- Status = fetch_resource_status(Mod, OnStatus, ResId),
- {ok, Status};
- not_found ->
- {error, {resource_not_found, ResId}}
- end.
-
--spec(get_resource_params(resource_id()) -> {ok, map()} | {error, Reason :: term()}).
-get_resource_params(ResId) ->
- case emqx_rule_registry:find_resource_params(ResId) of
- {ok, #resource_params{params = Params}} ->
- {ok, Params};
- not_found ->
- {error, resource_not_initialized}
- end.
-
--spec(delete_resource(resource_id()) -> ok | {error, Reason :: term()}).
-delete_resource(ResId) ->
- case emqx_rule_registry:find_resource(ResId) of
- {ok, #resource{type = ResType}} ->
- {ok, #resource_type{on_destroy = {ModD, Destroy}}}
- = emqx_rule_registry:find_resource_type(ResType),
- try
- case emqx_rule_registry:remove_resource(ResId) of
- ok ->
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_resource, [ModD, Destroy, ResId]),
- ok;
- {error, _} = R -> R
- end
- catch
- throw:Reason -> {error, Reason}
- end;
+ {ok, Rule} ->
+ ok = emqx_rule_registry:remove_rule(Rule),
+ _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, clear_rule_metrics, [RuleId]),
+ ok;
not_found ->
{error, not_found}
end.
-%%------------------------------------------------------------------------------
-%% Re-establish resources
-%%------------------------------------------------------------------------------
-
--spec(refresh_resources() -> ok).
-refresh_resources() ->
- lists:foreach(fun refresh_resource/1,
- emqx_rule_registry:get_resources()).
-
-refresh_resource(Type) when is_atom(Type) ->
- lists:foreach(fun refresh_resource/1,
- emqx_rule_registry:get_resources_by_type(Type));
-
-refresh_resource(#resource{id = ResId}) ->
- emqx_rule_monitor:ensure_resource_retrier(ResId, ?T_RETRY).
-
--spec(refresh_rules() -> ok).
-refresh_rules() ->
- lists:foreach(fun
- (#rule{enabled = true} = Rule) ->
- try refresh_rule(Rule)
- catch _:_ ->
- emqx_rule_registry:add_rule(Rule#rule{enabled = false, state = refresh_failed_at_bootup})
- end;
- (_) -> ok
- end, emqx_rule_registry:get_rules()).
-
-refresh_rule(#rule{id = RuleId, for = Topics, actions = Actions}) ->
- ok = emqx_rule_metrics:create_rule_metrics(RuleId),
- lists:foreach(fun emqx_rule_events:load/1, Topics),
- refresh_actions(Actions).
-
--spec(refresh_resource_status() -> ok).
-refresh_resource_status() ->
- lists:foreach(
- fun(#resource{id = ResId, type = ResType}) ->
- case emqx_rule_registry:find_resource_type(ResType) of
- {ok, #resource_type{on_status = {Mod, OnStatus}}} ->
- _ = fetch_resource_status(Mod, OnStatus, ResId);
- _ -> ok
- end
- end, emqx_rule_registry:get_resources()).
-
%%------------------------------------------------------------------------------
%% Internal Functions
%%------------------------------------------------------------------------------
-prepare_actions(Actions, NeedInit) ->
- [prepare_action(Action, NeedInit) || Action <- Actions].
-prepare_action(#{name := Name, args := Args0} = Action, NeedInit) ->
- case emqx_rule_registry:find_action(Name) of
- {ok, #action{module = Mod, on_create = Create, params_spec = ParamSpec}} ->
- Args = emqx_rule_validator:validate_params(Args0, ParamSpec),
- ActionInstId = maps:get(id, Action, action_instance_id(Name)),
- case NeedInit of
- true ->
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_action, [Mod, Create, ActionInstId,
- with_resource_params(Args)]),
- ok;
- false -> ok
- end,
- #action_instance{
- id = ActionInstId, name = Name, args = Args,
- fallbacks = prepare_actions(maps:get(fallbacks, Action, []), NeedInit)
- };
- not_found ->
- throw({action_not_found, Name})
- end.
-
-with_resource_params(Args = #{<<"$resource">> := ResId}) ->
- case emqx_rule_registry:find_resource_params(ResId) of
- {ok, #resource_params{params = Params}} ->
- maps:merge(Args, Params);
- not_found ->
- throw({resource_not_initialized, ResId})
- end;
-with_resource_params(Args) -> Args.
-
--dialyzer([{nowarn_function, may_update_rule_params/2}]).
-may_update_rule_params(Rule, Params = #{rawsql := SQL}) ->
- case emqx_rule_sqlparser:parse_select(SQL) of
+do_create_rule(Params = #{id := RuleId, sql := Sql, outputs := Outputs}) ->
+ case emqx_rule_sqlparser:parse(Sql) of
{ok, Select} ->
- may_update_rule_params(
- Rule#rule{
- rawsql = SQL,
- for = emqx_rule_sqlparser:select_from(Select),
- is_foreach = emqx_rule_sqlparser:select_is_foreach(Select),
- fields = emqx_rule_sqlparser:select_fields(Select),
- doeach = emqx_rule_sqlparser:select_doeach(Select),
- incase = emqx_rule_sqlparser:select_incase(Select),
- conditions = emqx_rule_sqlparser:select_where(Select)
- },
- maps:remove(rawsql, Params));
- Reason -> throw(Reason)
- end;
-may_update_rule_params(Rule = #rule{enabled = OldEnb, actions = Actions, state = OldState},
- Params = #{enabled := NewEnb}) ->
- State = case {OldEnb, NewEnb} of
- {false, true} ->
- refresh_rule(Rule),
- force_changed;
- {true, false} ->
- clear_actions(Actions),
- force_changed;
- _NoChange -> OldState
- end,
- may_update_rule_params(Rule#rule{enabled = NewEnb, state = State}, maps:remove(enabled, Params));
-may_update_rule_params(Rule, Params = #{description := Descr}) ->
- may_update_rule_params(Rule#rule{description = Descr}, maps:remove(description, Params));
-may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) ->
- may_update_rule_params(Rule#rule{on_action_failed = OnFailed},
- maps:remove(on_action_failed, Params));
-may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) ->
- %% prepare new actions before removing old ones
- NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)),
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_actions, [OldActions]),
- may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params));
-may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params
- Rule.
-
-ignore_lib_apps(Apps) ->
- LibApps = [kernel, stdlib, sasl, appmon, eldap, erts,
- syntax_tools, ssl, crypto, mnesia, os_mon,
- inets, goldrush, gproc, runtime_tools,
- snmp, otp_mibs, public_key, asn1, ssh, hipe,
- common_test, observer, webtool, xmerl, tools,
- test_server, compiler, debugger, eunit, et,
- wx],
- [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)].
-
-resource_id() ->
- gen_id("resource:", fun emqx_rule_registry:find_resource/1).
-
-rule_id() ->
- gen_id("rule:", fun emqx_rule_registry:get_rule/1).
-
-gen_id(Prefix, TestFun) ->
- Id = iolist_to_binary([Prefix, emqx_misc:gen_id()]),
- case TestFun(Id) of
- not_found -> Id;
- _Res -> gen_id(Prefix, TestFun)
+ Rule = #rule{
+ id = RuleId,
+ created_at = erlang:system_time(millisecond),
+ info = #{
+ enabled => maps:get(enabled, Params, true),
+ sql => Sql,
+ from => emqx_rule_sqlparser:select_from(Select),
+ outputs => parse_outputs(Outputs),
+ description => maps:get(description, Params, ""),
+ %% -- calculated fields:
+ is_foreach => emqx_rule_sqlparser:select_is_foreach(Select),
+ fields => emqx_rule_sqlparser:select_fields(Select),
+ doeach => emqx_rule_sqlparser:select_doeach(Select),
+ incase => emqx_rule_sqlparser:select_incase(Select),
+ conditions => emqx_rule_sqlparser:select_where(Select)
+ %% -- calculated fields end
+ }
+ },
+ ok = emqx_rule_registry:add_rule(Rule),
+ _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, create_rule_metrics, [RuleId]),
+ {ok, Rule};
+ {error, Reason} -> {error, Reason}
end.
-action_instance_id(ActionName) ->
- iolist_to_binary([atom_to_list(ActionName), "_", integer_to_list(erlang:system_time())]).
+parse_outputs(Outputs) ->
+ [do_parse_outputs(Out) || Out <- Outputs].
-init_resource(Module, OnCreate, ResId, Config) ->
- Params = ?RAISE(Module:OnCreate(ResId, Config),
- {{Module, OnCreate}, {_EXCLASS_, _EXCPTION_, _ST_}}),
- ResParams = #resource_params{id = ResId,
- params = Params,
- status = #{is_alive => true}},
- emqx_rule_registry:add_resource_params(ResParams).
+do_parse_outputs(#{type := bridge, target := ChId}) ->
+ #{type => bridge, target => ChId};
+do_parse_outputs(#{type := builtin, target := Repub, args := Args})
+ when Repub == republish; Repub == <<"republish">> ->
+ #{type => builtin, target => republish, args => pre_process_repub_args(Args)};
+do_parse_outputs(#{type := Type, target := Name} = Output)
+ when Type == func; Type == builtin ->
+ #{type => Type, target => Name, args => maps:get(args, Output, #{})}.
-init_action(Module, OnCreate, ActionInstId, Params) ->
- ok = emqx_rule_metrics:create_metrics(ActionInstId),
- case ?RAISE(Module:OnCreate(ActionInstId, Params),
- {{init_action_failure, node()},
- {{Module, OnCreate}, {_EXCLASS_, _EXCPTION_, _ST_}}}) of
- {Apply, NewParams} when is_function(Apply) -> %% BACKW: =< e4.2.2
- ok = emqx_rule_registry:add_action_instance_params(
- #action_instance_params{id = ActionInstId, params = NewParams, apply = Apply});
- {Bindings, NewParams} when is_list(Bindings) ->
- ok = emqx_rule_registry:add_action_instance_params(
- #action_instance_params{
- id = ActionInstId, params = NewParams,
- apply = #{mod => Module, bindings => maps:from_list(Bindings)}});
- Apply when is_function(Apply) -> %% BACKW: =< e4.2.2
- ok = emqx_rule_registry:add_action_instance_params(
- #action_instance_params{id = ActionInstId, params = Params, apply = Apply})
- end.
+pre_process_repub_args(#{<<"topic">> := Topic} = Args) ->
+ QoS = maps:get(<<"qos">>, Args, <<"${qos}">>),
+ Retain = maps:get(<<"retain">>, Args, <<"${retain}">>),
+ Payload = maps:get(<<"payload">>, Args, <<"${payload}">>),
+ #{topic => Topic, qos => QoS, payload => Payload, retain => Retain,
+ preprocessed_tmpl => #{
+ topic => emqx_plugin_libs_rule:preproc_tmpl(Topic),
+ qos => preproc_vars(QoS),
+ retain => preproc_vars(Retain),
+ payload => emqx_plugin_libs_rule:preproc_tmpl(Payload)
+ }}.
-clear_resource(_Module, undefined, ResId) ->
- ok = emqx_rule_registry:remove_resource_params(ResId);
-clear_resource(Module, Destroy, ResId) ->
- case emqx_rule_registry:find_resource_params(ResId) of
- {ok, #resource_params{params = Params}} ->
- ?RAISE(Module:Destroy(ResId, Params),
- {{destroy_resource_failure, node()}, {{Module, Destroy}, {_EXCLASS_,_EXCPTION_,_ST_}}}),
- ok = emqx_rule_registry:remove_resource_params(ResId);
- not_found ->
- ok
- end.
-
-clear_rule(#rule{id = RuleId, actions = Actions}) ->
- clear_actions(Actions),
- emqx_rule_metrics:clear_rule_metrics(RuleId),
- ok.
-
-clear_actions(Actions) ->
- lists:foreach(
- fun(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks}) ->
- {ok, #action{module = Mod, on_destroy = Destory}} = emqx_rule_registry:find_action(ActName),
- clear_action(Mod, Destory, Id),
- clear_actions(Fallbacks)
- end, Actions).
-
-clear_action(_Module, undefined, ActionInstId) ->
- emqx_rule_metrics:clear_metrics(ActionInstId),
- ok = emqx_rule_registry:remove_action_instance_params(ActionInstId);
-clear_action(Module, Destroy, ActionInstId) ->
- case erlang:function_exported(Module, Destroy, 2) of
- true ->
- emqx_rule_metrics:clear_metrics(ActionInstId),
- case emqx_rule_registry:get_action_instance_params(ActionInstId) of
- {ok, #action_instance_params{params = Params}} ->
- ?RAISE(Module:Destroy(ActionInstId, Params),{{destroy_action_failure, node()},
- {{Module, Destroy}, {_EXCLASS_,_EXCPTION_,_ST_}}}),
- ok = emqx_rule_registry:remove_action_instance_params(ActionInstId);
- not_found ->
- ok
- end;
- false -> ok
- end.
-
-fetch_resource_status(Module, OnStatus, ResId) ->
- case emqx_rule_registry:find_resource_params(ResId) of
- {ok, ResParams = #resource_params{params = Params, status = #{is_alive := LastIsAlive}}} ->
- NewStatus = try
- case Module:OnStatus(ResId, Params) of
- #{is_alive := LastIsAlive} = Status -> Status;
- #{is_alive := true} = Status ->
- {ok, Type} = find_type(ResId),
- Name = alarm_name_of_resource_down(Type, ResId),
- emqx_alarm:deactivate(Name),
- Status;
- #{is_alive := false} = Status ->
- {ok, Type} = find_type(ResId),
- Name = alarm_name_of_resource_down(Type, ResId),
- emqx_alarm:activate(Name, #{id => ResId, type => Type}),
- Status
- end
- catch _Error:Reason:STrace ->
- ?LOG(error, "get resource status for ~p failed: ~0p", [ResId, {Reason, STrace}]),
- #{is_alive => false}
- end,
- emqx_rule_registry:add_resource_params(ResParams#resource_params{status = NewStatus}),
- NewStatus;
- not_found ->
- #{is_alive => false}
- end.
-
-refresh_actions_of_a_resource(ResId) ->
- R = fun (#action_instance{args = #{<<"$resource">> := ResId0}})
- when ResId0 =:= ResId -> true;
- (_) -> false
- end,
- F = fun(#rule{actions = Actions}) -> refresh_actions(Actions, R) end,
- lists:foreach(F, emqx_rule_registry:get_rules()).
-
-refresh_actions(Actions) ->
- refresh_actions(Actions, fun(_) -> true end).
-refresh_actions(Actions, Pred) ->
- lists:foreach(
- fun(#action_instance{args = Args,
- id = Id, name = ActName,
- fallbacks = Fallbacks} = ActionInst) ->
- case Pred(ActionInst) of
- true ->
- {ok, #action{module = Mod, on_create = Create}}
- = emqx_rule_registry:find_action(ActName),
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_action, [Mod, Create, Id, with_resource_params(Args)]),
- refresh_actions(Fallbacks, Pred);
- false -> ok
- end
- end, Actions).
-
-find_type(ResId) ->
- {ok, #resource{type = Type}} = emqx_rule_registry:find_resource(ResId),
- {ok, Type}.
-
-alarm_name_of_resource_down(Type, ResId) ->
- list_to_binary(io_lib:format("resource/~s/~s/down", [Type, ResId])).
+preproc_vars(Data) when is_binary(Data) ->
+ emqx_plugin_libs_rule:preproc_tmpl(Data);
+preproc_vars(Data) ->
+ Data.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 122ae7705..b097c7169 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -19,536 +19,312 @@
-include("rule_engine.hrl").
-include_lib("emqx/include/logger.hrl").
+-behaviour(minirest_api).
--rest_api(#{name => create_rule,
- method => 'POST',
- path => "/rules/",
- func => create_rule,
- descr => "Create a rule"
- }).
+-export([api_spec/0]).
--rest_api(#{name => update_rule,
- method => 'PUT',
- path => "/rules/:bin:id",
- func => update_rule,
- descr => "Update a rule"
- }).
-
--rest_api(#{name => list_rules,
- method => 'GET',
- path => "/rules/",
- func => list_rules,
- descr => "A list of all rules"
- }).
-
--rest_api(#{name => show_rule,
- method => 'GET',
- path => "/rules/:bin:id",
- func => show_rule,
- descr => "Show a rule"
- }).
-
--rest_api(#{name => delete_rule,
- method => 'DELETE',
- path => "/rules/:bin:id",
- func => delete_rule,
- descr => "Delete a rule"
- }).
-
--rest_api(#{name => list_actions,
- method => 'GET',
- path => "/actions/",
- func => list_actions,
- descr => "A list of all actions"
- }).
-
--rest_api(#{name => show_action,
- method => 'GET',
- path => "/actions/:atom:name",
- func => show_action,
- descr => "Show an action"
- }).
-
--rest_api(#{name => list_resources,
- method => 'GET',
- path => "/resources/",
- func => list_resources,
- descr => "A list of all resources"
- }).
-
--rest_api(#{name => create_resource,
- method => 'POST',
- path => "/resources/",
- func => create_resource,
- descr => "Create a resource"
- }).
-
--rest_api(#{name => update_resource,
- method => 'PUT',
- path => "/resources/:bin:id",
- func => update_resource,
- descr => "Update a resource"
- }).
-
--rest_api(#{name => show_resource,
- method => 'GET',
- path => "/resources/:bin:id",
- func => show_resource,
- descr => "Show a resource"
- }).
-
--rest_api(#{name => get_resource_status,
- method => 'GET',
- path => "/resource_status/:bin:id",
- func => get_resource_status,
- descr => "Get status of a resource"
- }).
-
--rest_api(#{name => start_resource,
- method => 'POST',
- path => "/resources/:bin:id",
- func => start_resource,
- descr => "Start a resource"
- }).
-
--rest_api(#{name => delete_resource,
- method => 'DELETE',
- path => "/resources/:bin:id",
- func => delete_resource,
- descr => "Delete a resource"
- }).
-
--rest_api(#{name => list_resource_types,
- method => 'GET',
- path => "/resource_types/",
- func => list_resource_types,
- descr => "List all resource types"
- }).
-
--rest_api(#{name => show_resource_type,
- method => 'GET',
- path => "/resource_types/:atom:name",
- func => show_resource_type,
- descr => "Show a resource type"
- }).
-
--rest_api(#{name => list_resources_by_type,
- method => 'GET',
- path => "/resource_types/:atom:type/resources",
- func => list_resources_by_type,
- descr => "List all resources of a resource type"
- }).
-
--rest_api(#{name => list_events,
- method => 'GET',
- path => "/rule_events/",
- func => list_events,
- descr => "List all events with detailed info"
- }).
-
--export([ create_rule/2
- , update_rule/2
- , list_rules/2
- , show_rule/2
- , delete_rule/2
+-export([ crud_rules/2
+ , list_events/2
+ , crud_rules_by_id/2
+ , rule_test/2
]).
--export([ list_actions/2
- , show_action/2
- ]).
-
--export([ create_resource/2
- , list_resources/2
- , show_resource/2
- , get_resource_status/2
- , start_resource/2
- , delete_resource/2
- , update_resource/2
- ]).
-
--export([ list_resource_types/2
- , list_resources_by_type/2
- , show_resource_type/2
- ]).
-
--export([list_events/2]).
-
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))).
--define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))).
--define(ERR_NO_RESOURCE(RESID), list_to_binary(io_lib:format("Resource ~s Not Found", [(RESID)]))).
--define(ERR_NO_RESOURCE_TYPE(TYPE), list_to_binary(io_lib:format("Resource Type ~s Not Found", [(TYPE)]))).
--define(ERR_DEP_RULES_EXISTS(RULEIDS), list_to_binary(io_lib:format("Found rules ~0p depends on this resource, disable them first", [(RULEIDS)]))).
-define(ERR_BADARGS(REASON),
begin
- R0 = list_to_binary(io_lib:format("~0p", [REASON])),
+ R0 = err_msg(REASON),
<<"Bad Arguments: ", R0/binary>>
end).
+-define(CHECK_PARAMS(PARAMS, TAG, EXPR),
+ case emqx_rule_api_schema:check_params(PARAMS, TAG) of
+ {ok, CheckedParams} ->
+ EXPR;
+ {error, REASON} ->
+ {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(REASON)}}
+ end).
--dialyzer({nowarn_function, [create_rule/2,
- test_rule_sql/1,
- do_create_rule/1,
- update_rule/2
- ]}).
+api_spec() ->
+ {
+ [ api_rules_list_create()
+ , api_rules_crud()
+ , api_rule_test()
+ , api_events_list()
+ ],
+ []
+ }.
+
+api_rules_list_create() ->
+ Metadata = #{
+ get => #{
+ description => <<"List all rules">>,
+ responses => #{
+ <<"200">> =>
+ emqx_mgmt_util:array_schema(resp_schema(), <<"List rules successfully">>)}},
+ post => #{
+ description => <<"Create a new rule using given Id to all nodes in the cluster">>,
+ requestBody => emqx_mgmt_util:schema(post_req_schema(), <<"Rule parameters">>),
+ responses => #{
+ <<"400">> =>
+ emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']),
+ <<"201">> =>
+ emqx_mgmt_util:schema(resp_schema(), <<"Create rule successfully">>)}}
+ },
+ {"/rules", Metadata, crud_rules}.
+
+api_events_list() ->
+ Metadata = #{
+ get => #{
+ description => <<"List all events can be used in rules">>,
+ responses => #{
+ <<"200">> =>
+ emqx_mgmt_util:array_schema(resp_schema(), <<"List events successfully">>)}}
+ },
+ {"/rule_events", Metadata, list_events}.
+
+api_rules_crud() ->
+ Metadata = #{
+ get => #{
+ description => <<"Get a rule by given Id">>,
+ parameters => [param_path_id()],
+ responses => #{
+ <<"404">> =>
+ emqx_mgmt_util:error_schema(<<"Rule not found">>, ['NOT_FOUND']),
+ <<"200">> =>
+ emqx_mgmt_util:schema(resp_schema(), <<"Get rule successfully">>)}},
+ put => #{
+ description => <<"Create or update a rule by given Id to all nodes in the cluster">>,
+ parameters => [param_path_id()],
+ requestBody => emqx_mgmt_util:schema(put_req_schema(), <<"Rule parameters">>),
+ responses => #{
+ <<"400">> =>
+ emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']),
+ <<"200">> =>
+ emqx_mgmt_util:schema(resp_schema(), <<"Create or update rule successfully">>)}},
+ delete => #{
+ description => <<"Delete a rule by given Id from all nodes in the cluster">>,
+ parameters => [param_path_id()],
+ responses => #{
+ <<"200">> =>
+ emqx_mgmt_util:schema(<<"Delete rule successfully">>)}}
+ },
+ {"/rules/:id", Metadata, crud_rules_by_id}.
+
+api_rule_test() ->
+ Metadata = #{
+ post => #{
+ description => <<"Test a rule">>,
+ requestBody => emqx_mgmt_util:schema(rule_test_req_schema(), <<"Rule parameters">>),
+ responses => #{
+ <<"400">> =>
+ emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']),
+ <<"412">> =>
+ emqx_mgmt_util:error_schema(<<"SQL Not Match">>, ['NOT_MATCH']),
+ <<"200">> =>
+ emqx_mgmt_util:schema(rule_test_resp_schema(), <<"Rule Test Pass">>)}}
+ },
+ {"/rule_test", Metadata, rule_test}.
+
+put_req_schema() ->
+ #{type => object,
+ properties => #{
+ sql => #{
+ description => <<"The SQL">>,
+ type => string,
+ example => <<"SELECT * from \"t/1\"">>
+ },
+ enable => #{
+ description => <<"Enable or disable the rule">>,
+ type => boolean,
+ example => true
+ },
+ outputs => #{
+ description => <<"The outputs of the rule">>,
+ type => array,
+ items => #{
+ type => object,
+ properties => #{
+ type => #{
+ type => string,
+ enum => [<<"bridge">>, <<"builtin">>],
+ example => <<"builtin">>
+ },
+ target => #{
+ type => string,
+ example => <<"console">>
+ },
+ args => #{
+ type => object
+ }
+ }
+ }
+ },
+ description => #{
+ description => <<"The description for the rule">>,
+ type => string,
+ example => <<"A simple rule that handles MQTT messages from topic \"t/1\"">>
+ }
+ }
+ }.
+
+post_req_schema() ->
+ Req = #{properties := Prop} = put_req_schema(),
+ Req#{properties => Prop#{
+ id => #{
+ description => <<"The Id for the rule">>,
+ example => <<"my_rule">>,
+ type => string
+ }
+ }}.
+
+resp_schema() ->
+ Req = #{properties := Prop} = put_req_schema(),
+ Req#{properties => Prop#{
+ id => #{
+ description => <<"The Id for the rule">>,
+ type => string
+ },
+ created_at => #{
+ description => <<"The time that this rule was created, in rfc3339 format">>,
+ type => string,
+ example => <<"2021-09-18T13:57:29+08:00">>
+ }
+ }}.
+
+rule_test_req_schema() ->
+ #{type => object, properties => #{
+ sql => #{
+ description => <<"The SQL">>,
+ type => string,
+ example => <<"SELECT * from \"t/1\"">>
+ },
+ context => #{
+ type => object,
+ properties => #{
+ event_type => #{
+ description => <<"Event Type">>,
+ type => string,
+ enum => [<<"message_publish">>, <<"message_acked">>, <<"message_delivered">>,
+ <<"message_dropped">>, <<"session_subscribed">>, <<"session_unsubscribed">>,
+ <<"client_connected">>, <<"client_disconnected">>],
+ example => <<"message_publish">>
+ },
+ clientid => #{
+ description => <<"The Client ID">>,
+ type => string,
+ example => <<"\"c_emqx\"">>
+ },
+ topic => #{
+ description => <<"The Topic">>,
+ type => string,
+ example => <<"t/1">>
+ }
+ }
+ }
+ }}.
+
+rule_test_resp_schema() ->
+ #{type => object}.
+
+param_path_id() ->
+ #{
+ name => id,
+ in => path,
+ schema => #{type => string},
+ required => true
+ }.
%%------------------------------------------------------------------------------
%% Rules API
%%------------------------------------------------------------------------------
-create_rule(_Bindings, Params) ->
- if_test(fun() -> test_rule_sql(Params) end,
- fun() -> do_create_rule(Params) end,
- Params).
-
-test_rule_sql(Params) ->
- case emqx_rule_sqltester:test(emqx_json:decode(emqx_json:encode(Params), [return_maps])) of
- {ok, Result} -> return({ok, Result});
- {error, nomatch} -> return({error, 404, <<"SQL Not Match">>});
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-do_create_rule(Params) ->
- case emqx_rule_engine:create_rule(parse_rule_params(Params)) of
- {ok, Rule} -> return({ok, record_to_map(Rule)});
- {error, {action_not_found, ActionName}} ->
- return({error, 400, ?ERR_NO_ACTION(ActionName)});
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-update_rule(#{id := Id}, Params) ->
- case emqx_rule_engine:update_rule(parse_rule_params(Params, #{id => Id})) of
- {ok, Rule} -> return({ok, record_to_map(Rule)});
- {error, {not_found, RuleId}} ->
- return({error, 400, ?ERR_NO_RULE(RuleId)});
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-list_rules(_Bindings, _Params) ->
- return_all(emqx_rule_registry:get_rules_ordered_by_ts()).
-
-show_rule(#{id := Id}, _Params) ->
- reply_with(fun emqx_rule_registry:get_rule/1, Id).
-
-delete_rule(#{id := Id}, _Params) ->
- ok = emqx_rule_engine:delete_rule(Id),
- return(ok).
-
-%%------------------------------------------------------------------------------
-%% Actions API
-%%------------------------------------------------------------------------------
-
-list_actions(#{}, _Params) ->
- return_all(
- sort_by_title(action,
- emqx_rule_registry:get_actions())).
-
-show_action(#{name := Name}, _Params) ->
- reply_with(fun emqx_rule_registry:find_action/1, Name).
-
-%%------------------------------------------------------------------------------
-%% Resources API
-%%------------------------------------------------------------------------------
-create_resource(#{}, Params) ->
- case parse_resource_params(Params) of
- {ok, ParsedParams} ->
- if_test(fun() -> do_create_resource(test_resource, ParsedParams) end,
- fun() -> do_create_resource(create_resource, ParsedParams) end,
- Params);
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-do_create_resource(Create, ParsedParams) ->
- case emqx_rule_engine:Create(ParsedParams) of
- ok ->
- return(ok);
- {ok, Resource} ->
- return({ok, record_to_map(Resource)});
- {error, {resource_type_not_found, Type}} ->
- return({error, 400, ?ERR_NO_RESOURCE_TYPE(Type)});
- {error, {init_resource, _}} ->
- return({error, 500, <<"Init resource failure!">>});
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-list_resources(#{}, _Params) ->
- Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()),
- Data = lists:map(fun(Res = #{id := Id}) ->
- Status = lists:all(fun(Node) ->
- case rpc:call(Node, emqx_rule_registry, find_resource_params, [Id]) of
- {ok, #resource_params{status = #{is_alive := true}}} -> true;
- _ -> false
- end
- end, ekka_mnesia:running_nodes()),
- maps:put(status, Status, Res)
- end, Data0),
- return({ok, Data}).
-
-list_resources_by_type(#{type := Type}, _Params) ->
- return_all(emqx_rule_registry:get_resources_by_type(Type)).
-
-show_resource(#{id := Id}, _Params) ->
- case emqx_rule_registry:find_resource(Id) of
- {ok, R} ->
- Status =
- [begin
- {ok, St} = rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]),
- maps:put(node, Node, St)
- end || Node <- ekka_mnesia:running_nodes()],
- return({ok, maps:put(status, Status, record_to_map(R))});
- not_found ->
- return({error, 404, <<"Not Found">>})
- end.
-
-get_resource_status(#{id := Id}, _Params) ->
- case emqx_rule_engine:get_resource_status(Id) of
- {ok, Status} ->
- return({ok, Status});
- {error, {resource_not_found, ResId}} ->
- return({error, 400, ?ERR_NO_RESOURCE(ResId)})
- end.
-
-start_resource(#{id := Id}, _Params) ->
- case emqx_rule_engine:start_resource(Id) of
- ok ->
- return(ok);
- {error, {resource_not_found, ResId}} ->
- return({error, 400, ?ERR_NO_RESOURCE(ResId)});
- {error, Reason} ->
- ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-update_resource(#{id := Id}, NewParams) ->
- P1 = case proplists:get_value(<<"description">>, NewParams) of
- undefined -> #{};
- Value -> #{<<"description">> => Value}
- end,
- P2 = case proplists:get_value(<<"config">>, NewParams) of
- undefined -> #{};
- [{}] -> #{};
- Config -> #{<<"config">> => ?RAISE(json_term_to_map(Config), {invalid_config, Config})}
- end,
- case emqx_rule_engine:update_resource(Id, maps:merge(P1, P2)) of
- ok ->
- return(ok);
- {error, not_found} ->
- return({error, 400, <<"Resource not found:", Id/binary>>});
- {error, {init_resource, _}} ->
- return({error, 500, <<"Init resource failure:", Id/binary>>});
- {error, {dependent_rules_exists, RuleIds}} ->
- return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)});
- {error, Reason} ->
- ?LOG(error, "Resource update failed: ~0p", [Reason]),
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-delete_resource(#{id := Id}, _Params) ->
- case emqx_rule_engine:delete_resource(Id) of
- ok -> return(ok);
- {error, not_found} -> return(ok);
- {error, {dependent_rules_exists, RuleIds}} ->
- return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)});
- {error, Reason} ->
- return({error, 400, ?ERR_BADARGS(Reason)})
- end.
-
-%%------------------------------------------------------------------------------
-%% Resource Types API
-%%------------------------------------------------------------------------------
-
-list_resource_types(#{}, _Params) ->
- return_all(
- sort_by_title(resource_type,
- emqx_rule_registry:get_resource_types())).
-
-show_resource_type(#{name := Name}, _Params) ->
- reply_with(fun emqx_rule_registry:find_resource_type/1, Name).
-
-
-%%------------------------------------------------------------------------------
-%% Events API
-%%------------------------------------------------------------------------------
list_events(#{}, _Params) ->
- return({ok, emqx_rule_events:event_info()}).
+ {200, emqx_rule_events:event_info()}.
+
+crud_rules(get, _Params) ->
+ Records = emqx_rule_registry:get_rules_ordered_by_ts(),
+ {200, format_rule_resp(Records)};
+
+crud_rules(post, #{body := Params}) ->
+ ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:create_rule(CheckedParams) of
+ {ok, Rule} -> {201, format_rule_resp(Rule)};
+ {error, Reason} ->
+ ?SLOG(error, #{msg => "create_rule_failed", reason => Reason}),
+ {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}}
+ end).
+
+rule_test(post, #{body := Params}) ->
+ ?CHECK_PARAMS(Params, rule_test, case emqx_rule_sqltester:test(CheckedParams) of
+ {ok, Result} -> {200, Result};
+ {error, nomatch} -> {412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}}
+ end).
+
+crud_rules_by_id(get, #{bindings := #{id := Id}}) ->
+ case emqx_rule_registry:get_rule(Id) of
+ {ok, Rule} ->
+ {200, format_rule_resp(Rule)};
+ not_found ->
+ {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}}
+ end;
+
+crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params0}) ->
+ Params = maps:merge(Params0, #{id => Id}),
+ ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:update_rule(CheckedParams) of
+ {ok, Rule} -> {200, format_rule_resp(Rule)};
+ {error, not_found} ->
+ {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}};
+ {error, Reason} ->
+ ?SLOG(error, #{msg => "update_rule_failed",
+ id => Id,
+ reason => Reason}),
+ {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}}
+ end);
+
+crud_rules_by_id(delete, #{bindings := #{id := Id}}) ->
+ case emqx_rule_engine:delete_rule(Id) of
+ ok -> {200};
+ {error, not_found} -> {200}
+ end.
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------
+err_msg(Msg) ->
+ list_to_binary(io_lib:format("~0p", [Msg])).
-if_test(True, False, Params) ->
- case proplists:get_value(<<"test">>, Params) of
- Test when Test =:= true; Test =:= <<"true">> ->
- True();
- _ ->
- False()
- end.
-return_all(Records) ->
- Data = lists:foldr(fun maybe_record_to_map/2, [], Records),
- return({ok, Data}).
+format_rule_resp(Rules) when is_list(Rules) ->
+ [format_rule_resp(R) || R <- Rules];
-maybe_record_to_map(Rec, Acc) ->
- case record_to_map(Rec) of
- ignore -> Acc;
- Map -> [Map | Acc]
- end.
-
-reply_with(Find, Key) ->
- case Find(Key) of
- {ok, R} ->
- return({ok, record_to_map(R)});
- not_found ->
- return({error, 404, <<"Not Found">>})
- end.
-
-record_to_map(#rule{id = Id,
- for = Hook,
- rawsql = RawSQL,
- actions = Actions,
- on_action_failed = OnFailed,
- enabled = Enabled,
- description = Descr}) ->
+format_rule_resp(#rule{id = Id, created_at = CreatedAt,
+ info = #{
+ from := Topics,
+ outputs := Output,
+ sql := SQL,
+ enabled := Enabled,
+ description := Descr}}) ->
#{id => Id,
- for => Hook,
- rawsql => RawSQL,
- actions => printable_actions(Actions),
- on_action_failed => OnFailed,
+ from => Topics,
+ outputs => format_output(Output),
+ sql => SQL,
metrics => get_rule_metrics(Id),
enabled => Enabled,
- description => Descr
- };
-
-record_to_map(#action{hidden = true}) ->
- ignore;
-record_to_map(#action{name = Name,
- category = Category,
- app = App,
- for = Hook,
- types = Types,
- params_spec = Params,
- title = Title,
- description = Descr}) ->
- #{name => Name,
- category => Category,
- app => App,
- for => Hook,
- types => Types,
- params => Params,
- title => Title,
- description => Descr
- };
-
-record_to_map(#resource{id = Id,
- type = Type,
- config = Config,
- description = Descr}) ->
- #{id => Id,
- type => Type,
- config => Config,
- description => Descr
- };
-
-record_to_map(#resource_type{name = Name,
- provider = Provider,
- params_spec = Params,
- title = Title,
- description = Descr}) ->
- #{name => Name,
- provider => Provider,
- params => Params,
- title => Title,
+ created_at => format_datetime(CreatedAt, millisecond),
description => Descr
}.
-printable_actions(Actions) ->
- [#{id => Id, name => Name, params => Args,
- metrics => get_action_metrics(Id),
- fallbacks => printable_actions(Fallbacks)}
- || #action_instance{id = Id, name = Name, args = Args, fallbacks = Fallbacks} <- Actions].
+format_datetime(Timestamp, Unit) ->
+ list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])).
-parse_rule_params(Params) ->
- parse_rule_params(Params, #{description => <<"">>}).
-parse_rule_params([], Rule) ->
- Rule;
-parse_rule_params([{<<"id">>, Id} | Params], Rule) ->
- parse_rule_params(Params, Rule#{id => Id});
-parse_rule_params([{<<"rawsql">>, RawSQL} | Params], Rule) ->
- parse_rule_params(Params, Rule#{rawsql => RawSQL});
-parse_rule_params([{<<"enabled">>, Enabled} | Params], Rule) ->
- parse_rule_params(Params, Rule#{enabled => enabled(Enabled)});
-parse_rule_params([{<<"on_action_failed">>, OnFailed} | Params], Rule) ->
- parse_rule_params(Params, Rule#{on_action_failed => on_failed(OnFailed)});
-parse_rule_params([{<<"actions">>, Actions} | Params], Rule) ->
- parse_rule_params(Params, Rule#{actions => parse_actions(Actions)});
-parse_rule_params([{<<"description">>, Descr} | Params], Rule) ->
- parse_rule_params(Params, Rule#{description => Descr});
-parse_rule_params([_ | Params], Rule) ->
- parse_rule_params(Params, Rule).
+format_output(Outputs) ->
+ [do_format_output(Out) || Out <- Outputs].
-on_failed(<<"continue">>) -> continue;
-on_failed(<<"stop">>) -> stop;
-on_failed(OnFailed) -> error({invalid_on_failed, OnFailed}).
-
-enabled(Enabled) when is_boolean(Enabled) -> Enabled;
-enabled(Enabled) -> error({invalid_enabled, Enabled}).
-
-parse_actions(Actions) ->
- [parse_action(json_term_to_map(A)) || A <- Actions].
-
-parse_action(Action) ->
- #{name => binary_to_existing_atom(maps:get(<<"name">>, Action), utf8),
- args => maps:get(<<"params">>, Action, #{}),
- fallbacks => parse_actions(maps:get(<<"fallbacks">>, Action, []))}.
-
-parse_resource_params(Params) ->
- parse_resource_params(Params, #{config => #{}, description => <<"">>}).
-parse_resource_params([], Res) ->
- {ok, Res};
-parse_resource_params([{<<"id">>, Id} | Params], Res) ->
- parse_resource_params(Params, Res#{id => Id});
-parse_resource_params([{<<"type">>, ResourceType} | Params], Res) ->
- try parse_resource_params(Params, Res#{type => binary_to_existing_atom(ResourceType, utf8)})
- catch error:badarg ->
- {error, {resource_type_not_found, ResourceType}}
- end;
-parse_resource_params([{<<"config">>, Config} | Params], Res) ->
- parse_resource_params(Params, Res#{config => json_term_to_map(Config)});
-parse_resource_params([{<<"description">>, Descr} | Params], Res) ->
- parse_resource_params(Params, Res#{description => Descr});
-parse_resource_params([_ | Params], Res) ->
- parse_resource_params(Params, Res).
-
-json_term_to_map(List) ->
- emqx_json:decode(emqx_json:encode(List), [return_maps]).
-
-sort_by_title(action, Actions) ->
- sort_by(#action.title, Actions);
-sort_by_title(resource_type, ResourceTypes) ->
- sort_by(#resource_type.title, ResourceTypes).
-
-sort_by(Pos, TplList) ->
- lists:sort(
- fun(RecA, RecB) ->
- maps:get(en, element(Pos, RecA), 0)
- =< maps:get(en, element(Pos, RecB), 0)
- end, TplList).
+do_format_output(#{type := func}) ->
+ #{type => func, target => <<"internal_function">>};
+do_format_output(#{type := builtin, target := Name, args := Args}) ->
+ #{type => builtin, target => Name, args => maps:remove(preprocessed_tmpl, Args)};
+do_format_output(#{type := bridge, target := Name}) ->
+ #{type => bridge, target => Name}.
get_rule_metrics(Id) ->
[maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_rule_metrics, [Id]))
|| Node <- ekka_mnesia:running_nodes()].
-
-get_action_metrics(Id) ->
- [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id]))
- || Node <- ekka_mnesia:running_nodes()].
-
-%% TODO: V5 API
-return(_) -> ok.
\ No newline at end of file
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl
index 5893f9827..e9b71f89d 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl
@@ -23,14 +23,8 @@
-export([stop/1]).
start(_Type, _Args) ->
- {ok, Sup} = emqx_rule_engine_sup:start_link(),
- _ = emqx_rule_engine_sup:start_locker(),
- ok = emqx_rule_engine:load_providers(),
- ok = emqx_rule_engine:refresh_resources(),
- ok = emqx_rule_engine:refresh_rules(),
- ok = emqx_rule_engine_cli:load(),
- {ok, Sup}.
+ ok = emqx_rule_events:reload(),
+ emqx_rule_engine_sup:start_link().
stop(_State) ->
- ok = emqx_rule_events:unload(),
- ok = emqx_rule_engine_cli:unload().
+ ok = emqx_rule_events:unload().
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl
deleted file mode 100644
index bcb869aec..000000000
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl
+++ /dev/null
@@ -1,387 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_engine_cli).
-
--include("rule_engine.hrl").
-
--export([ load/0
- , commands/0
- , unload/0
- ]).
-
--export([ rules/1
- , actions/1
- , resources/1
- , resource_types/1
- ]).
-
--import(proplists, [get_value/2]).
-
--define(OPTSPEC_RESOURCE_TYPE,
- [{type, $t, "type", {atom, undefined}, "Resource Type"}]).
--define(OPTSPEC_ACTION_TYPE,
- [ {eventype, $k, "eventype", {atom, undefined}, "Event Type"}
- ]).
-
--define(OPTSPEC_RESOURCES_CREATE,
- [ {type, undefined, undefined, atom, "Resource Type"}
- , {id, $i, "id", {binary, <<"">>}, "The resource id. A random resource id will be used if not provided"}
- , {config, $c, "config", {binary, <<"{}">>}, "Config"}
- , {descr, $d, "descr", {binary, <<"">>}, "Description"}
- ]).
-
--define(OPTSPEC_RESOURCES_UPDATE,
- [ {id, undefined, undefined, binary, "The resource id"}
- , {config, $c, "config", {binary, undefined}, "Config"}
- , {description, $d, "descr", {binary, undefined}, "Description"}
- ]).
-
--define(OPTSPEC_RULES_CREATE,
- [ {sql, undefined, undefined, binary, "Filter Condition SQL"}
- , {actions, undefined, undefined, binary, "Action List in JSON format: [{\"name\": , \"params\": {: }}]"}
- , {id, $i, "id", {binary, <<"">>}, "The rule id. A random rule id will be used if not provided"}
- , {enabled, $e, "enabled", {atom, true}, "'true' or 'false' to enable or disable the rule"}
- , {on_action_failed, $g, "on_action_failed", {atom, continue}, "'continue' or 'stop' when an action in the rule fails"}
- , {descr, $d, "descr", {binary, <<"">>}, "Description"}
- ]).
-
--define(OPTSPEC_RULES_UPDATE,
- [ {id, undefined, undefined, binary, "Rule ID"}
- , {sql, $s, "sql", {binary, undefined}, "Filter Condition SQL"}
- , {actions, $a, "actions", {binary, undefined}, "Action List in JSON format: [{\"name\": , \"params\": {: }}]"}
- , {enabled, $e, "enabled", {atom, undefined}, "'true' or 'false' to enable or disable the rule"}
- , {on_action_failed, $g, "on_action_failed", {atom, undefined}, "'continue' or 'stop' when an action in the rule fails"}
- , {descr, $d, "descr", {binary, undefined}, "Description"}
- ]).
-%%-----------------------------------------------------------------------------
-%% Load/Unload Commands
-%%-----------------------------------------------------------------------------
-
--spec(load() -> ok).
-load() ->
- lists:foreach(
- fun({Cmd, Func}) ->
- emqx_ctl:register_command(Cmd, {?MODULE, Func}, []);
- (Cmd) ->
- emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, [])
- end, commands()).
-
--spec(commands() -> list(atom())).
-commands() ->
- [rules, {'rule-actions', actions}, resources, {'resource-types', resource_types}].
-
--spec(unload() -> ok).
-unload() ->
- lists:foreach(
- fun({Cmd, _Func}) ->
- emqx_ctl:unregister_command(Cmd);
- (Cmd) ->
- emqx_ctl:unregister_command(Cmd)
- end, commands()).
-
-%%-----------------------------------------------------------------------------
-%% 'rules' command
-%%-----------------------------------------------------------------------------
--dialyzer([{nowarn_function, [rules/1]}]).
-rules(["list"]) ->
- print_all(emqx_rule_registry:get_rules_ordered_by_ts());
-
-rules(["show", RuleId]) ->
- print_with(fun emqx_rule_registry:get_rule/1, list_to_binary(RuleId));
-
-rules(["create" | Params]) ->
- with_opts(fun({Opts, _}) ->
- case emqx_rule_engine:create_rule(make_rule(Opts)) of
- {ok, #rule{id = RuleId}} ->
- emqx_ctl:print("Rule ~s created~n", [RuleId]);
- {error, Reason} ->
- emqx_ctl:print("Invalid options: ~0p~n", [Reason])
- end
- end, Params, ?OPTSPEC_RULES_CREATE, {?FUNCTION_NAME, create});
-
-rules(["update" | Params]) ->
- with_opts(fun({Opts, _}) ->
- case emqx_rule_engine:update_rule(make_updated_rule(Opts)) of
- {ok, #rule{id = RuleId}} ->
- emqx_ctl:print("Rule ~s updated~n", [RuleId]);
- {error, Reason} ->
- emqx_ctl:print("Invalid options: ~0p~n", [Reason])
- end
- end, Params, ?OPTSPEC_RULES_UPDATE, {?FUNCTION_NAME, update});
-
-rules(["delete", RuleId]) ->
- ok = emqx_rule_engine:delete_rule(list_to_binary(RuleId)),
- emqx_ctl:print("ok~n");
-
-rules(_Usage) ->
- emqx_ctl:usage([{"rules list", "List all rules"},
- {"rules show ", "Show a rule"},
- {"rules create", "Create a rule"},
- {"rules delete ", "Delete a rule"}
- ]).
-
-%%-----------------------------------------------------------------------------
-%% 'rule-actions' command
-%%-----------------------------------------------------------------------------
-
-actions(["list"]) ->
- print_all(get_actions());
-
-actions(["show", ActionId]) ->
- print_with(fun emqx_rule_registry:find_action/1,
- ?RAISE(list_to_existing_atom(ActionId), {not_found, ActionId}));
-
-actions(_Usage) ->
- emqx_ctl:usage([{"rule-actions list", "List actions"},
- {"rule-actions show ", "Show a rule action"}
- ]).
-
-%%------------------------------------------------------------------------------
-%% 'resources' command
-%%------------------------------------------------------------------------------
-
-resources(["create" | Params]) ->
- with_opts(fun({Opts, _}) ->
- case emqx_rule_engine:create_resource(make_resource(Opts)) of
- {ok, #resource{id = ResId}} ->
- emqx_ctl:print("Resource ~s created~n", [ResId]);
- {error, Reason} ->
- emqx_ctl:print("Invalid options: ~0p~n", [Reason])
- end
- end, Params, ?OPTSPEC_RESOURCES_CREATE, {?FUNCTION_NAME, create});
-
-
-resources(["update" | Params]) ->
- with_opts(fun({Opts, _}) ->
- Id = proplists:get_value(id, Opts),
- Maps = make_updated_resource(Opts),
- case emqx_rule_engine:update_resource(Id, Maps) of
- ok ->
- emqx_ctl:print("Resource update successfully~n");
- {error, Reason} ->
- emqx_ctl:print("Resource update failed: ~0p~n", [Reason])
- end
- end, Params, ?OPTSPEC_RESOURCES_UPDATE, {?FUNCTION_NAME, update});
-
-resources(["test" | Params]) ->
- with_opts(fun({Opts, _}) ->
- case emqx_rule_engine:test_resource(make_resource(Opts)) of
- ok ->
- emqx_ctl:print("Test creating resource successfully (dry-run)~n");
- {error, Reason} ->
- emqx_ctl:print("Test creating resource failed: ~0p~n", [Reason])
- end
- end, Params, ?OPTSPEC_RESOURCES_CREATE, {?FUNCTION_NAME, test});
-
-resources(["list"]) ->
- print_all(emqx_rule_registry:get_resources());
-
-resources(["list" | Params]) ->
- with_opts(fun({Opts, _}) ->
- print_all(emqx_rule_registry:get_resources_by_type(
- get_value(type, Opts)))
- end, Params, ?OPTSPEC_RESOURCE_TYPE, {?FUNCTION_NAME, list});
-
-resources(["show", ResourceId]) ->
- print_with(fun emqx_rule_registry:find_resource/1, list_to_binary(ResourceId));
-
-resources(["delete", ResourceId]) ->
- case emqx_rule_engine:delete_resource(list_to_binary(ResourceId)) of
- ok -> emqx_ctl:print("ok~n");
- {error, not_found} -> emqx_ctl:print("ok~n");
- {error, Reason} ->
- emqx_ctl:print("Cannot delete resource as ~0p~n", [Reason])
- end;
-
-resources(_Usage) ->
- emqx_ctl:usage([{"resources create", "Create a resource"},
- {"resources list [-t ]", "List resources"},
- {"resources show ", "Show a resource"},
- {"resources delete ", "Delete a resource"},
- {"resources update [-c ] [-d ]", "Update a resource"}
- ]).
-
-%%------------------------------------------------------------------------------
-%% 'resource-types' command
-%%------------------------------------------------------------------------------
-resource_types(["list"]) ->
- print_all(emqx_rule_registry:get_resource_types());
-
-resource_types(["show", Name]) ->
- print_with(fun emqx_rule_registry:find_resource_type/1, list_to_atom(Name));
-
-resource_types(_Usage) ->
- emqx_ctl:usage([{"resource-types list", "List all resource-types"},
- {"resource-types show ", "Show a resource-type"}
- ]).
-
-%%------------------------------------------------------------------------------
-%% Internal functions
-%%------------------------------------------------------------------------------
-
-print(Data) ->
- emqx_ctl:print(untilde(format(Data))).
-
-print_all(DataList) ->
- lists:map(fun(Data) ->
- print(Data)
- end, DataList).
-
-print_with(FindFun, Key) ->
- case FindFun(Key) of
- {ok, R} ->
- print(R);
- not_found ->
- emqx_ctl:print("Cannot found ~s~n", [Key])
- end.
-
-format(#rule{id = Id,
- for = Hook,
- rawsql = Sql,
- actions = Actions,
- on_action_failed = OnFailed,
- enabled = Enabled,
- description = Descr}) ->
- lists:flatten(io_lib:format("rule(id='~s', for='~0p', rawsql='~s', actions=~0p, on_action_failed='~s', metrics=~0p, enabled='~s', description='~s')~n", [Id, Hook, rmlf(Sql), printable_actions(Actions), OnFailed, get_rule_metrics(Id), Enabled, Descr]));
-
-format(#action{hidden = true}) ->
- ok;
-format(#action{name = Name,
- for = Hook,
- app = App,
- types = Types,
- title = #{en := Title},
- description = #{en := Descr}}) ->
- lists:flatten(io_lib:format("action(name='~s', app='~s', for='~s', types=~0p, title ='~s', description='~s')~n", [Name, App, Hook, Types, Title, Descr]));
-
-format(#resource{id = Id,
- type = Type,
- config = Config,
- description = Descr}) ->
- Status =
- [begin
- {ok, St} = rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]),
- maps:put(node, Node, St)
- end || Node <- [node()| nodes()]],
- lists:flatten(io_lib:format("resource(id='~s', type='~s', config=~0p, status=~0p, description='~s')~n", [Id, Type, Config, Status, Descr]));
-
-format(#resource_type{name = Name,
- provider = Provider,
- title = #{en := Title},
- description = #{en := Descr}}) ->
- lists:flatten(io_lib:format("resource_type(name='~s', provider='~s', title ='~s', description='~s')~n", [Name, Provider, Title, Descr])).
-
-make_rule(Opts) ->
- Actions = get_value(actions, Opts),
- may_with_opt(
- #{rawsql => get_value(sql, Opts),
- enabled => get_value(enabled, Opts),
- actions => parse_actions(emqx_json:decode(Actions, [return_maps])),
- on_action_failed => on_failed(get_value(on_action_failed, Opts)),
- description => get_value(descr, Opts)}, id, <<"">>, Opts).
-
-make_updated_rule(Opts) ->
- KeyNameParsers = [{sql, rawsql, fun(SQL) -> SQL end},
- enabled,
- {actions, actions, fun(Actions) ->
- parse_actions(emqx_json:decode(Actions, [return_maps]))
- end},
- on_action_failed,
- {descr, description, fun(Descr) -> Descr end}],
- lists:foldl(fun
- ({Key, Name, Parser}, ParamsAcc) ->
- case get_value(Key, Opts) of
- undefined -> ParamsAcc;
- Val -> ParamsAcc#{Name => Parser(Val)}
- end;
- (Key, ParamsAcc) ->
- case get_value(Key, Opts) of
- undefined -> ParamsAcc;
- Val -> ParamsAcc#{Key => Val}
- end
- end, #{id => get_value(id, Opts)}, KeyNameParsers).
-
-make_resource(Opts) ->
- Config = get_value(config, Opts),
- may_with_opt(
- #{type => get_value(type, Opts),
- config => ?RAISE(emqx_json:decode(Config, [return_maps]), {invalid_config, Config}),
- description => get_value(descr, Opts)}, id, <<"">>, Opts).
-
-make_updated_resource(Opts) ->
- P1 = case proplists:get_value(description, Opts) of
- undefined -> #{};
- Value -> #{<<"description">> => Value}
- end,
- P2 = case proplists:get_value(config, Opts) of
- undefined -> #{};
- Map -> #{<<"config">> => ?RAISE((emqx_json:decode(Map, [return_maps])), {invalid_config, Map})}
- end,
- maps:merge(P1, P2).
-
-printable_actions(Actions) when is_list(Actions) ->
- emqx_json:encode([#{id => Id, name => Name, params => Args,
- metrics => get_action_metrics(Id),
- fallbacks => printable_actions(Fallbacks)}
- || #action_instance{id = Id, name = Name, args = Args, fallbacks = Fallbacks} <- Actions]).
-
-may_with_opt(Params, OptName, DefaultVal, Options) when is_map(Params) ->
- case get_value(OptName, Options) of
- DefaultVal -> Params;
- Val -> Params#{OptName => Val}
- end.
-
-with_opts(Action, RawParams, OptSpecList, {CmdObject, CmdName}) ->
- case getopt:parse_and_check(OptSpecList, RawParams) of
- {ok, Params} ->
- Action(Params);
- {error, Reason} ->
- getopt:usage(OptSpecList,
- io_lib:format("emqx_ctl ~s ~s", [CmdObject, CmdName]), standard_io),
- emqx_ctl:print("~0p~n", [Reason])
- end.
-
-parse_actions(Actions) ->
- ?RAISE([parse_action(Action) || Action <- Actions],
- {invalid_action_params, {_EXCLASS_,_EXCPTION_,_ST_}}).
-
-parse_action(Action) ->
- ActName = maps:get(<<"name">>, Action),
- #{name => ?RAISE(binary_to_existing_atom(ActName, utf8), {action_not_found, ActName}),
- args => maps:get(<<"params">>, Action, #{}),
- fallbacks => parse_actions(maps:get(<<"fallbacks">>, Action, []))}.
-
-get_actions() ->
- emqx_rule_registry:get_actions().
-
-get_rule_metrics(Id) ->
- [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_rule_metrics, [Id]))
- || Node <- [node()| nodes()]].
-
-get_action_metrics(Id) ->
- [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id]))
- || Node <- [node()| nodes()]].
-
-on_failed(continue) -> continue;
-on_failed(stop) -> stop;
-on_failed(OnFailed) -> error({invalid_on_failed, OnFailed}).
-
-rmlf(Str) ->
- re:replace(Str, "\n", "", [global]).
-
-untilde(Str) ->
- re:replace(Str, "~", "&&", [{return, list}, global]).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
index 9ff5ce741..4fad54a4b 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
@@ -22,17 +22,12 @@
-export([start_link/0]).
--export([start_locker/0]).
-
-export([init/1]).
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->
- Opts = [public, named_table, set, {read_concurrency, true}],
- _ = ets:new(?ACTION_INST_PARAMS_TAB, [{keypos, #action_instance_params.id}|Opts]),
- _ = ets:new(?RES_PARAMS_TAB, [{keypos, #resource_params.id}|Opts]),
Registry = #{id => emqx_rule_registry,
start => {emqx_rule_registry, start_link, []},
restart => permanent,
@@ -45,19 +40,4 @@ init([]) ->
shutdown => 5000,
type => worker,
modules => [emqx_rule_metrics]},
- Monitor = #{id => emqx_rule_monitor,
- start => {emqx_rule_monitor, start_link, []},
- restart => permanent,
- shutdown => 5000,
- type => worker,
- modules => [emqx_rule_monitor]},
- {ok, {{one_for_one, 10, 10}, [Registry, Metrics, Monitor]}}.
-
-start_locker() ->
- Locker = #{id => emqx_rule_locker,
- start => {emqx_rule_locker, start_link, []},
- restart => permanent,
- shutdown => 5000,
- type => worker,
- modules => [emqx_rule_locker]},
- supervisor:start_child(?MODULE, Locker).
+ {ok, {{one_for_one, 10, 10}, [Registry, Metrics]}}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl
index a0960df25..d030917ef 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_events.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl
@@ -21,7 +21,8 @@
-include_lib("emqx/include/logger.hrl").
--export([ load/1
+-export([ reload/0
+ , load/1
, unload/0
, unload/1
, event_name/1
@@ -36,6 +37,7 @@
, on_message_dropped/4
, on_message_delivered/3
, on_message_acked/3
+ , on_bridge_message_received/2
]).
-export([ event_info/0
@@ -61,6 +63,12 @@
]).
-endif.
+reload() ->
+ emqx_rule_registry:load_hooks_for_rule(emqx_rule_registry:get_rules()).
+
+load(<<"$bridges/", _ChannelId/binary>> = BridgeTopic) ->
+ emqx_hooks:put(BridgeTopic, {?MODULE, on_bridge_message_received,
+ [#{bridge_topic => BridgeTopic}]});
load(Topic) ->
HookPoint = event_name(Topic),
emqx_hooks:put(HookPoint, {?MODULE, hook_fun(HookPoint), [[]]}).
@@ -77,12 +85,17 @@ unload(Topic) ->
%%--------------------------------------------------------------------
%% Callbacks
%%--------------------------------------------------------------------
+on_bridge_message_received(Message, #{bridge_topic := BridgeTopic}) ->
+ case emqx_rule_registry:get_rules_for_topic(BridgeTopic) of
+ [] -> ok;
+ Rules -> emqx_rule_runtime:apply_rules(Rules, Message)
+ end.
+
on_message_publish(Message = #message{topic = Topic}, _Env) ->
case ignore_sys_message(Message) of
- true ->
- ok;
+ true -> ok;
false ->
- case emqx_rule_registry:get_rules_for(Topic) of
+ case emqx_rule_registry:get_rules_for_topic(Topic) of
[] -> ok;
Rules -> emqx_rule_runtime:apply_rules(Rules, eventmsg_publish(Message))
end
@@ -297,7 +310,7 @@ with_basic_columns(EventName, Data) when is_map(Data) ->
%%--------------------------------------------------------------------
apply_event(EventName, GenEventMsg, _Env) ->
EventTopic = event_topic(EventName),
- case emqx_rule_registry:get_rules_for(EventTopic) of
+ case emqx_rule_registry:get_rules_for_topic(EventTopic) of
[] -> ok;
Rules -> emqx_rule_runtime:apply_rules(Rules, GenEventMsg())
end.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
index f73858f32..fd922db86 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
@@ -17,6 +17,8 @@
-module(emqx_rule_funcs).
-include("rule_engine.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
%% IoT Funcs
-export([ msgid/0
@@ -313,8 +315,10 @@ null() ->
'+'(X, Y) when is_number(X), is_number(Y) ->
X + Y;
-%% concat 2 strings
-'+'(X, Y) when is_binary(X), is_binary(Y) ->
+%% string concatenation
+%% this requires one of the arguments is string, the other argument will be converted
+%% to string automatically (implicit conversion)
+'+'(X, Y) when is_binary(X); is_binary(Y) ->
concat(X, Y).
'-'(X, Y) when is_number(X), is_number(Y) ->
@@ -615,8 +619,9 @@ tokens(S, Separators) ->
tokens(S, Separators, <<"nocrlf">>) ->
[list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])].
-concat(S1, S2) when is_binary(S1), is_binary(S2) ->
- unicode:characters_to_binary([S1, S2], unicode).
+%% implicit convert args to strings, and then do concatenation
+concat(S1, S2) ->
+ unicode:characters_to_binary([str(S1), str(S2)], unicode).
sprintf_s(Format, Args) when is_list(Args) ->
erlang:iolist_to_binary(io_lib:format(binary_to_list(Format), Args)).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_locker.erl b/apps/emqx_rule_engine/src/emqx_rule_locker.erl
deleted file mode 100644
index 9e45b8c09..000000000
--- a/apps/emqx_rule_engine/src/emqx_rule_locker.erl
+++ /dev/null
@@ -1,34 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_locker).
-
--export([start_link/0]).
-
--export([ lock/1
- , unlock/1
- ]).
-
-start_link() ->
- ekka_locker:start_link(?MODULE).
-
--spec(lock(binary()) -> ekka_locker:lock_result()).
-lock(Id) ->
- ekka_locker:acquire(?MODULE, Id, local).
-
--spec(unlock(binary()) -> {boolean(), [node()]}).
-unlock(Id) ->
- ekka_locker:release(?MODULE, Id, local).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl
index 8db444a7c..990911801 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl
@@ -26,42 +26,17 @@
]).
-export([ get_rules_matched/1
- , get_actions_taken/1
- , get_actions_success/1
- , get_actions_error/1
- , get_actions_exception/1
- , get_actions_retry/1
- ]).
-
--export([ inc_rules_matched/1
- , inc_rules_matched/2
- , inc_actions_taken/1
- , inc_actions_taken/2
- , inc_actions_success/1
- , inc_actions_success/2
- , inc_actions_error/1
- , inc_actions_error/2
- , inc_actions_exception/1
- , inc_actions_exception/2
- , inc_actions_retry/1
- , inc_actions_retry/2
]).
-export([ inc/2
, inc/3
, get/2
- , get_overall/1
, get_rule_speed/1
- , get_overall_rule_speed/0
, create_rule_metrics/1
- , create_metrics/1
, clear_rule_metrics/1
- , clear_metrics/1
- , overall_metrics/0
]).
-export([ get_rule_metrics/1
- , get_action_metrics/1
]).
%% gen_server callbacks
@@ -82,7 +57,7 @@
-define(SAMPLING, 1).
-endif.
--define(CRefID(ID), {?MODULE, ID}).
+-define(CntrRef, ?MODULE).
-define(SAMPCOUNT_5M, (?SECS_5M div ?SAMPLING)).
-record(rule_speed, {
@@ -99,48 +74,32 @@
-record(state, {
metric_ids = sets:new(),
- rule_speeds :: undefined | #{rule_id() => #rule_speed{}},
- overall_rule_speed :: #rule_speed{}
+ rule_speeds :: undefined | #{rule_id() => #rule_speed{}}
}).
%%------------------------------------------------------------------------------
%% APIs
%%------------------------------------------------------------------------------
+
-spec(create_rule_metrics(rule_id()) -> ok).
create_rule_metrics(Id) ->
gen_server:call(?MODULE, {create_rule_metrics, Id}).
--spec(create_metrics(rule_id()) -> ok).
-create_metrics(Id) ->
- gen_server:call(?MODULE, {create_metrics, Id}).
-
-spec(clear_rule_metrics(rule_id()) -> ok).
clear_rule_metrics(Id) ->
gen_server:call(?MODULE, {delete_rule_metrics, Id}).
--spec(clear_metrics(rule_id()) -> ok).
-clear_metrics(Id) ->
- gen_server:call(?MODULE, {delete_metrics, Id}).
-
-spec(get(rule_id(), atom()) -> number()).
get(Id, Metric) ->
- case couters_ref(Id) of
+ case get_couters_ref(Id) of
not_found -> 0;
Ref -> counters:get(Ref, metrics_idx(Metric))
end.
--spec(get_overall(atom()) -> number()).
-get_overall(Metric) ->
- emqx_metrics:val(Metric).
-
-spec(get_rule_speed(rule_id()) -> map()).
get_rule_speed(Id) ->
gen_server:call(?MODULE, {get_rule_speed, Id}).
--spec(get_overall_rule_speed() -> map()).
-get_overall_rule_speed() ->
- gen_server:call(?MODULE, get_overall_rule_speed).
-
-spec(get_rule_metrics(rule_id()) -> map()).
get_rule_metrics(Id) ->
#{max := Max, current := Current, last5m := Last5M} = get_rule_speed(Id),
@@ -150,95 +109,34 @@ get_rule_metrics(Id) ->
speed_last5m => Last5M
}.
--spec(get_action_metrics(action_instance_id()) -> map()).
-get_action_metrics(Id) ->
- #{success => get_actions_success(Id),
- failed => get_actions_error(Id) + get_actions_exception(Id),
- taken => get_actions_taken(Id)
- }.
-
-spec inc(rule_id(), atom()) -> ok.
inc(Id, Metric) ->
inc(Id, Metric, 1).
-spec inc(rule_id(), atom(), pos_integer()) -> ok.
inc(Id, Metric, Val) ->
- case couters_ref(Id) of
+ case get_couters_ref(Id) of
not_found ->
%% this may occur when increasing a counter for
%% a rule that was created from a remove node.
- case atom_to_list(Metric) of
- "rules." ++ _ -> create_rule_metrics(Id);
- _ -> create_metrics(Id)
- end,
- counters:add(couters_ref(Id), metrics_idx(Metric), Val);
+ create_rule_metrics(Id),
+ counters:add(get_couters_ref(Id), metrics_idx(Metric), Val);
Ref ->
counters:add(Ref, metrics_idx(Metric), Val)
- end,
- inc_overall(Metric, Val).
-
--spec(inc_overall(atom(), pos_integer()) -> ok).
-inc_overall(Metric, Val) ->
- emqx_metrics:inc(Metric, Val).
-
-inc_rules_matched(Id) ->
- inc_rules_matched(Id, 1).
-inc_rules_matched(Id, Val) ->
- inc(Id, 'rules.matched', Val).
-
-inc_actions_taken(Id) ->
- inc_actions_taken(Id, 1).
-inc_actions_taken(Id, Val) ->
- inc(Id, 'actions.taken', Val).
-
-inc_actions_success(Id) ->
- inc_actions_success(Id, 1).
-inc_actions_success(Id, Val) ->
- inc(Id, 'actions.success', Val).
-
-inc_actions_error(Id) ->
- inc_actions_error(Id, 1).
-inc_actions_error(Id, Val) ->
- inc(Id, 'actions.error', Val).
-
-inc_actions_exception(Id) ->
- inc_actions_exception(Id, 1).
-inc_actions_exception(Id, Val) ->
- inc(Id, 'actions.exception', Val).
-
-inc_actions_retry(Id) ->
- inc_actions_retry(Id, 1).
-inc_actions_retry(Id, Val) ->
- inc(Id, 'actions.retry', Val).
+ end.
get_rules_matched(Id) ->
get(Id, 'rules.matched').
-get_actions_taken(Id) ->
- get(Id, 'actions.taken').
-
-get_actions_success(Id) ->
- get(Id, 'actions.success').
-
-get_actions_error(Id) ->
- get(Id, 'actions.error').
-
-get_actions_exception(Id) ->
- get(Id, 'actions.exception').
-
-get_actions_retry(Id) ->
- get(Id, 'actions.retry').
-
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init([]) ->
erlang:process_flag(trap_exit, true),
- %% the overall counters
- [ok = emqx_metrics:ensure(Metric)|| Metric <- overall_metrics()],
%% the speed metrics
erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
- {ok, #state{overall_rule_speed = #rule_speed{}}}.
+ persistent_term:put(?CntrRef, #{}),
+ {ok, #state{}}.
handle_call({get_rule_speed, _Id}, _From, State = #state{rule_speeds = undefined}) ->
{reply, format_rule_speed(#rule_speed{}), State};
@@ -248,12 +146,6 @@ handle_call({get_rule_speed, Id}, _From, State = #state{rule_speeds = RuleSpeeds
Speed -> format_rule_speed(Speed)
end, State};
-handle_call(get_overall_rule_speed, _From, State = #state{overall_rule_speed = RuleSpeed}) ->
- {reply, format_rule_speed(RuleSpeed), State};
-
-handle_call({create_metrics, Id}, _From, State = #state{metric_ids = MIDs}) ->
- {reply, create_counters(Id), State#state{metric_ids = sets:add_element(Id, MIDs)}};
-
handle_call({create_rule_metrics, Id}, _From,
State = #state{metric_ids = MIDs, rule_speeds = RuleSpeeds}) ->
{reply, create_counters(Id),
@@ -263,10 +155,6 @@ handle_call({create_rule_metrics, Id}, _From,
_ -> RuleSpeeds#{Id => #rule_speed{}}
end}};
-handle_call({delete_metrics, Id}, _From,
- State = #state{metric_ids = MIDs, rule_speeds = undefined}) ->
- {reply, delete_counters(Id), State#state{metric_ids = sets:del_element(Id, MIDs)}};
-
handle_call({delete_rule_metrics, Id}, _From,
State = #state{metric_ids = MIDs, rule_speeds = RuleSpeeds}) ->
{reply, delete_counters(Id),
@@ -283,21 +171,16 @@ handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(ticking, State = #state{rule_speeds = undefined}) ->
- async_refresh_resource_status(),
erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
{noreply, State};
-handle_info(ticking, State = #state{rule_speeds = RuleSpeeds0,
- overall_rule_speed = OverallRuleSpeed0}) ->
+handle_info(ticking, State = #state{rule_speeds = RuleSpeeds0}) ->
RuleSpeeds = maps:map(
fun(Id, RuleSpeed) ->
calculate_speed(get_rules_matched(Id), RuleSpeed)
end, RuleSpeeds0),
- OverallRuleSpeed = calculate_speed(get_overall('rules.matched'), OverallRuleSpeed0),
- async_refresh_resource_status(),
erlang:send_after(timer:seconds(?SAMPLING), self(), ticking),
- {noreply, State#state{rule_speeds = RuleSpeeds,
- overall_rule_speed = OverallRuleSpeed}};
+ {noreply, State#state{rule_speeds = RuleSpeeds}};
handle_info(_Info, State) ->
{noreply, State}.
@@ -307,7 +190,7 @@ code_change(_OldVsn, State, _Extra) ->
terminate(_Reason, #state{metric_ids = MIDs}) ->
[delete_counters(Id) || Id <- sets:to_list(MIDs)],
- persistent_term:erase(?MODULE).
+ persistent_term:erase(?CntrRef).
stop() ->
gen_server:stop(?MODULE).
@@ -316,26 +199,23 @@ stop() ->
%% Internal Functions
%%------------------------------------------------------------------------------
-async_refresh_resource_status() ->
- spawn(emqx_rule_engine, refresh_resource_status, []).
-
create_counters(Id) ->
- case couters_ref(Id) of
+ case get_couters_ref(Id) of
not_found ->
- ok = persistent_term:put(?CRefID(Id),
- counters:new(max_counters_size(), [write_concurrency]));
+ Counters = get_all_counters(),
+ CntrRef = counters:new(max_counters_size(), [write_concurrency]),
+ persistent_term:put(?CntrRef, Counters#{Id => CntrRef});
_Ref -> ok
end.
delete_counters(Id) ->
- persistent_term:erase(?CRefID(Id)),
- ok.
+ persistent_term:put(?CntrRef, maps:remove(Id, get_all_counters())).
-couters_ref(Id) ->
- try persistent_term:get(?CRefID(Id))
- catch
- error:badarg -> not_found
- end.
+get_couters_ref(Id) ->
+ maps:get(Id, get_all_counters(), not_found).
+
+get_all_counters() ->
+ persistent_term:get(?CntrRef, #{}).
calculate_speed(_CurrVal, undefined) ->
undefined;
@@ -379,21 +259,7 @@ precision(Float, N) ->
%% Metrics Definitions
%%------------------------------------------------------------------------------
-max_counters_size() -> 7.
-
+max_counters_size() -> 2.
metrics_idx('rules.matched') -> 1;
-metrics_idx('actions.success') -> 2;
-metrics_idx('actions.error') -> 3;
-metrics_idx('actions.taken') -> 4;
-metrics_idx('actions.exception') -> 5;
-metrics_idx('actions.retry') -> 6;
-metrics_idx(_) -> 7.
+metrics_idx(_) -> 2.
-overall_metrics() ->
- [ 'rules.matched'
- , 'actions.success'
- , 'actions.error'
- , 'actions.taken'
- , 'actions.exception'
- , 'actions.retry'
- ].
diff --git a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl
deleted file mode 100644
index dd2f6237c..000000000
--- a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl
+++ /dev/null
@@ -1,126 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_monitor).
-
--behavior(gen_server).
-
--include("rule_engine.hrl").
--include_lib("emqx/include/logger.hrl").
-
--export([init/1,
- handle_call/3,
- handle_cast/2,
- handle_info/2,
- terminate/2,
- code_change/3]).
-
--export([ start_link/0
- , stop/0
- , ensure_resource_retrier/2
- , retry_loop/3
- ]).
-
-start_link() ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-stop() ->
- gen_server:stop(?MODULE).
-
-init([]) ->
- _ = erlang:process_flag(trap_exit, true),
- {ok, #{retryers => #{}}}.
-
-ensure_resource_retrier(ResId, Interval) ->
- gen_server:cast(?MODULE, {create_restart_handler, resource, ResId, Interval}).
-
-handle_call(_Msg, _From, State) ->
- {reply, ok, State}.
-
-handle_cast({create_restart_handler, Tag, Obj, Interval}, State) ->
- Objects = maps:get(Tag, State, #{}),
- NewState = case maps:find(Obj, Objects) of
- error ->
- update_object(Tag, Obj,
- create_restart_handler(Tag, Obj, Interval), State);
- {ok, _Pid} ->
- State
- end,
- {noreply, NewState};
-
-handle_cast(_Msg, State) ->
- {noreply, State}.
-
-handle_info({'EXIT', Pid, Reason}, State = #{retryers := Retryers}) ->
- case maps:take(Pid, Retryers) of
- {{Tag, Obj}, Retryers2} ->
- Objects = maps:get(Tag, State, #{}),
- {noreply, State#{Tag => maps:remove(Obj, Objects),
- retryers => Retryers2}};
- error ->
- ?LOG(error, "got unexpected proc down: ~p ~p", [Pid, Reason]),
- {noreply, State}
- end;
-
-handle_info(_Info, State) ->
- {noreply, State}.
-
-terminate(_Reason, _State) ->
- ok.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-update_object(Tag, Obj, Retryer, State) ->
- Objects = maps:get(Tag, State, #{}),
- Retryers = maps:get(retryers, State, #{}),
- State#{
- Tag => Objects#{Obj => Retryer},
- retryers => Retryers#{Retryer => {Tag, Obj}}
- }.
-
-create_restart_handler(Tag, Obj, Interval) ->
- ?LOG(info, "keep restarting ~p ~p, interval: ~p", [Tag, Obj, Interval]),
- %% spawn a dedicated process to handle the restarting asynchronously
- spawn_link(?MODULE, retry_loop, [Tag, Obj, Interval]).
-
-retry_loop(resource, ResId, Interval) ->
- case emqx_rule_registry:find_resource(ResId) of
- {ok, #resource{type = Type, config = Config}} ->
- try
- {ok, #resource_type{on_create = {M, F}}} =
- emqx_rule_registry:find_resource_type(Type),
- ok = emqx_rule_engine:init_resource(M, F, ResId, Config),
- refresh_and_enable_rules_of_resource(ResId)
- catch
- Err:Reason:ST ->
- ?LOG(warning, "init_resource failed: ~p, ~0p",
- [{Err, Reason}, ST]),
- timer:sleep(Interval),
- ?MODULE:retry_loop(resource, ResId, Interval)
- end;
- not_found ->
- ok
- end.
-
-refresh_and_enable_rules_of_resource(ResId) ->
- lists:foreach(
- fun (#rule{id = Id, enabled = false, state = refresh_failed_at_bootup} = Rule) ->
- emqx_rule_engine:refresh_rule(Rule),
- emqx_rule_registry:add_rule(Rule#rule{enabled = true, state = normal}),
- ?LOG(info, "rule ~s is refreshed and re-enabled", [Id]);
- (_) -> ok
- end, emqx_rule_registry:find_rules_depends_on_resource(ResId)).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
new file mode 100644
index 000000000..cd59d3fa5
--- /dev/null
+++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
@@ -0,0 +1,82 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% Define the default actions.
+-module(emqx_rule_outputs).
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+
+-export([ console/3
+ , republish/3
+ ]).
+
+-spec console(map(), map(), map()) -> any().
+console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) ->
+ ?ULOG("[rule output] ~s~n"
+ "\tOutput Data: ~p~n"
+ "\tEnvs: ~p~n", [RuleId, Selected, Envs]).
+
+republish(_Selected, #{topic := Topic, headers := #{republish_by := RuleId},
+ metadata := #{rule_id := RuleId}}, _Args) ->
+ ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic});
+
+%% republish a PUBLISH message
+republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}},
+ #{preprocessed_tmpl := #{
+ qos := QoSTks,
+ retain := RetainTks,
+ topic := TopicTks,
+ payload := PayloadTks}}) ->
+ Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
+ Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
+ QoS = replace_simple_var(QoSTks, Selected),
+ Retain = replace_simple_var(RetainTks, Selected),
+ ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}),
+ safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
+
+%% in case this is a "$events/" event
+republish(Selected, #{metadata := #{rule_id := RuleId}},
+ #{preprocessed_tmpl := #{
+ qos := QoSTks,
+ retain := RetainTks,
+ topic := TopicTks,
+ payload := PayloadTks}}) ->
+ Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
+ Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
+ QoS = replace_simple_var(QoSTks, Selected),
+ Retain = replace_simple_var(RetainTks, Selected),
+ ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}),
+ safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload).
+
+safe_publish(RuleId, Topic, QoS, Flags, Payload) ->
+ Msg = #message{
+ id = emqx_guid:gen(),
+ qos = QoS,
+ from = RuleId,
+ flags = Flags,
+ headers = #{republish_by => RuleId},
+ topic = Topic,
+ payload = Payload,
+ timestamp = erlang:system_time(millisecond)
+ },
+ _ = emqx_broker:safe_publish(Msg),
+ emqx_metrics:inc_msg(Msg).
+
+replace_simple_var(Tokens, Data) when is_list(Tokens) ->
+ [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
+ Var;
+replace_simple_var(Val, _Data) ->
+ Val.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl
index a0b8b48d5..370a72933 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl
@@ -27,7 +27,7 @@
%% Rule Management
-export([ get_rules/0
- , get_rules_for/1
+ , get_rules_for_topic/1
, get_rules_with_same_event/1
, get_rules_ordered_by_ts/0
, get_rule/1
@@ -37,39 +37,6 @@
, remove_rules/1
]).
-%% Action Management
--export([ add_action/1
- , add_actions/1
- , get_actions/0
- , find_action/1
- , remove_action/1
- , remove_actions/1
- , remove_actions_of/1
- , add_action_instance_params/1
- , get_action_instance_params/1
- , remove_action_instance_params/1
- ]).
-
-%% Resource Management
--export([ get_resources/0
- , add_resource/1
- , add_resource_params/1
- , find_resource/1
- , find_resource_params/1
- , get_resources_by_type/1
- , remove_resource/1
- , remove_resource_params/1
- ]).
-
-%% Resource Types
--export([ get_resource_types/0
- , find_resource_type/1
- , find_rules_depends_on_resource/1
- , find_enabled_rules_depends_on_resource/1
- , register_resource_types/1
- , unregister_resource_types_of/1
- ]).
-
-export([ load_hooks_for_rule/1
, unload_hooks_for_rule/1
]).
@@ -110,53 +77,15 @@ mnesia(boot) ->
{rlog_shard, ?RULE_ENGINE_SHARD},
{disc_copies, [node()]},
{record_name, rule},
- {index, [#rule.for]},
{attributes, record_info(fields, rule)},
- {storage_properties, StoreProps}]),
- %% Rule action table
- ok = ekka_mnesia:create_table(?ACTION_TAB, [
- {rlog_shard, ?RULE_ENGINE_SHARD},
- {ram_copies, [node()]},
- {record_name, action},
- {index, [#action.for, #action.app]},
- {attributes, record_info(fields, action)},
- {storage_properties, StoreProps}]),
- %% Resource table
- ok = ekka_mnesia:create_table(?RES_TAB, [
- {rlog_shard, ?RULE_ENGINE_SHARD},
- {disc_copies, [node()]},
- {record_name, resource},
- {index, [#resource.type]},
- {attributes, record_info(fields, resource)},
- {storage_properties, StoreProps}]),
- %% Resource type table
- ok = ekka_mnesia:create_table(?RES_TYPE_TAB, [
- {rlog_shard, ?RULE_ENGINE_SHARD},
- {ram_copies, [node()]},
- {record_name, resource_type},
- {index, [#resource_type.provider]},
- {attributes, record_info(fields, resource_type)},
{storage_properties, StoreProps}]);
mnesia(copy) ->
%% Copy rule table
- ok = ekka_mnesia:copy_table(?RULE_TAB, disc_copies),
- %% Copy rule action table
- ok = ekka_mnesia:copy_table(?ACTION_TAB, ram_copies),
- %% Copy resource table
- ok = ekka_mnesia:copy_table(?RES_TAB, disc_copies),
- %% Copy resource type table
- ok = ekka_mnesia:copy_table(?RES_TYPE_TAB, ram_copies).
+ ok = ekka_mnesia:copy_table(?RULE_TAB, disc_copies).
dump() ->
- ?ULOG("Rules: ~p~n"
- "ActionInstParams: ~p~n"
- "Resources: ~p~n"
- "ResourceParams: ~p~n",
- [ets:tab2list(?RULE_TAB),
- ets:tab2list(?ACTION_INST_PARAMS_TAB),
- ets:tab2list(?RES_TAB),
- ets:tab2list(?RES_PARAMS_TAB)]).
+ ?ULOG("Rules: ~p~n", [ets:tab2list(?RULE_TAB)]).
%%------------------------------------------------------------------------------
%% Start the registry
@@ -182,16 +111,16 @@ get_rules_ordered_by_ts() ->
{atomic, List} = ekka_mnesia:transaction(?RULE_ENGINE_SHARD, F),
List.
--spec(get_rules_for(Topic :: binary()) -> list(emqx_rule_engine:rule())).
-get_rules_for(Topic) ->
- [Rule || Rule = #rule{for = For} <- get_rules(),
- emqx_plugin_libs_rule:can_topic_match_oneof(Topic, For)].
+-spec(get_rules_for_topic(Topic :: binary()) -> list(emqx_rule_engine:rule())).
+get_rules_for_topic(Topic) ->
+ [Rule || Rule = #rule{info = #{from := From}} <- get_rules(),
+ emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)].
-spec(get_rules_with_same_event(Topic :: binary()) -> list(emqx_rule_engine:rule())).
get_rules_with_same_event(Topic) ->
EventName = emqx_rule_events:event_name(Topic),
- [Rule || Rule = #rule{for = For} <- get_rules(),
- lists:any(fun(T) -> is_of_event_name(EventName, T) end, For)].
+ [Rule || Rule = #rule{info = #{from := From}} <- get_rules(),
+ lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)].
is_of_event_name(EventName, Topic) ->
EventName =:= emqx_rule_events:event_name(Topic).
@@ -223,7 +152,7 @@ remove_rules(Rules) ->
insert_rules([]) -> ok;
insert_rules(Rules) ->
- _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]),
+ _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]),
[mnesia:write(?RULE_TAB, Rule, write) ||Rule <- Rules].
%% @private
@@ -235,7 +164,7 @@ delete_rules(Rules = [R|_]) when is_binary(R) ->
{ok, Rule} -> [Rule|Acc];
not_found -> Acc
end
- end, [], Rules),
+ end, [], Rules),
delete_rules_unload_hooks(RuleRecs);
delete_rules(Rules = [Rule|_]) when is_record(Rule, rule) ->
delete_rules_unload_hooks(Rules).
@@ -245,209 +174,20 @@ delete_rules_unload_hooks(Rules) ->
[mnesia:delete_object(?RULE_TAB, Rule, write) ||Rule <- Rules].
load_hooks_for_rule(Rules) ->
- lists:foreach(fun(#rule{for = Topics}) ->
- lists:foreach(fun emqx_rule_events:load/1, Topics)
- end, Rules).
+ lists:foreach(fun(#rule{info = #{from := Topics}}) ->
+ lists:foreach(fun emqx_rule_events:load/1, Topics)
+ end, Rules).
unload_hooks_for_rule(Rules) ->
- lists:foreach(fun(#rule{id = Id, for = Topics}) ->
+ lists:foreach(fun(#rule{id = Id, info = #{from := Topics}}) ->
lists:foreach(fun(Topic) ->
case get_rules_with_same_event(Topic) of
- [#rule{id = Id}] -> %% we are now deleting the last rule
+ [#rule{id = Id0}] when Id0 == Id -> %% we are now deleting the last rule
emqx_rule_events:unload(Topic);
_ -> ok
end
- end, Topics)
- end, Rules).
-
-%%------------------------------------------------------------------------------
-%% Action Management
-%%------------------------------------------------------------------------------
-
-%% @doc Get all actions.
--spec(get_actions() -> list(emqx_rule_engine:action())).
-get_actions() ->
- get_all_records(?ACTION_TAB).
-
-%% @doc Find an action by name.
--spec(find_action(Name :: action_name()) -> {ok, emqx_rule_engine:action()} | not_found).
-find_action(Name) ->
- case mnesia:dirty_read(?ACTION_TAB, Name) of
- [Action] -> {ok, Action};
- [] -> not_found
- end.
-
-%% @doc Add an action.
--spec(add_action(emqx_rule_engine:action()) -> ok).
-add_action(Action) when is_record(Action, action) ->
- trans(fun insert_action/1, [Action]).
-
-%% @doc Add actions.
--spec(add_actions(list(emqx_rule_engine:action())) -> ok).
-add_actions(Actions) when is_list(Actions) ->
- trans(fun lists:foreach/2, [fun insert_action/1, Actions]).
-
-%% @doc Remove an action.
--spec(remove_action(emqx_rule_engine:action() | atom()) -> ok).
-remove_action(Action) when is_record(Action, action) ->
- trans(fun delete_action/1, [Action]);
-
-remove_action(Name) ->
- trans(fun mnesia:delete/1, [{?ACTION_TAB, Name}]).
-
-%% @doc Remove actions.
--spec(remove_actions(list(emqx_rule_engine:action())) -> ok).
-remove_actions(Actions) ->
- trans(fun lists:foreach/2, [fun delete_action/1, Actions]).
-
-%% @doc Remove actions of the App.
--spec(remove_actions_of(App :: atom()) -> ok).
-remove_actions_of(App) ->
- trans(fun() ->
- lists:foreach(fun delete_action/1, mnesia:index_read(?ACTION_TAB, App, #action.app))
- end).
-
-%% @private
-insert_action(Action) ->
- mnesia:write(?ACTION_TAB, Action, write).
-
-%% @private
-delete_action(Action) when is_record(Action, action) ->
- mnesia:delete_object(?ACTION_TAB, Action, write);
-delete_action(Name) when is_atom(Name) ->
- mnesia:delete(?ACTION_TAB, Name, write).
-
-%% @doc Add an action instance params.
--spec(add_action_instance_params(emqx_rule_engine:action_instance_params()) -> ok).
-add_action_instance_params(ActionInstParams) when is_record(ActionInstParams, action_instance_params) ->
- ets:insert(?ACTION_INST_PARAMS_TAB, ActionInstParams),
- ok.
-
--spec(get_action_instance_params(action_instance_id()) -> {ok, emqx_rule_engine:action_instance_params()} | not_found).
-get_action_instance_params(ActionInstId) ->
- case ets:lookup(?ACTION_INST_PARAMS_TAB, ActionInstId) of
- [ActionInstParams] -> {ok, ActionInstParams};
- [] -> not_found
- end.
-
-%% @doc Delete an action instance params.
--spec(remove_action_instance_params(action_instance_id()) -> ok).
-remove_action_instance_params(ActionInstId) ->
- ets:delete(?ACTION_INST_PARAMS_TAB, ActionInstId),
- ok.
-
-%%------------------------------------------------------------------------------
-%% Resource Management
-%%------------------------------------------------------------------------------
-
--spec(get_resources() -> list(emqx_rule_engine:resource())).
-get_resources() ->
- get_all_records(?RES_TAB).
-
--spec(add_resource(emqx_rule_engine:resource()) -> ok).
-add_resource(Resource) when is_record(Resource, resource) ->
- trans(fun insert_resource/1, [Resource]).
-
--spec(add_resource_params(emqx_rule_engine:resource_params()) -> ok).
-add_resource_params(ResParams) when is_record(ResParams, resource_params) ->
- ets:insert(?RES_PARAMS_TAB, ResParams),
- ok.
-
--spec(find_resource(Id :: resource_id()) -> {ok, emqx_rule_engine:resource()} | not_found).
-find_resource(Id) ->
- case mnesia:dirty_read(?RES_TAB, Id) of
- [Res] -> {ok, Res};
- [] -> not_found
- end.
-
--spec(find_resource_params(Id :: resource_id())
- -> {ok, emqx_rule_engine:resource_params()} | not_found).
-find_resource_params(Id) ->
- case ets:lookup(?RES_PARAMS_TAB, Id) of
- [ResParams] -> {ok, ResParams};
- [] -> not_found
- end.
-
--spec(remove_resource(emqx_rule_engine:resource() | emqx_rule_engine:resource_id()) -> ok | {error, term()}).
-remove_resource(Resource) when is_record(Resource, resource) ->
- trans(fun delete_resource/1, [Resource#resource.id]);
-
-remove_resource(ResId) when is_binary(ResId) ->
- trans(fun delete_resource/1, [ResId]).
-
--spec(remove_resource_params(emqx_rule_engine:resource_id()) -> ok).
-remove_resource_params(ResId) ->
- ets:delete(?RES_PARAMS_TAB, ResId),
- ok.
-
-%% @private
-delete_resource(ResId) ->
- case find_enabled_rules_depends_on_resource(ResId) of
- [] -> mnesia:delete(?RES_TAB, ResId, write);
- Rules ->
- {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}}
- end.
-
-%% @private
-insert_resource(Resource) ->
- mnesia:write(?RES_TAB, Resource, write).
-
-find_enabled_rules_depends_on_resource(ResId) ->
- [R || #rule{enabled = true} = R <- find_rules_depends_on_resource(ResId)].
-
-find_rules_depends_on_resource(ResId) ->
- lists:foldl(fun(#rule{actions = Actions} = R, Rules) ->
- case search_action_despends_on_resource(ResId, Actions) of
- false -> Rules;
- {value, _} -> [R | Rules]
- end
- end, [], get_rules()).
-
-search_action_despends_on_resource(ResId, Actions) ->
- lists:search(fun
- (#action_instance{args = #{<<"$resource">> := ResId0}}) ->
- ResId0 =:= ResId;
- (_) ->
- false
- end, Actions).
-
-%%------------------------------------------------------------------------------
-%% Resource Type Management
-%%------------------------------------------------------------------------------
-
--spec(get_resource_types() -> list(emqx_rule_engine:resource_type())).
-get_resource_types() ->
- get_all_records(?RES_TYPE_TAB).
-
--spec(find_resource_type(Name :: resource_type_name()) -> {ok, emqx_rule_engine:resource_type()} | not_found).
-find_resource_type(Name) ->
- case mnesia:dirty_read(?RES_TYPE_TAB, Name) of
- [ResType] -> {ok, ResType};
- [] -> not_found
- end.
-
--spec(get_resources_by_type(Type :: resource_type_name()) -> list(emqx_rule_engine:resource())).
-get_resources_by_type(Type) ->
- mnesia:dirty_index_read(?RES_TAB, Type, #resource.type).
-
--spec(register_resource_types(list(emqx_rule_engine:resource_type())) -> ok).
-register_resource_types(Types) ->
- trans(fun lists:foreach/2, [fun insert_resource_type/1, Types]).
-
-%% @doc Unregister resource types of the App.
--spec(unregister_resource_types_of(App :: atom()) -> ok).
-unregister_resource_types_of(App) ->
- trans(fun() ->
- lists:foreach(fun delete_resource_type/1, mnesia:index_read(?RES_TYPE_TAB, App, #resource_type.provider))
- end).
-
-%% @private
-insert_resource_type(Type) ->
- mnesia:write(?RES_TYPE_TAB, Type, write).
-
-%% @private
-delete_resource_type(Type) ->
- mnesia:delete_object(?RES_TYPE_TAB, Type, write).
+ end, Topics)
+ end, Rules).
%%------------------------------------------------------------------------------
%% gen_server callbacks
@@ -467,15 +207,15 @@ handle_call({remove_rules, Rules}, _From, State) ->
{reply, ok, State};
handle_call(Req, _From, State) ->
- ?LOG(error, "[RuleRegistry]: unexpected call - ~p", [Req]),
+ ?SLOG(error, #{msg => "unexpected_call", request => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
- ?LOG(error, "[RuleRegistry]: unexpected cast ~p", [Msg]),
+ ?SLOG(error, #{msg => "unexpected_cast", request => Msg}),
{noreply, State}.
handle_info(Info, State) ->
- ?LOG(error, "[RuleRegistry]: unexpected info ~p", [Info]),
+ ?SLOG(error, #{msg => "unexpected_info", request => Info}),
{noreply, State}.
terminate(_Reason, _State) ->
@@ -500,7 +240,6 @@ get_all_records(Tab) ->
end),
Ret.
-trans(Fun) -> trans(Fun, []).
trans(Fun, Args) ->
case ekka_mnesia:transaction(?RULE_ENGINE_SHARD, Fun, Args) of
{atomic, Result} -> Result;
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index f9e210ab3..aafc6cddc 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -33,9 +33,9 @@
-compile({no_auto_import,[alias/1]}).
--type(input() :: map()).
--type(alias() :: atom()).
--type(collection() :: {alias(), [term()]}).
+-type input() :: map().
+-type alias() :: atom().
+-type collection() :: {alias(), [term()]}.
-define(ephemeral_alias(TYPE, NAME),
iolist_to_binary(io_lib:format("_v_~s_~p_~p", [TYPE, NAME, erlang:system_time()]))).
@@ -48,27 +48,31 @@
-spec(apply_rules(list(emqx_rule_engine:rule()), input()) -> ok).
apply_rules([], _Input) ->
ok;
-apply_rules([#rule{enabled = false}|More], Input) ->
+apply_rules([#rule{info = #{enabled := false}}|More], Input) ->
apply_rules(More, Input);
apply_rules([Rule = #rule{id = RuleID}|More], Input) ->
try apply_rule_discard_result(Rule, Input)
catch
%% ignore the errors if select or match failed
_:{select_and_transform_error, Error} ->
- ?LOG(warning, "SELECT clause exception for ~s failed: ~p",
- [RuleID, Error]);
+ ?SLOG(warning, #{msg => "SELECT_clause_exception",
+ rule_id => RuleID, reason => Error});
_:{match_conditions_error, Error} ->
- ?LOG(warning, "WHERE clause exception for ~s failed: ~p",
- [RuleID, Error]);
+ ?SLOG(warning, #{msg => "WHERE_clause_exception",
+ rule_id => RuleID, reason => Error});
_:{select_and_collect_error, Error} ->
- ?LOG(warning, "FOREACH clause exception for ~s failed: ~p",
- [RuleID, Error]);
+ ?SLOG(warning, #{msg => "FOREACH_clause_exception",
+ rule_id => RuleID, reason => Error});
_:{match_incase_error, Error} ->
- ?LOG(warning, "INCASE clause exception for ~s failed: ~p",
- [RuleID, Error]);
- _:Error:StkTrace ->
- ?LOG(error, "Apply rule ~s failed: ~p. Stacktrace:~n~p",
- [RuleID, Error, StkTrace])
+ ?SLOG(warning, #{msg => "INCASE_clause_exception",
+ rule_id => RuleID, reason => Error});
+ Class:Error:StkTrace ->
+ ?SLOG(error, #{msg => "apply_rule_failed",
+ rule_id => RuleID,
+ exception => Class,
+ reason => Error,
+ stacktrace => StkTrace
+ })
end,
apply_rules(More, Input).
@@ -80,14 +84,14 @@ apply_rule(Rule = #rule{id = RuleID}, Input) ->
clear_rule_payload(),
do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID})).
-do_apply_rule(#rule{id = RuleId,
- is_foreach = true,
- fields = Fields,
- doeach = DoEach,
- incase = InCase,
- conditions = Conditions,
- on_action_failed = OnFailed,
- actions = Actions}, Input) ->
+do_apply_rule(#rule{id = RuleId, info = #{
+ is_foreach := true,
+ fields := Fields,
+ doeach := DoEach,
+ incase := InCase,
+ conditions := Conditions,
+ outputs := Outputs
+ }}, Input) ->
{Selected, Collection} = ?RAISE(select_and_collect(Fields, Input),
{select_and_collect_error, {_EXCLASS_,_EXCPTION_,_ST_}}),
ColumnsAndSelected = maps:merge(Input, Selected),
@@ -96,24 +100,24 @@ do_apply_rule(#rule{id = RuleId,
true ->
ok = emqx_rule_metrics:inc(RuleId, 'rules.matched'),
Collection2 = filter_collection(Input, InCase, DoEach, Collection),
- {ok, [take_actions(Actions, Coll, Input, OnFailed) || Coll <- Collection2]};
+ {ok, [handle_output_list(Outputs, Coll, Input) || Coll <- Collection2]};
false ->
{error, nomatch}
end;
-do_apply_rule(#rule{id = RuleId,
- is_foreach = false,
- fields = Fields,
- conditions = Conditions,
- on_action_failed = OnFailed,
- actions = Actions}, Input) ->
+do_apply_rule(#rule{id = RuleId, info = #{
+ is_foreach := false,
+ fields := Fields,
+ conditions := Conditions,
+ outputs := Outputs
+ }}, Input) ->
Selected = ?RAISE(select_and_transform(Fields, Input),
{select_and_transform_error, {_EXCLASS_,_EXCPTION_,_ST_}}),
case ?RAISE(match_conditions(Conditions, maps:merge(Input, Selected)),
{match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of
true ->
ok = emqx_rule_metrics:inc(RuleId, 'rules.matched'),
- {ok, take_actions(Actions, Selected, Input, OnFailed)};
+ {ok, handle_output_list(Outputs, Selected, Input)};
false ->
{error, nomatch}
end.
@@ -166,7 +170,6 @@ select_and_collect([Field|More], Input, {Output, LastKV}) ->
{nested_put(Key, Val, Output), LastKV}).
%% Filter each item got from FOREACH
--dialyzer({nowarn_function, filter_collection/4}).
filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) ->
lists:filtermap(
fun(Item) ->
@@ -198,8 +201,6 @@ match_conditions({'fun', {_, Name}, Args}, Data) ->
apply_func(Name, [eval(Arg, Data) || Arg <- Args], Data);
match_conditions({Op, L, R}, Data) when ?is_comp(Op) ->
compare(Op, eval(L, Data), eval(R, Data));
-%%match_conditions({'like', Var, Pattern}, Data) ->
-%% match_like(eval(Var, Data), Pattern);
match_conditions({}, _Data) ->
true.
@@ -229,81 +230,43 @@ number(Bin) ->
catch error:badarg -> binary_to_float(Bin)
end.
-%% Step3 -> Take actions
-take_actions(Actions, Selected, Envs, OnFailed) ->
- [take_action(ActInst, Selected, Envs, OnFailed, ?ActionMaxRetry)
- || ActInst <- Actions].
+handle_output_list(Outputs, Selected, Envs) ->
+ [handle_output(Out, Selected, Envs) || Out <- Outputs].
-take_action(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks} = ActInst,
- Selected, Envs, OnFailed, RetryN) when RetryN >= 0 ->
+handle_output(OutId, Selected, Envs) ->
try
- {ok, #action_instance_params{apply = Apply}}
- = emqx_rule_registry:get_action_instance_params(Id),
- emqx_rule_metrics:inc_actions_taken(Id),
- apply_action_func(Selected, Envs, Apply, ActName)
- of
- {badact, Reason} ->
- handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, Reason);
- Result -> Result
+ do_handle_output(OutId, Selected, Envs)
catch
- error:{badfun, _Func}:_ST ->
- %?LOG(warning, "Action ~p maybe outdated, refresh it and try again."
- % "Func: ~p~nST:~0p", [Id, Func, ST]),
- _ = trans_action_on(Id, fun() ->
- emqx_rule_engine:refresh_actions([ActInst])
- end, 5000),
- emqx_rule_metrics:inc_actions_retry(Id),
- take_action(ActInst, Selected, Envs, OnFailed, RetryN-1);
- Error:Reason:Stack ->
- emqx_rule_metrics:inc_actions_exception(Id),
- handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, {Error, Reason, Stack})
- end;
-
-take_action(#action_instance{id = Id, fallbacks = Fallbacks}, Selected, Envs, OnFailed, _RetryN) ->
- emqx_rule_metrics:inc_actions_error(Id),
- handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, {max_try_reached, ?ActionMaxRetry}).
-
-apply_action_func(Data, Envs, #{mod := Mod, bindings := Bindings}, Name) ->
- %% TODO: Build the Func Name when creating the action
- Func = cbk_on_action_triggered(Name),
- Mod:Func(Data, Envs#{'__bindings__' => Bindings});
-apply_action_func(Data, Envs, Func, _Name) when is_function(Func) ->
- erlang:apply(Func, [Data, Envs]).
-
-cbk_on_action_triggered(Name) ->
- list_to_atom("on_action_" ++ atom_to_list(Name)).
-
-trans_action_on(Id, Callback, Timeout) ->
- case emqx_rule_locker:lock(Id) of
- true -> try Callback() after emqx_rule_locker:unlock(Id) end;
- _ ->
- wait_action_on(Id, Timeout div 10)
+ Err:Reason:ST ->
+ ?SLOG(error, #{msg => "output_failed",
+ output => OutId,
+ exception => Err,
+ reason => Reason,
+ stacktrace => ST
+ })
end.
-wait_action_on(_, 0) ->
- {error, timeout};
-wait_action_on(Id, RetryN) ->
- timer:sleep(10),
- case emqx_rule_registry:get_action_instance_params(Id) of
- not_found ->
- {error, not_found};
- {ok, #action_instance_params{apply = Apply}} ->
- case catch apply_action_func(baddata, #{}, Apply, tryit) of
- {'EXIT', {{badfun, _}, _}} ->
- wait_action_on(Id, RetryN-1);
- _ ->
- ok
- end
+do_handle_output(#{type := bridge, target := ChannelId}, Selected, _Envs) ->
+ ?SLOG(debug, #{msg => "output to bridge", channel_id => ChannelId}),
+ emqx_bridge:send_message(ChannelId, Selected);
+do_handle_output(#{type := func, target := Func} = Out, Selected, Envs) ->
+ erlang:apply(Func, [Selected, Envs, maps:get(args, Out, #{})]);
+do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs)
+ when is_atom(Output) ->
+ handle_builtin_output(Output, Selected, Envs, maps:get(args, Out, #{}));
+do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs)
+ when is_binary(Output) ->
+ try binary_to_existing_atom(Output) of
+ Func -> handle_builtin_output(Func, Selected, Envs, maps:get(args, Out, #{}))
+ catch
+ error:badarg -> error(not_found)
end.
-handle_action_failure(continue, Id, Fallbacks, Selected, Envs, Reason) ->
- ?LOG(error, "Take action ~p failed, continue next action, reason: ~0p", [Id, Reason]),
- _ = take_actions(Fallbacks, Selected, Envs, continue),
- failed;
-handle_action_failure(stop, Id, Fallbacks, Selected, Envs, Reason) ->
- ?LOG(error, "Take action ~p failed, skip all actions, reason: ~0p", [Id, Reason]),
- _ = take_actions(Fallbacks, Selected, Envs, continue),
- error({take_action_failed, {Id, Reason}}).
+handle_builtin_output(Func, Selected, Envs, Args) ->
+ case erlang:function_exported(emqx_rule_outputs, Func, 3) of
+ true -> erlang:apply(emqx_rule_outputs, Func, [Selected, Envs, Args]);
+ false -> error(not_found)
+ end.
eval({path, [{key, <<"payload">>} | Path]}, #{payload := Payload}) ->
nested_get({path, Path}, may_decode_payload(Payload));
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl
index 9a8ce55ea..02c5a02e9 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl
@@ -18,7 +18,7 @@
-include("rule_engine.hrl").
--export([parse_select/1]).
+-export([parse/1]).
-export([ select_fields/1
, select_is_foreach/1
@@ -36,26 +36,21 @@
-opaque(select() :: #select{}).
--type(const() :: {const, number()|binary()}).
+-type const() :: {const, number()|binary()}.
--type(variable() :: binary() | list(binary())).
+-type variable() :: binary() | list(binary()).
--type(alias() :: binary() | list(binary())).
+-type alias() :: binary() | list(binary()).
--type(field() :: const() | variable()
+-type field() :: const() | variable()
| {as, field(), alias()}
- | {'fun', atom(), list(field())}).
+ | {'fun', atom(), list(field())}.
-export_type([select/0]).
-%% Dialyzer gives up on the generated code.
-%% probably due to stack depth, or inlines.
--dialyzer({nowarn_function, [parse_select/1]}).
-
%% Parse one select statement.
--spec(parse_select(string() | binary())
- -> {ok, select()} | {parse_error, term()} | {lex_error, term()}).
-parse_select(Sql) ->
+-spec(parse(string() | binary()) -> {ok, select()} | {error, term()}).
+parse(Sql) ->
try case rulesql:parsetree(Sql) of
{ok, {select, Clauses}} ->
{ok, #select{
@@ -75,11 +70,11 @@ parse_select(Sql) ->
from = get_value(from, Clauses),
where = get_value(where, Clauses)
}};
- Error -> Error
+ Error -> {error, Error}
end
catch
_Error:Reason:StackTrace ->
- {parse_error, {Reason, StackTrace}}
+ {error, {Reason, StackTrace}}
end.
-spec(select_fields(select()) -> list(field())).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index 2f1edbeb2..ec263b35a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -18,22 +18,14 @@
-include_lib("emqx/include/logger.hrl").
-export([ test/1
+ , echo_action/2
+ , get_selected_data/3
]).
-%% Dialyzer gives up on the generated code.
-%% probably due to stack depth, or inlines.
--dialyzer({nowarn_function, [test/1,
- test_rule/4,
- flatten/1,
- sql_test_action/0,
- fill_default_values/2,
- envs_examp/1
- ]}).
-
--spec(test(#{}) -> {ok, map() | list()} | {error, term()}).
-test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) ->
- {ok, Select} = emqx_rule_sqlparser:parse_select(Sql),
- InTopic = maps:get(<<"topic">>, Context, <<>>),
+-spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, nomatch}.
+test(#{sql := Sql, context := Context}) ->
+ {ok, Select} = emqx_rule_sqlparser:parse(Sql),
+ InTopic = maps:get(topic, Context, <<>>),
EventTopics = emqx_rule_sqlparser:select_from(Select),
case lists:all(fun is_publish_topic/1, EventTopics) of
true ->
@@ -48,40 +40,36 @@ test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) ->
end.
test_rule(Sql, Select, Context, EventTopics) ->
- RuleId = iolist_to_binary(["test_rule", emqx_misc:gen_id()]),
- ActInstId = iolist_to_binary(["test_action", emqx_misc:gen_id()]),
+ RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]),
ok = emqx_rule_metrics:create_rule_metrics(RuleId),
- ok = emqx_rule_metrics:create_metrics(ActInstId),
Rule = #rule{
id = RuleId,
- rawsql = Sql,
- for = EventTopics,
- is_foreach = emqx_rule_sqlparser:select_is_foreach(Select),
- fields = emqx_rule_sqlparser:select_fields(Select),
- doeach = emqx_rule_sqlparser:select_doeach(Select),
- incase = emqx_rule_sqlparser:select_incase(Select),
- conditions = emqx_rule_sqlparser:select_where(Select),
- actions = [#action_instance{
- id = ActInstId,
- name = test_rule_sql}]
+ info = #{
+ sql => Sql,
+ from => EventTopics,
+ outputs => [#{type => func, target => fun ?MODULE:get_selected_data/3, args => #{}}],
+ enabled => true,
+ is_foreach => emqx_rule_sqlparser:select_is_foreach(Select),
+ fields => emqx_rule_sqlparser:select_fields(Select),
+ doeach => emqx_rule_sqlparser:select_doeach(Select),
+ incase => emqx_rule_sqlparser:select_incase(Select),
+ conditions => emqx_rule_sqlparser:select_where(Select)
+ },
+ created_at = erlang:system_time(millisecond)
},
FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)),
try
- ok = emqx_rule_registry:add_action_instance_params(
- #action_instance_params{id = ActInstId,
- params = #{},
- apply = sql_test_action()}),
- R = emqx_rule_runtime:apply_rule(Rule, FullContext),
- emqx_rule_metrics:clear_rule_metrics(RuleId),
- emqx_rule_metrics:clear_metrics(ActInstId),
- R
+ emqx_rule_runtime:apply_rule(Rule, FullContext)
of
{ok, Data} -> {ok, flatten(Data)};
{error, nomatch} -> {error, nomatch}
after
- ok = emqx_rule_registry:remove_action_instance_params(ActInstId)
+ emqx_rule_metrics:clear_rule_metrics(RuleId)
end.
+get_selected_data(Selected, _Envs, _Args) ->
+ Selected.
+
is_publish_topic(<<"$events/", _/binary>>) -> false;
is_publish_topic(_Topic) -> true.
@@ -90,10 +78,9 @@ flatten([D1]) -> D1;
flatten([D1 | L]) when is_list(D1) ->
D1 ++ flatten(L).
-sql_test_action() ->
- fun(Data, _Envs) ->
- ?LOG(info, "Testing Rule SQL OK"), Data
- end.
+echo_action(Data, Envs) ->
+ ?SLOG(debug, #{msg => "testing_rule_sql_ok", data => Data, envs => Envs}),
+ Data.
fill_default_values(Event, Context) ->
maps:merge(envs_examp(Event), Context).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl
deleted file mode 100644
index 8f39d2d1c..000000000
--- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl
+++ /dev/null
@@ -1,195 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_validator).
-
--include("rule_engine.hrl").
-
--export([ validate_params/2
- , validate_spec/1
- ]).
-
--type name() :: atom().
-
--type spec() :: #{
- type := data_type(),
- required => boolean(),
- default => term(),
- enum => list(term()),
- schema => spec()
-}.
-
--type data_type() :: string | password | number | boolean
- | object | array | file | cfgselect.
-
--type params_spec() :: #{name() => spec()} | any.
--type params() :: #{binary() => term()}.
-
--define(DATA_TYPES,
- [ string
- , password %% TODO: [5.0] remove this, use string instead
- , number
- , boolean
- , object
- , array
- , file
- , cfgselect %% TODO: [5.0] refactor this
- ]).
-
-%%------------------------------------------------------------------------------
-%% APIs
-%%------------------------------------------------------------------------------
-
-%% Validate the params according to the spec.
-%% Some keys will be added into the result if they have default values in spec.
-%% Note that this function throws exception in case of validation failure.
--spec(validate_params(params(), params_spec()) -> params()).
-validate_params(Params, any) -> Params;
-validate_params(Params, ParamsSepc) ->
- maps:fold(fun(Name, Spec, Params0) ->
- IsRequired = maps:get(required, Spec, false),
- BinName = bin(Name),
- find_field(Name, Params,
- fun (not_found) when IsRequired =:= true ->
- throw({required_field_missing, BinName});
- (not_found) when IsRequired =:= false ->
- case maps:find(default, Spec) of
- {ok, Default} -> Params0#{BinName => Default};
- error -> Params0
- end;
- (Val) ->
- Params0#{BinName => validate_value(Val, Spec)}
- end)
- end, Params, ParamsSepc).
-
--spec(validate_spec(params_spec()) -> ok).
-validate_spec(any) -> ok;
-validate_spec(ParamsSepc) ->
- map_foreach(fun do_validate_spec/2, ParamsSepc).
-
-%%------------------------------------------------------------------------------
-%% Internal Functions
-%%------------------------------------------------------------------------------
-
-validate_value(Val, #{enum := Enum}) ->
- validate_enum(Val, Enum);
-validate_value(Val, #{type := object} = Spec) ->
- validate_params(Val, maps:get(schema, Spec, any));
-validate_value(Val, #{type := Type} = Spec) ->
- validate_type(Val, Type, Spec).
-
-validate_type(Val, file, _Spec) ->
- validate_file(Val);
-validate_type(Val, String, Spec) when String =:= string;
- String =:= password ->
- validate_string(Val, reg_exp(maps:get(format, Spec, any)));
-validate_type(Val, number, Spec) ->
- validate_number(Val, maps:get(range, Spec, any));
-validate_type(Val, boolean, _Spec) ->
- validate_boolean(Val);
-validate_type(Val, array, Spec) ->
- ItemsSpec = maps:get(items, Spec),
- [validate_value(V, ItemsSpec) || V <- Val];
-validate_type(Val, cfgselect, _Spec) ->
- %% TODO: [5.0] refactor this.
- Val.
-
-validate_enum(Val, Enum) ->
- case lists:member(Val, Enum) of
- true -> Val;
- false -> throw({invalid_data_type, {enum, {Val, Enum}}})
- end.
-
-validate_string(Val, RegExp) ->
- try re:run(Val, RegExp) of
- nomatch -> throw({invalid_data_type, {string, Val}});
- _Match -> Val
- catch
- _:_ -> throw({invalid_data_type, {string, Val}})
- end.
-
-validate_number(Val, any) when is_integer(Val); is_float(Val) ->
- Val;
-validate_number(Val, _Range = [Min, Max])
- when (is_integer(Val) orelse is_float(Val)),
- (Val >= Min andalso Val =< Max) ->
- Val;
-validate_number(Val, Range) ->
- throw({invalid_data_type, {number, {Val, Range}}}).
-
-validate_boolean(true) -> true;
-validate_boolean(<<"true">>) -> true;
-validate_boolean(false) -> false;
-validate_boolean(<<"false">>) -> false;
-validate_boolean(Val) -> throw({invalid_data_type, {boolean, Val}}).
-
-validate_file(Val) when is_map(Val) -> Val;
-validate_file(Val) when is_list(Val) -> Val;
-validate_file(Val) when is_binary(Val) -> Val;
-validate_file(Val) -> throw({invalid_data_type, {file, Val}}).
-
-reg_exp(url) -> "^https?://\\w+(\.\\w+)*(:[0-9]+)?";
-reg_exp(topic) -> "^/?(\\w|\\#|\\+)+(/?(\\w|\\#|\\+))*/?$";
-reg_exp(resource_type) -> "[a-zA-Z0-9_:-]";
-reg_exp(any) -> ".*";
-reg_exp(RegExp) -> RegExp.
-
-do_validate_spec(Name, #{type := object} = Spec) ->
- find_field(schema, Spec,
- fun (not_found) -> throw({required_field_missing, {schema, {in, Name}}});
- (Schema) -> validate_spec(Schema)
- end);
-do_validate_spec(Name, #{type := array} = Spec) ->
- find_field(items, Spec,
- fun (not_found) -> throw({required_field_missing, {items, {in, Name}}});
- (Items) -> do_validate_spec(Name, Items)
- end);
-do_validate_spec(_Name, #{type := Type}) ->
- _ = supported_data_type(Type, ?DATA_TYPES);
-
-do_validate_spec(Name, _Spec) ->
- throw({required_field_missing, {type, {in, Name}}}).
-
-supported_data_type(Type, Supported) ->
- case lists:member(Type, Supported) of
- false -> throw({unsupported_data_types, Type});
- true -> ok
- end.
-
-map_foreach(Fun, Map) ->
- Iterator = maps:iterator(Map),
- map_foreach_loop(Fun, maps:next(Iterator)).
-
-map_foreach_loop(_Fun, none) -> ok;
-map_foreach_loop(Fun, {Key, Value, Iterator}) ->
- _ = Fun(Key, Value),
- map_foreach_loop(Fun, maps:next(Iterator)).
-
-find_field(Field, Spec, Func) ->
- do_find_field([bin(Field), Field], Spec, Func).
-
-do_find_field([], _Spec, Func) ->
- Func(not_found);
-do_find_field([F | Fields], Spec, Func) ->
- case maps:find(F, Spec) of
- {ok, Value} -> Func(Value);
- error ->
- do_find_field(Fields, Spec, Func)
- end.
-
-bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
-bin(Str) when is_list(Str) -> iolist_to_binary(Str);
-bin(Bin) when is_binary(Bin) -> Bin.
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 47eb4faad..45b30223a 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -26,17 +26,15 @@
-include_lib("common_test/include/ct.hrl").
%%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())).
+-define(TMP_RULEID, atom_to_binary(?FUNCTION_NAME)).
all() ->
[ {group, engine}
- , {group, actions}
-%% , {group, api}
- , {group, cli}
+ , {group, api}
, {group, funcs}
, {group, registry}
, {group, runtime}
, {group, events}
- , {group, multi_actions}
, {group, bugs}
].
@@ -45,49 +43,21 @@ suite() ->
groups() ->
[{engine, [sequence],
- [t_register_provider,
- t_unregister_provider,
- t_create_rule,
- t_create_resource
- ]},
- {actions, [],
- [t_inspect_action
- ,t_republish_action
- ]},
-%% TODO: V5 API
-%% {api, [],
-%% [t_crud_rule_api,
-%% t_list_actions_api,
-%% t_show_action_api,
-%% t_crud_resources_api,
-%% t_list_resource_types_api,
-%% t_show_resource_type_api
-%% ]},
- {cli, [],
- [t_rules_cli,
- t_actions_cli,
- t_resources_cli,
- t_resource_types_cli
+ [t_create_rule
]},
+ {api, [],
+ [t_crud_rule_api
+ ]},
{funcs, [],
- [t_topic_func,
- t_kv_store
+ [t_kv_store
]},
{registry, [sequence],
[t_add_get_remove_rule,
t_add_get_remove_rules,
t_create_existing_rule,
- t_update_rule,
- t_disable_rule,
- t_get_rules_for,
- t_get_rules_for_2,
- t_get_rules_with_same_event,
- t_add_get_remove_action,
- t_add_get_remove_actions,
- t_remove_actions_of,
- t_get_resources,
- t_add_get_remove_resource,
- t_resource_types
+ t_get_rules_for_topic,
+ t_get_rules_for_topic_2,
+ t_get_rules_with_same_event
]},
{runtime, [],
[t_match_atom_and_binary,
@@ -97,9 +67,6 @@ groups() ->
t_sqlselect_02,
t_sqlselect_1,
t_sqlselect_2,
- t_sqlselect_2_1,
- t_sqlselect_2_2,
- t_sqlselect_2_3,
t_sqlselect_3,
t_sqlparse_event_1,
t_sqlparse_event_2,
@@ -132,14 +99,6 @@ groups() ->
{bugs, [],
[t_sqlparse_payload_as,
t_sqlparse_nested_get
- ]},
- {multi_actions, [],
- [t_sqlselect_multi_actoins_1,
- t_sqlselect_multi_actoins_1_1,
- t_sqlselect_multi_actoins_2,
- t_sqlselect_multi_actoins_3,
- t_sqlselect_multi_actoins_3_1,
- t_sqlselect_multi_actoins_4
]}
].
@@ -151,7 +110,7 @@ init_per_suite(Config) ->
application:load(emqx_machine),
ok = ekka_mnesia:start(),
ok = emqx_rule_registry:mnesia(boot),
- ok = emqx_ct_helpers:start_apps([emqx_rule_engine], fun set_special_configs/1),
+ ok = emqx_ct_helpers:start_apps([emqx_rule_engine]),
Config.
end_per_suite(_Config) ->
@@ -182,16 +141,8 @@ end_per_group(_Groupname, _Config) ->
%%------------------------------------------------------------------------------
init_per_testcase(t_events, Config) ->
- {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
- ok = emqx_rule_engine:load_providers(),
init_events_counters(),
- ok = emqx_rule_registry:register_resource_types([make_simple_resource_type(simple_resource_type)]),
- ok = emqx_rule_registry:add_action(
- #action{name = 'hook-metrics-action', app = ?APP,
- module = ?MODULE, on_create = hook_metrics_action,
- types=[], params_spec = #{},
- title = #{en => <<"Hook metrics action">>},
- description = #{en => <<"Hook metrics action">>}}),
+ {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
SQL = "SELECT * FROM \"$events/client_connected\", "
"\"$events/client_disconnected\", "
"\"$events/session_subscribed\", "
@@ -201,411 +152,87 @@ init_per_testcase(t_events, Config) ->
"\"$events/message_dropped\", "
"\"t1\"",
{ok, Rule} = emqx_rule_engine:create_rule(
- #{id => <<"rule:t_events">>,
- rawsql => SQL,
- actions => [#{id => <<"action:inspect">>, name => 'inspect', args => #{}},
- #{id => <<"action:hook-metrics-action">>, name => 'hook-metrics-action', args => #{}}],
- description => <<"Debug rule">>}),
+ #{id => <<"rule:t_events">>,
+ sql => SQL,
+ outputs => [
+ #{type => builtin, target => console},
+ #{type => func, target => fun ?MODULE:output_record_triggered_events/3,
+ args => #{}}
+ ],
+ description => <<"to console and record triggered events">>}),
?assertMatch(#rule{id = <<"rule:t_events">>}, Rule),
[{hook_points_rules, Rule} | Config];
-init_per_testcase(Test, Config)
- when Test =:= t_sqlselect_multi_actoins_1
- ;Test =:= t_sqlselect_multi_actoins_1_1
- ;Test =:= t_sqlselect_multi_actoins_2
- ;Test =:= t_sqlselect_multi_actoins_3
- ;Test =:= t_sqlselect_multi_actoins_3_1
- ;Test =:= t_sqlselect_multi_actoins_4
- ->
- emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
- ok = emqx_rule_engine:load_providers(),
- ok = emqx_rule_registry:add_action(
- #action{name = 'crash_action', app = ?APP,
- module = ?MODULE, on_create = crash_action,
- types=[], params_spec = #{},
- title = #{en => <<"Crash Action">>},
- description = #{en => <<"This action will always fail!">>}}),
- ok = emqx_rule_registry:add_action(
- #action{name = 'failure_action', app = ?APP,
- module = ?MODULE, on_create = failure_action,
- types=[], params_spec = #{},
- title = #{en => <<"Crash Action">>},
- description = #{en => <<"This action will always fail!">>}}),
- ok = emqx_rule_registry:add_action(
- #action{name = 'plus_by_one', app = ?APP,
- module = ?MODULE, on_create = plus_by_one_action,
- types=[], params_spec = #{},
- title = #{en => <<"Plus an integer by 1">>}
- }),
- init_plus_by_one_action(),
- SQL = "SELECT * "
- "FROM \"$events/client_connected\" "
- "WHERE username = 'emqx1'",
- {ok, SubClient} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]),
- {ok, _} = emqtt:connect(SubClient),
- {ok, _, _} = emqtt:subscribe(SubClient, <<"t2">>, 0),
- ct:sleep(100),
-
- {ok, ConnClient} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]),
- TriggerConnEvent = fun() ->
- {ok, _} = emqtt:connect(ConnClient)
- end,
- [{subclient, SubClient},
- {connclient, ConnClient},
- {conn_event, TriggerConnEvent},
- {connsql, SQL}
- | Config];
init_per_testcase(_TestCase, Config) ->
emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
- ok = emqx_rule_registry:register_resource_types(
- [#resource_type{
- name = built_in,
- provider = ?APP,
- params_spec = #{},
- on_create = {?MODULE, on_resource_create},
- on_destroy = {?MODULE, on_resource_destroy},
- on_status = {?MODULE, on_get_resource_status},
- title = #{en => <<"Built-In Resource Type (debug)">>},
- description = #{en => <<"The built in resource type for debug purpose">>}}]),
- %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]),
Config.
-
end_per_testcase(t_events, Config) ->
ets:delete(events_record_tab),
- ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)),
- ok = emqx_rule_registry:remove_action('hook-metrics-action');
-end_per_testcase(Test, Config)
- when Test =:= t_sqlselect_multi_actoins_1,
- Test =:= t_sqlselect_multi_actoins_2
- ->
- emqtt:stop(?config(subclient, Config)),
- emqtt:stop(?config(connclient, Config)),
- Config;
+ ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config));
end_per_testcase(_TestCase, _Config) ->
ok.
%%------------------------------------------------------------------------------
%% Test cases for rule engine
%%------------------------------------------------------------------------------
-
-t_register_provider(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- ?assert(length(emqx_rule_registry:get_actions()) >= 2),
- ok.
-
-t_unregister_provider(_Config) ->
- ok = emqx_rule_engine:unload_providers(),
- ?assert(length(emqx_rule_registry:get_actions()) == 0),
- ok.
-
t_create_rule(_Config) ->
- ok = emqx_rule_engine:load_providers(),
{ok, #rule{id = Id}} = emqx_rule_engine:create_rule(
- #{rawsql => <<"select * from \"t/a\"">>,
- actions => [#{name => 'inspect', args => #{arg1 => 1}}],
+ #{sql => <<"select * from \"t/a\"">>,
+ id => <<"t_create_rule">>,
+ outputs => [#{type => builtin, target => console}],
description => <<"debug rule">>}),
- %ct:pal("======== emqx_rule_registry:get_rules :~p", [emqx_rule_registry:get_rules()]),
- ?assertMatch({ok,#rule{id = Id, for = [<<"t/a">>]}}, emqx_rule_registry:get_rule(Id)),
- ok = emqx_rule_engine:unload_providers(),
+ ct:pal("======== emqx_rule_registry:get_rules :~p", [emqx_rule_registry:get_rules()]),
+ ?assertMatch({ok, #rule{id = Id, info = #{from := [<<"t/a">>]}}},
+ emqx_rule_registry:get_rule(Id)),
emqx_rule_registry:remove_rule(Id),
ok.
-t_create_resource(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource(
- #{type => built_in,
- config => #{},
- description => <<"debug resource">>}),
- ?assert(true, is_binary(ResId)),
- ok = emqx_rule_engine:unload_providers(),
- emqx_rule_registry:remove_resource(ResId),
- ok.
-
-%%------------------------------------------------------------------------------
-%% Test cases for rule actions
-%%------------------------------------------------------------------------------
-
-t_inspect_action(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource(
- #{type => built_in,
- config => #{},
- description => <<"debug resource">>}),
- {ok, #rule{id = Id}} = emqx_rule_engine:create_rule(
- #{rawsql => "select clientid as c, username as u "
- "from \"t1\" ",
- actions => [#{name => 'inspect',
- args => #{'$resource' => ResId, a=>1, b=>2}}],
- type => built_in,
- description => <<"Inspect rule">>
- }),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- emqtt:publish(Client, <<"t1">>, <<"{\"id\": 1, \"name\": \"ha\"}">>, 0),
- emqtt:stop(Client),
- emqx_rule_registry:remove_rule(Id),
- emqx_rule_registry:remove_resource(ResId),
- ok.
-
-t_republish_action(_Config) ->
- Qos0Received = emqx_metrics:val('messages.qos0.received'),
- Received = emqx_metrics:val('messages.received'),
- ok = emqx_rule_engine:load_providers(),
- {ok, #rule{id = Id, for = [<<"t1">>]}} =
- emqx_rule_engine:create_rule(
- #{rawsql => <<"select topic, payload, qos from \"t1\"">>,
- actions => [#{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"${payload}">>}}],
- description => <<"builtin-republish-rule">>}),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
-
- Msg = <<"{\"id\": 1, \"name\": \"ha\"}">>,
- emqtt:publish(Client, <<"t1">>, Msg, 0),
- receive {publish, #{topic := <<"t2">>, payload := Payload}} ->
- ?assertEqual(Msg, Payload)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
- emqtt:stop(Client),
- emqx_rule_registry:remove_rule(Id),
- ?assertEqual(2, emqx_metrics:val('messages.qos0.received') - Qos0Received),
- ?assertEqual(2, emqx_metrics:val('messages.received') - Received),
- ok.
-
%%------------------------------------------------------------------------------
%% Test cases for rule engine api
%%------------------------------------------------------------------------------
t_crud_rule_api(_Config) ->
- {ok, #{code := 0, data := Rule}} =
- emqx_rule_engine_api:create_rule(#{},
- [{<<"name">>, <<"debug-rule">>},
- {<<"rawsql">>, <<"select * from \"t/a\"">>},
- {<<"actions">>, [[{<<"name">>,<<"inspect">>},
- {<<"params">>,[{<<"arg1">>,1}]}]]},
- {<<"description">>, <<"debug rule">>}]),
- RuleID = maps:get(id, Rule),
- %ct:pal("RCreated : ~p", [Rule]),
+ RuleID = <<"my_rule">>,
+ Params0 = #{
+ <<"description">> => <<"A simple rule">>,
+ <<"enable">> => true,
+ <<"id">> => RuleID,
+ <<"outputs">> => [#{<<"type">> => <<"builtin">>, <<"target">> => <<"console">>}],
+ <<"sql">> => <<"SELECT * from \"t/1\"">>
+ },
+ {201, Rule} = emqx_rule_engine_api:crud_rules(post, #{body => Params0}),
- {ok, #{code := 0, data := Rules}} = emqx_rule_engine_api:list_rules(#{}, []),
- %ct:pal("RList : ~p", [Rules]),
+ ?assertEqual(RuleID, maps:get(id, Rule)),
+ {200, Rules} = emqx_rule_engine_api:crud_rules(get, #{}),
+ ct:pal("RList : ~p", [Rules]),
?assert(length(Rules) > 0),
- {ok, #{code := 0, data := Rule1}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []),
- %ct:pal("RShow : ~p", [Rule1]),
+ {200, Rule1} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}),
+ ct:pal("RShow : ~p", [Rule1]),
?assertEqual(Rule, Rule1),
- {ok, #{code := 0, data := Rule2}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
- [{<<"rawsql">>, <<"select * from \"t/b\"">>}]),
+ {200, Rule2} = emqx_rule_engine_api:crud_rules_by_id(put, #{
+ bindings => #{id => RuleID},
+ body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>}
+ }),
- {ok, #{code := 0, data := Rule3}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []),
- %ct:pal("RShow : ~p", [Rule1]),
+ {200, Rule3} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}),
+ %ct:pal("RShow : ~p", [Rule3]),
?assertEqual(Rule3, Rule2),
- ?assertEqual(<<"select * from \"t/b\"">>, maps:get(rawsql, Rule3)),
+ ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)),
- {ok, #{code := 0, data := Rule4}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
- [{<<"actions">>,
- [[
- {<<"name">>,<<"republish">>},
- {<<"params">>,[
- {<<"arg1">>,1},
- {<<"target_topic">>, <<"t2">>},
- {<<"target_qos">>, -1},
- {<<"payload_tmpl">>, <<"${payload}">>}
- ]}
- ]]
- }]),
+ ?assertMatch({200}, emqx_rule_engine_api:crud_rules_by_id(delete,
+ #{bindings => #{id => RuleID}})),
- {ok, #{code := 0, data := Rule5}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []),
- %ct:pal("RShow : ~p", [Rule1]),
- ?assertEqual(Rule5, Rule4),
- ?assertMatch([#{name := republish }], maps:get(actions, Rule5)),
-
- ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => RuleID}, [])),
-
- NotFound = emqx_rule_engine_api:show_rule(#{id => RuleID}, []),
%ct:pal("Show After Deleted: ~p", [NotFound]),
- ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound),
- ok.
-
-t_list_actions_api(_Config) ->
- {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []),
- %ct:pal("RList : ~p", [Actions]),
- ?assert(length(Actions) > 0),
- ok.
-
-t_show_action_api(_Config) ->
- {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:show_action(#{name => 'inspect'}, []),
- ?assertEqual('inspect', maps:get(name, Actions)),
- ok.
-
-t_crud_resources_api(_Config) ->
- {ok, #{code := 0, data := Resources1}} =
- emqx_rule_engine_api:create_resource(#{},
- [{<<"name">>, <<"Simple Resource">>},
- {<<"type">>, <<"built_in">>},
- {<<"config">>, [{<<"a">>, 1}]},
- {<<"description">>, <<"Simple Resource">>}]),
- ResId = maps:get(id, Resources1),
- {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{}, []),
- ?assert(length(Resources) > 0),
- {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources2)),
- %
- {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
- [{<<"config">>, [{<<"a">>, 2}]},
- {<<"description">>, <<"2">>}]),
- {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources3)),
- ?assertEqual(#{<<"a">> => 2}, maps:get(config, Resources3)),
- ?assertEqual(<<"2">>, maps:get(description, Resources3)),
- %
- {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
- [{<<"config">>, [{<<"a">>, 3}]}]),
- {ok, #{code := 0, data := Resources4}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources4)),
- ?assertEqual(#{<<"a">> => 3}, maps:get(config, Resources4)),
- ?assertEqual(<<"2">>, maps:get(description, Resources4)),
- % Only config
- {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
- [{<<"config">>, [{<<"a">>, 1},
- {<<"b">>, 2},
- {<<"c">>, 3}]}]),
- {ok, #{code := 0, data := Resources5}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources5)),
- ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources5)),
- ?assertEqual(<<"2">>, maps:get(description, Resources5)),
- % Only description
- {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
- [{<<"description">>, <<"new5">>}]),
- {ok, #{code := 0, data := Resources6}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources6)),
- ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources6)),
- ?assertEqual(<<"new5">>, maps:get(description, Resources6)),
- % None
- {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, []),
- {ok, #{code := 0, data := Resources7}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
- ?assertEqual(ResId, maps:get(id, Resources7)),
- ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources7)),
- ?assertEqual(<<"new5">>, maps:get(description, Resources7)),
- %
- ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_resource(#{id => ResId},#{})),
- ?assertMatch({ok, #{code := 404}}, emqx_rule_engine_api:show_resource(#{id => ResId}, [])),
- ok.
-
-t_list_resource_types_api(_Config) ->
- {ok, #{code := 0, data := ResourceTypes}} = emqx_rule_engine_api:list_resource_types(#{}, []),
- ?assert(length(ResourceTypes) > 0),
- ok.
-
-t_show_resource_type_api(_Config) ->
- {ok, #{code := 0, data := RShow}} = emqx_rule_engine_api:show_resource_type(#{name => 'built_in'}, []),
- %ct:pal("RShow : ~p", [RShow]),
- ?assertEqual(built_in, maps:get(name, RShow)),
- ok.
-
-%%------------------------------------------------------------------------------
-%% Test cases for rule engine cli
-%%------------------------------------------------------------------------------
-
-t_rules_cli(_Config) ->
- mock_print(),
- RCreate = emqx_rule_engine_cli:rules(["create",
- "select * from \"t1\" where topic='t1'",
- "[{\"name\":\"inspect\", \"params\": {\"arg1\": 1}}]",
- "-d", "Debug Rule"]),
- %ct:pal("Result : ~p", [RCreate]),
- ?assertMatch({match, _}, re:run(RCreate, "created")),
-
- RuleId = re:replace(re:replace(RCreate, "Rule\s", "", [{return, list}]), "\screated\n", "", [{return, list}]),
-
- RList = emqx_rule_engine_cli:rules(["list"]),
- ?assertMatch({match, _}, re:run(RList, RuleId)),
- %ct:pal("RList : ~p", [RList]),
- %ct:pal("table action params: ~p", [ets:tab2list(emqx_action_instance_params)]),
-
- RShow = emqx_rule_engine_cli:rules(["show", RuleId]),
- ?assertMatch({match, _}, re:run(RShow, RuleId)),
- %ct:pal("RShow : ~p", [RShow]),
-
- RUpdate = emqx_rule_engine_cli:rules(["update",
- RuleId,
- "-s", "select * from \"t2\""]),
- ?assertMatch({match, _}, re:run(RUpdate, "updated")),
-
- RDelete = emqx_rule_engine_cli:rules(["delete", RuleId]),
- ?assertEqual("ok~n", RDelete),
- %ct:pal("RDelete : ~p", [RDelete]),
- %ct:pal("table action params after deleted: ~p", [ets:tab2list(emqx_action_instance_params)]),
-
- RShow2 = emqx_rule_engine_cli:rules(["show", RuleId]),
- ?assertMatch({match, _}, re:run(RShow2, "Cannot found")),
- %ct:pal("RShow2 : ~p", [RShow2]),
- unmock_print(),
- ok.
-
-t_actions_cli(_Config) ->
- mock_print(),
- RList = emqx_rule_engine_cli:actions(["list"]),
- ?assertMatch({match, _}, re:run(RList, "inspect")),
- %ct:pal("RList : ~p", [RList]),
-
- RShow = emqx_rule_engine_cli:actions(["show", "inspect"]),
- ?assertMatch({match, _}, re:run(RShow, "inspect")),
- %ct:pal("RShow : ~p", [RShow]),
- unmock_print(),
- ok.
-
-t_resources_cli(_Config) ->
- mock_print(),
- RCreate = emqx_rule_engine_cli:resources(["create", "built_in", "{\"a\" : 1}", "-d", "test resource"]),
- ResId = re:replace(re:replace(RCreate, "Resource\s", "", [{return, list}]), "\screated\n", "", [{return, list}]),
-
- RList = emqx_rule_engine_cli:resources(["list"]),
- ?assertMatch({match, _}, re:run(RList, "test resource")),
- %ct:pal("RList : ~p", [RList]),
-
- RListT = emqx_rule_engine_cli:resources(["list", "-t", "built_in"]),
- ?assertMatch({match, _}, re:run(RListT, "test resource")),
- %ct:pal("RListT : ~p", [RListT]),
-
- RShow = emqx_rule_engine_cli:resources(["show", ResId]),
- ?assertMatch({match, _}, re:run(RShow, "test resource")),
- %ct:pal("RShow : ~p", [RShow]),
-
- RDelete = emqx_rule_engine_cli:resources(["delete", ResId]),
- ?assertEqual("ok~n", RDelete),
-
- RShow2 = emqx_rule_engine_cli:resources(["show", ResId]),
- ?assertMatch({match, _}, re:run(RShow2, "Cannot found")),
- %ct:pal("RShow2 : ~p", [RShow2]),
- unmock_print(),
- ok.
-
-t_resource_types_cli(_Config) ->
- mock_print(),
- RList = emqx_rule_engine_cli:resource_types(["list"]),
- ?assertMatch({match, _}, re:run(RList, "built_in")),
- %ct:pal("RList : ~p", [RList]),
-
- RShow = emqx_rule_engine_cli:resource_types(["show", "inspect"]),
- ?assertMatch({match, _}, re:run(RShow, "inspect")),
- %ct:pal("RShow : ~p", [RShow]),
- unmock_print(),
+ ?assertMatch({404, #{code := _, message := _Message}},
+ emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}})),
ok.
%%------------------------------------------------------------------------------
%% Test cases for rule funcs
%%------------------------------------------------------------------------------
-t_topic_func(_Config) ->
- %%TODO:
- ok.
-
t_kv_store(_) ->
undefined = emqx_rule_funcs:kv_store_get(<<"abc">>),
<<"not_found">> = emqx_rule_funcs:kv_store_get(<<"abc">>, <<"not_found">>),
@@ -619,7 +246,6 @@ t_kv_store(_) ->
%%------------------------------------------------------------------------------
t_add_get_remove_rule(_Config) ->
- mock_print(),
RuleId0 = <<"rule-debug-0">>,
ok = emqx_rule_registry:add_rule(make_simple_rule(RuleId0)),
?assertMatch({ok, #rule{id = RuleId0}}, emqx_rule_registry:get_rule(RuleId0)),
@@ -632,7 +258,6 @@ t_add_get_remove_rule(_Config) ->
?assertMatch({ok, #rule{id = RuleId1}}, emqx_rule_registry:get_rule(RuleId1)),
ok = emqx_rule_registry:remove_rule(Rule1),
?assertEqual(not_found, emqx_rule_registry:get_rule(RuleId1)),
- unmock_print(),
ok.
t_add_get_remove_rules(_Config) ->
@@ -656,96 +281,22 @@ t_create_existing_rule(_Config) ->
%% create a rule using given rule id
{ok, _} = emqx_rule_engine:create_rule(
#{id => <<"an_existing_rule">>,
- rawsql => <<"select * from \"t/#\"">>,
- actions => [
- #{name => 'inspect', args => #{}}
- ]
+ sql => <<"select * from \"t/#\"">>,
+ outputs => [#{type => builtin, target => console}]
}),
- {ok, #rule{rawsql = SQL}} = emqx_rule_registry:get_rule(<<"an_existing_rule">>),
+ {ok, #rule{info = #{sql := SQL}}} = emqx_rule_registry:get_rule(<<"an_existing_rule">>),
?assertEqual(<<"select * from \"t/#\"">>, SQL),
ok = emqx_rule_engine:delete_rule(<<"an_existing_rule">>),
?assertEqual(not_found, emqx_rule_registry:get_rule(<<"an_existing_rule">>)),
ok.
-t_update_rule(_Config) ->
- {ok, #rule{actions = [#action_instance{id = ActInsId0}]}} = emqx_rule_engine:create_rule(
- #{id => <<"an_existing_rule">>,
- rawsql => <<"select * from \"t/#\"">>,
- actions => [
- #{name => 'inspect', args => #{}}
- ]
- }),
- ?assertMatch({ok, #action_instance_params{apply = _}},
- emqx_rule_registry:get_action_instance_params(ActInsId0)),
- %% update the rule and verify the old action instances has been cleared
- %% and the new action instances has been created.
- emqx_rule_engine:update_rule(#{ id => <<"an_existing_rule">>,
- actions => [
- #{name => 'do_nothing', args => #{}}
- ]}),
-
- {ok, #rule{actions = [#action_instance{id = ActInsId1}]}}
- = emqx_rule_registry:get_rule(<<"an_existing_rule">>),
-
- ?assertMatch(not_found,
- emqx_rule_registry:get_action_instance_params(ActInsId0)),
-
- ?assertMatch({ok, #action_instance_params{apply = _}},
- emqx_rule_registry:get_action_instance_params(ActInsId1)),
-
- ok = emqx_rule_engine:delete_rule(<<"an_existing_rule">>),
- ?assertEqual(not_found, emqx_rule_registry:get_rule(<<"an_existing_rule">>)),
- ok.
-
-t_disable_rule(_Config) ->
- ets:new(simple_action_2, [named_table, set, public]),
- ets:insert(simple_action_2, {created, 0}),
- ets:insert(simple_action_2, {destroyed, 0}),
- Now = erlang:timestamp(),
- emqx_rule_registry:add_action(
- #action{name = 'simple_action_2', app = ?APP,
- module = ?MODULE,
- on_create = simple_action_2_create,
- on_destroy = simple_action_2_destroy,
- types=[], params_spec = #{},
- title = #{en => <<"Simple Action">>},
- description = #{en => <<"Simple Action">>}}),
- {ok, #rule{actions = [#action_instance{}]}} = emqx_rule_engine:create_rule(
- #{id => <<"simple_rule_2">>,
- rawsql => <<"select * from \"t/#\"">>,
- actions => [#{name => 'simple_action_2', args => #{}}]
- }),
- [{_, CAt}] = ets:lookup(simple_action_2, created),
- ?assert(CAt > Now),
- [{_, DAt}] = ets:lookup(simple_action_2, destroyed),
- ?assert(DAt < Now),
-
- %% disable the rule and verify the old action instances has been cleared
- Now2 = erlang:timestamp(),
- emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>,
- enabled => false}),
- [{_, CAt2}] = ets:lookup(simple_action_2, created),
- ?assert(CAt2 < Now2),
- [{_, DAt2}] = ets:lookup(simple_action_2, destroyed),
- ?assert(DAt2 > Now2),
-
- %% enable the rule again and verify the action instances has been created
- Now3 = erlang:timestamp(),
- emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>,
- enabled => true}),
- [{_, CAt3}] = ets:lookup(simple_action_2, created),
- ?assert(CAt3 > Now3),
- [{_, DAt3}] = ets:lookup(simple_action_2, destroyed),
- ?assert(DAt3 < Now3),
- ok = emqx_rule_engine:delete_rule(<<"simple_rule_2">>).
-
-t_get_rules_for(_Config) ->
- Len0 = length(emqx_rule_registry:get_rules_for(<<"simple/topic">>)),
+t_get_rules_for_topic(_Config) ->
+ Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>)),
ok = emqx_rule_registry:add_rules(
[make_simple_rule(<<"rule-debug-1">>),
make_simple_rule(<<"rule-debug-2">>)]),
- ?assertEqual(Len0+2, length(emqx_rule_registry:get_rules_for(<<"simple/topic">>))),
+ ?assertEqual(Len0+2, length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>))),
ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>]),
ok.
@@ -762,8 +313,8 @@ t_get_rules_ordered_by_ts(_Config) ->
#rule{id = <<"rule-debug-2">>}
], emqx_rule_registry:get_rules_ordered_by_ts()).
-t_get_rules_for_2(_Config) ->
- Len0 = length(emqx_rule_registry:get_rules_for(<<"simple/1">>)),
+t_get_rules_for_topic_2(_Config) ->
+ Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>)),
ok = emqx_rule_registry:add_rules(
[make_simple_rule(<<"rule-debug-1">>, <<"select * from \"simple/#\"">>, [<<"simple/#">>]),
make_simple_rule(<<"rule-debug-2">>, <<"select * from \"simple/+\"">>, [<<"simple/+">>]),
@@ -772,7 +323,7 @@ t_get_rules_for_2(_Config) ->
make_simple_rule(<<"rule-debug-5">>, <<"select * from \"simple/2,simple/+,simple/3\"">>, [<<"simple/2">>,<<"simple/+">>, <<"simple/3">>]),
make_simple_rule(<<"rule-debug-6">>, <<"select * from \"simple/2,simple/3,simple/4\"">>, [<<"simple/2">>,<<"simple/3">>, <<"simple/4">>])
]),
- ?assertEqual(Len0+4, length(emqx_rule_registry:get_rules_for(<<"simple/1">>))),
+ ?assertEqual(Len0+4, length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>))),
ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>,<<"rule-debug-3">>, <<"rule-debug-4">>,<<"rule-debug-5">>, <<"rule-debug-6">>]),
ok.
@@ -809,93 +360,6 @@ t_get_rules_with_same_event(_Config) ->
ok = emqx_rule_registry:remove_rules([<<"r1">>, <<"r2">>,<<"r3">>, <<"r4">>,<<"r5">>, <<"r6">>, <<"r7">>, <<"r8">>, <<"r9">>, <<"r10">>]),
ok.
-t_add_get_remove_action(_Config) ->
- ActionName0 = 'action-debug-0',
- Action0 = make_simple_action(ActionName0),
- ok = emqx_rule_registry:add_action(Action0),
- ?assertMatch({ok, #action{name = ActionName0}}, emqx_rule_registry:find_action(ActionName0)),
- ok = emqx_rule_registry:remove_action(ActionName0),
- ?assertMatch(not_found, emqx_rule_registry:find_action(ActionName0)),
-
- ok = emqx_rule_registry:add_action(Action0),
- ?assertMatch({ok, #action{name = ActionName0}}, emqx_rule_registry:find_action(ActionName0)),
- ok = emqx_rule_registry:remove_action(Action0),
- ?assertMatch(not_found, emqx_rule_registry:find_action(ActionName0)),
- ok.
-
-t_add_get_remove_actions(_Config) ->
- InitActionLen = length(emqx_rule_registry:get_actions()),
- ActionName1 = 'action-debug-1',
- ActionName2 = 'action-debug-2',
- Action1 = make_simple_action(ActionName1),
- Action2 = make_simple_action(ActionName2),
- ok = emqx_rule_registry:add_actions([Action1, Action2]),
- ?assertMatch(2, length(emqx_rule_registry:get_actions()) - InitActionLen),
- ok = emqx_rule_registry:remove_actions([ActionName1, ActionName2]),
- ?assertMatch(InitActionLen, length(emqx_rule_registry:get_actions())),
-
- ok = emqx_rule_registry:add_actions([Action1, Action2]),
- ?assertMatch(2, length(emqx_rule_registry:get_actions()) - InitActionLen),
- ok = emqx_rule_registry:remove_actions([Action1, Action2]),
- ?assertMatch(InitActionLen, length(emqx_rule_registry:get_actions())),
- ok.
-
-t_remove_actions_of(_Config) ->
- ok = emqx_rule_registry:add_actions([make_simple_action('action-debug-1'),
- make_simple_action('action-debug-2')]),
- Len1 = length(emqx_rule_registry:get_actions()),
- ?assert(Len1 >= 2),
- ok = emqx_rule_registry:remove_actions_of(?APP),
- ?assert((Len1 - length(emqx_rule_registry:get_actions())) >= 2),
- ok.
-
-t_add_get_remove_resource(_Config) ->
- ResId = <<"resource-debug">>,
- Res = make_simple_resource(ResId),
- ok = emqx_rule_registry:add_resource(Res),
- ?assertMatch({ok, #resource{id = ResId}}, emqx_rule_registry:find_resource(ResId)),
- ok = emqx_rule_registry:remove_resource(ResId),
- ?assertEqual(not_found, emqx_rule_registry:find_resource(ResId)),
- ok = emqx_rule_registry:add_resource(Res),
- ?assertMatch({ok, #resource{id = ResId}}, emqx_rule_registry:find_resource(ResId)),
- ok = emqx_rule_registry:remove_resource(Res),
- ?assertEqual(not_found, emqx_rule_registry:find_resource(ResId)),
- ok.
-t_get_resources(_Config) ->
- Len0 = length(emqx_rule_registry:get_resources()),
- Res1 = make_simple_resource(<<"resource-debug-1">>),
- Res2 = make_simple_resource(<<"resource-debug-2">>),
- ok = emqx_rule_registry:add_resource(Res1),
- ok = emqx_rule_registry:add_resource(Res2),
- ?assertEqual(Len0+2, length(emqx_rule_registry:get_resources())),
- ok.
-
-t_resource_types(_Config) ->
- register_resource_types(),
- get_resource_type(),
- get_resource_types(),
- unregister_resource_types_of().
-
-register_resource_types() ->
- ResType1 = make_simple_resource_type(<<"resource-type-debug-1">>),
- ResType2 = make_simple_resource_type(<<"resource-type-debug-2">>),
- emqx_rule_registry:register_resource_types([ResType1,ResType2]),
- ok.
-get_resource_type() ->
- ?assertMatch({ok, #resource_type{name = <<"resource-type-debug-1">>}}, emqx_rule_registry:find_resource_type(<<"resource-type-debug-1">>)),
- ok.
-get_resource_types() ->
- ResTypes = emqx_rule_registry:get_resource_types(),
- ct:pal("resource types now: ~p", [ResTypes]),
- ?assert(length(ResTypes) > 0),
- ok.
-unregister_resource_types_of() ->
- NumOld = length(emqx_rule_registry:get_resource_types()),
- ok = emqx_rule_registry:unregister_resource_types_of(?APP),
- NumNow = length(emqx_rule_registry:get_resource_types()),
- ?assert((NumOld - NumNow) >= 2),
- ok.
-
%%------------------------------------------------------------------------------
%% Test cases for rule runtime
%%------------------------------------------------------------------------------
@@ -966,37 +430,14 @@ message_acked(_Client) ->
verify_event('message.acked'),
ok.
-t_mfa_action(_Config) ->
- ok = emqx_rule_registry:add_action(
- #action{name = 'mfa-action', app = ?APP,
- module = ?MODULE, on_create = mfa_action,
- types=[], params_spec = #{},
- title = #{en => <<"MFA callback action">>},
- description = #{en => <<"MFA callback action">>}}),
- SQL = "SELECT * FROM \"t1\"",
- {ok, #rule{id = Id}} = emqx_rule_engine:create_rule(
- #{id => <<"rule:t_mfa_action">>,
- rawsql => SQL,
- actions => [#{id => <<"action:mfa-test">>, name => 'mfa-action', args => #{}}],
- description => <<"Debug rule">>}),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- emqtt:publish(Client, <<"t1">>, <<"{\"id\": 1, \"name\": \"ha\"}">>, 0),
- emqtt:stop(Client),
- ct:sleep(500),
- ?assertEqual(1, persistent_term:get(<<"action:mfa-test">>, 0)),
- emqx_rule_registry:remove_rule(Id),
- emqx_rule_registry:remove_action('mfa-action'),
- ok.
-
t_match_atom_and_binary(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT connected_at as ts, * "
- "FROM \"$events/client_connected\" "
- "WHERE username = 'emqx2' ",
- <<"user:${ts}">>),
+ SQL = "SELECT connected_at as ts, * "
+ "FROM \"$events/client_connected\" "
+ "WHERE username = 'emqx2' ",
+ Repub = republish_output(<<"t2">>, <<"user:${ts}">>),
+ {ok, TopicRule} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{username, <<"emqx1">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1021,32 +462,32 @@ t_sqlselect_0(_Config) ->
"where payload.cmd.info = 'tt'",
?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql,
+ context =>
+ #{payload =>
<<"{\"cmd\": {\"info\":\"tt\"}}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
Sql2 = "select payload.cmd as cmd "
"from \"t/#\" "
"where cmd.info = 'tt'",
?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql2,
+ context =>
+ #{payload =>
<<"{\"cmd\": {\"info\":\"tt\"}}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
Sql3 = "select payload.cmd as cmd, cmd.info as info "
"from \"t/#\" "
"where cmd.info = 'tt' and info = 'tt'",
?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>},
<<"info">> := <<"tt">>}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql3,
+ context =>
+ #{payload =>
<<"{\"cmd\": {\"info\":\"tt\"}}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
%% cascaded as
Sql4 = "select payload.cmd as cmd, cmd.info as meta.info "
"from \"t/#\" "
@@ -1054,11 +495,11 @@ t_sqlselect_0(_Config) ->
?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>},
<<"meta">> := #{<<"info">> := <<"tt">>}}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql4,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql4,
+ context =>
+ #{payload =>
<<"{\"cmd\": {\"info\":\"tt\"}}">>,
- <<"topic">> => <<"t/a">>}})).
+ topic => <<"t/a">>}})).
t_sqlselect_00(_Config) ->
%% Verify plus/subtract and unary_add_or_subtract
@@ -1066,42 +507,43 @@ t_sqlselect_00(_Config) ->
"from \"t/#\" ",
?assertMatch({ok,#{<<"a">> := 0}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context =>
+ #{payload => <<"">>,
+ topic => <<"t/a">>}})),
Sql1 = "select -1 + 1 as a "
"from \"t/#\" ",
?assertMatch({ok,#{<<"a">> := 0}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> =>
- #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql1,
+ context =>
+ #{payload => <<"">>,
+ topic => <<"t/a">>}})),
Sql2 = "select 1 + 1 as a "
"from \"t/#\" ",
?assertMatch({ok,#{<<"a">> := 2}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql2,
+ context =>
+ #{payload => <<"">>,
+ topic => <<"t/a">>}})),
Sql3 = "select +1 as a "
"from \"t/#\" ",
?assertMatch({ok,#{<<"a">> := 1}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> =>
- #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql3,
+ context =>
+ #{payload => <<"">>,
+ topic => <<"t/a">>}})).
t_sqlselect_01(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- TopicRule1 = create_simple_repub_rule(
- <<"t2">>,
- "SELECT json_decode(payload) as p, payload "
- "FROM \"t3/#\", \"t1\" "
- "WHERE p.x = 1"),
+ SQL = "SELECT json_decode(payload) as p, payload "
+ "FROM \"t3/#\", \"t1\" "
+ "WHERE p.x = 1",
+ Repub = republish_output(<<"t2">>),
+ {ok, TopicRule1} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1133,12 +575,13 @@ t_sqlselect_01(_Config) ->
emqx_rule_registry:remove_rule(TopicRule1).
t_sqlselect_02(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- TopicRule1 = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"t3/#\", \"t1\" "
- "WHERE payload.x = 1"),
+ SQL = "SELECT * "
+ "FROM \"t3/#\", \"t1\" "
+ "WHERE payload.x = 1",
+ Repub = republish_output(<<"t2">>),
+ {ok, TopicRule1} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1170,12 +613,13 @@ t_sqlselect_02(_Config) ->
emqx_rule_registry:remove_rule(TopicRule1).
t_sqlselect_1(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT json_decode(payload) as p, payload "
- "FROM \"t1\" "
- "WHERE p.x = 1 and p.y = 2"),
+ SQL = "SELECT json_decode(payload) as p, payload "
+ "FROM \"t1\" "
+ "WHERE p.x = 1 and p.y = 2",
+ Repub = republish_output(<<"t2">>),
+ {ok, TopicRule} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1199,12 +643,12 @@ t_sqlselect_1(_Config) ->
emqx_rule_registry:remove_rule(TopicRule).
t_sqlselect_2(_Config) ->
- ok = emqx_rule_engine:load_providers(),
%% recursively republish to t2
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"t2\" "),
+ SQL = "SELECT * FROM \"t2\" ",
+ Repub = republish_output(<<"t2">>),
+ {ok, TopicRule} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1224,91 +668,15 @@ t_sqlselect_2(_Config) ->
emqtt:stop(Client),
emqx_rule_registry:remove_rule(TopicRule).
-t_sqlselect_2_1(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- %% recursively republish to t2, if the msg dropped
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"$events/message_dropped\" "),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0),
- Fun = fun() ->
- receive {publish, #{topic := <<"t2">>, payload := _}} ->
- received_t2
- after 500 ->
- received_nothing
- end
- end,
- received_nothing = Fun(),
-
- %% it should not keep republishing "t2"
- {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
- received_nothing = Fun(),
-
- emqtt:stop(Client),
- emqx_rule_registry:remove_rule(TopicRule).
-
-t_sqlselect_2_2(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- %% recursively republish to t2, if the msg acked
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"$events/message_acked\" "),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1),
- emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1),
- Fun = fun() ->
- receive {publish, #{topic := <<"t2">>, payload := _}} ->
- received_t2
- after 500 ->
- received_nothing
- end
- end,
- received_t2 = Fun(),
- received_t2 = Fun(),
- received_nothing = Fun(),
-
- emqtt:stop(Client),
- emqx_rule_registry:remove_rule(TopicRule).
-
-t_sqlselect_2_3(_Config) ->
- ok = emqx_rule_engine:load_providers(),
- %% recursively republish to t2, if the msg delivered
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"$events/message_delivered\" "),
- {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
- {ok, _} = emqtt:connect(Client),
- {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
- emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0),
- Fun = fun() ->
- receive {publish, #{topic := <<"t2">>, payload := _}} ->
- received_t2
- after 500 ->
- received_nothing
- end
- end,
- received_t2 = Fun(),
- received_t2 = Fun(),
- received_nothing = Fun(),
-
- emqtt:stop(Client),
- emqx_rule_registry:remove_rule(TopicRule).
-
t_sqlselect_3(_Config) ->
- ok = emqx_rule_engine:load_providers(),
%% republish the client.connected msg
- TopicRule = create_simple_repub_rule(
- <<"t2">>,
- "SELECT * "
- "FROM \"$events/client_connected\" "
- "WHERE username = 'emqx1'",
- <<"clientid=${clientid}">>),
+ SQL = "SELECT * "
+ "FROM \"$events/client_connected\" "
+ "WHERE username = 'emqx1'",
+ Repub = republish_output(<<"t2">>, <<"clientid=${clientid}">>),
+ {ok, TopicRule} = emqx_rule_engine:create_rule(
+ #{sql => SQL, id => ?TMP_RULEID,
+ outputs => [Repub]}),
{ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
@@ -1332,226 +700,29 @@ t_sqlselect_3(_Config) ->
emqtt:stop(Client),
emqx_rule_registry:remove_rule(TopicRule).
-t_sqlselect_multi_actoins_1(Config) ->
- %% We create 2 actions in the same rule:
- %% The first will fail and we need to make sure the
- %% second one can still execute as the on_action_failed
- %% defaults to 'continue'
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- actions => [
- #{name => 'crash_action', args => #{}, fallbacks => []},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- receive {publish, #{topic := T, payload := Payload}} ->
- ?assertEqual(<<"t2">>, T),
- ?assertEqual(<<"clientid=c_emqx1">>, Payload)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
-
- emqx_rule_registry:remove_rule(Rule).
-
-t_sqlselect_multi_actoins_1_1(Config) ->
- %% Try again but set on_action_failed = 'continue' explicitly
- {ok, Rule2} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- on_action_failed => 'continue',
- actions => [
- #{name => 'crash_action', args => #{}, fallbacks => []},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- receive {publish, #{topic := T2, payload := Payload2}} ->
- ?assertEqual(<<"t2">>, T2),
- ?assertEqual(<<"clientid=c_emqx1">>, Payload2)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
-
- emqx_rule_registry:remove_rule(Rule2).
-
-t_sqlselect_multi_actoins_2(Config) ->
- %% We create 2 actions in the same rule:
- %% The first will fail and we need to make sure the
- %% second one cannot execute as we've set the on_action_failed = 'stop'
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- on_action_failed => stop,
- actions => [
- #{name => 'crash_action', args => #{}, fallbacks => []},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- receive {publish, #{topic := <<"t2">>}} ->
- ct:fail(unexpected_t2)
- after 1000 ->
- ok
- end,
-
- emqx_rule_registry:remove_rule(Rule).
-
-t_sqlselect_multi_actoins_3(Config) ->
- %% We create 2 actions in the same rule (on_action_failed = continue):
- %% The first will fail and we need to make sure the
- %% fallback actions can be executed, and the next actoins
- %% will be run without influence
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- on_action_failed => continue,
- actions => [
- #{name => 'crash_action', args => #{}, fallbacks =>[
- #{name => 'plus_by_one', args => #{}, fallbacks =>[]},
- #{name => 'plus_by_one', args => #{}, fallbacks =>[]}
- ]},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- timer:sleep(100),
-
- %% verfiy the fallback actions has been run
- ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)),
-
- %% verfiy the next actions can be run
- receive {publish, #{topic := T, payload := Payload}} ->
- ?assertEqual(<<"t2">>, T),
- ?assertEqual(<<"clientid=c_emqx1">>, Payload)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
-
- emqx_rule_registry:remove_rule(Rule).
-
-t_sqlselect_multi_actoins_3_1(Config) ->
- %% We create 2 actions in the same rule (on_action_failed = continue):
- %% The first will fail (with a 'badact' return) and we need to make sure the
- %% fallback actions can be executed, and the next actoins
- %% will be run without influence
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- on_action_failed => continue,
- actions => [
- #{name => 'failure_action', args => #{}, fallbacks =>[
- #{name => 'plus_by_one', args => #{}, fallbacks =>[]},
- #{name => 'plus_by_one', args => #{}, fallbacks =>[]}
- ]},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- timer:sleep(100),
-
- %% verfiy the fallback actions has been run
- ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)),
-
- %% verfiy the next actions can be run
- receive {publish, #{topic := T, payload := Payload}} ->
- ?assertEqual(<<"t2">>, T),
- ?assertEqual(<<"clientid=c_emqx1">>, Payload)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
-
- emqx_rule_registry:remove_rule(Rule).
-
-t_sqlselect_multi_actoins_4(Config) ->
- %% We create 2 actions in the same rule (on_action_failed = continue):
- %% The first will fail and we need to make sure the
- %% fallback actions can be executed, and the next actoins
- %% will be run without influence
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => ?config(connsql, Config),
- on_action_failed => continue,
- actions => [
- #{name => 'crash_action', args => #{}, fallbacks => [
- #{name =>'plus_by_one', args => #{}, fallbacks =>[]},
- #{name =>'crash_action', args => #{}, fallbacks =>[]},
- #{name =>'plus_by_one', args => #{}, fallbacks =>[]}
- ]},
- #{name => 'republish',
- args => #{<<"target_topic">> => <<"t2">>,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => <<"clientid=${clientid}">>
- },
- fallbacks => []}
- ]
- }),
-
- (?config(conn_event, Config))(),
- timer:sleep(100),
-
- %% verfiy all the fallback actions were run, even if the second
- %% fallback action crashed
- ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)),
-
- %% verfiy the next actions can be run
- receive {publish, #{topic := T, payload := Payload}} ->
- ?assertEqual(<<"t2">>, T),
- ?assertEqual(<<"clientid=c_emqx1">>, Payload)
- after 1000 ->
- ct:fail(wait_for_t2)
- end,
-
- emqx_rule_registry:remove_rule(Rule).
-
t_sqlparse_event_1(_Config) ->
Sql = "select topic as tp "
"from \"$events/session_subscribed\" ",
?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})).
+ #{sql => Sql,
+ context => #{topic => <<"t/tt">>}})).
t_sqlparse_event_2(_Config) ->
Sql = "select clientid "
"from \"$events/client_connected\" ",
?assertMatch({ok,#{<<"clientid">> := <<"abc">>}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"clientid">> => <<"abc">>}})).
+ #{sql => Sql,
+ context => #{clientid => <<"abc">>}})).
t_sqlparse_event_3(_Config) ->
Sql = "select clientid, topic as tp "
"from \"t/tt\", \"$events/client_connected\" ",
?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})).
+ #{sql => Sql,
+ context => #{clientid => <<"abc">>, topic => <<"t/tt">>}})).
t_sqlparse_foreach_1(_Config) ->
%% Verify foreach with and without 'AS'
@@ -1559,34 +730,35 @@ t_sqlparse_foreach_1(_Config) ->
"from \"t/#\" ",
?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"sensors\": [1, 2]}">>,
+ topic => <<"t/a">>}})),
Sql2 = "foreach payload.sensors "
"from \"t/#\" ",
?assertMatch({ok,[#{item := 1}, #{item := 2}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql2,
+ context => #{payload => <<"{\"sensors\": [1, 2]}">>,
+ topic => <<"t/a">>}})),
Sql3 = "foreach payload.sensors "
"from \"t/#\" ",
?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>},
#{item := #{<<"cmd">> := <<"2">>, <<"name">> := <<"ct">>}, clientid := <<"c_a">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> => #{
- <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\",\"name\":\"ct\"}]}">>, <<"clientid">> => <<"c_a">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql3,
+ context => #{
+ payload => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\",\"name\":\"ct\"}]}">>,
+ clientid => <<"c_a">>,
+ topic => <<"t/a">>}})),
Sql4 = "foreach payload.sensors "
"from \"t/#\" ",
{ok,[#{metadata := #{rule_id := TRuleId}},
#{metadata := #{rule_id := TRuleId}}]} =
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql4,
- <<"ctx">> => #{
- <<"payload">> => <<"{\"sensors\": [1, 2]}">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql4,
+ context => #{
+ payload => <<"{\"sensors\": [1, 2]}">>,
+ topic => <<"t/a">>}}),
?assert(is_binary(TRuleId)).
t_sqlparse_foreach_2(_Config) ->
@@ -1596,31 +768,31 @@ t_sqlparse_foreach_2(_Config) ->
"from \"t/#\" ",
?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
Sql2 = "foreach payload.sensors "
"do item.cmd as msg_type "
"from \"t/#\" ",
?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql2,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
Sql3 = "foreach payload.sensors "
"do item as item "
"from \"t/#\" ",
?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql3,
+ context =>
+ #{payload =>
<<"{\"sensors\": [1, 2]}">>,
- <<"topic">> => <<"t/a">>}})).
+ topic => <<"t/a">>}})).
t_sqlparse_foreach_3(_Config) ->
%% Verify foreach-incase with and without 'AS'
@@ -1631,11 +803,11 @@ t_sqlparse_foreach_3(_Config) ->
#{<<"s">> := #{<<"cmd">> := 3}}
]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
Sql2 = "foreach payload.sensors "
"incase item.cmd != 1 "
"from \"t/#\" ",
@@ -1643,11 +815,11 @@ t_sqlparse_foreach_3(_Config) ->
#{item := #{<<"cmd">> := 3}}
]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql2,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>,
- <<"topic">> => <<"t/a">>}})).
+ topic => <<"t/a">>}})).
t_sqlparse_foreach_4(_Config) ->
%% Verify foreach-do-incase
@@ -1657,24 +829,24 @@ t_sqlparse_foreach_4(_Config) ->
"from \"t/#\" ",
?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
?assertMatch({ok,[#{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>}, #{<<"msg_type">> := <<"2">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> =>
+ #{sql => Sql,
+ context =>
+ #{payload =>
<<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, {\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>,
- <<"topic">> => <<"t/a">>}})),
+ topic => <<"t/a">>}})),
?assertMatch({ok,[]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql,
+ context =>
+ #{payload => <<"{\"sensors\": [1, 2]}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_foreach_5(_Config) ->
%% Verify foreach on a empty-list or non-list variable
@@ -1682,23 +854,23 @@ t_sqlparse_foreach_5(_Config) ->
"do s.cmd as msg_type, s.name as name "
"from \"t/#\" ",
?assertMatch({ok,[]}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => <<"{\"sensors\": 1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context =>
+ #{payload => <<"{\"sensors\": 1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok,[]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => <<"{\"sensors\": []}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context =>
+ #{payload => <<"{\"sensors\": []}">>,
+ topic => <<"t/a">>}})),
Sql2 = "foreach payload.sensors "
"from \"t/#\" ",
?assertMatch({ok,[]}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> => <<"{\"sensors\": 1}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql2,
+ context =>
+ #{payload => <<"{\"sensors\": 1}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_foreach_6(_Config) ->
%% Verify foreach on a empty-list or non-list variable
@@ -1706,10 +878,10 @@ t_sqlparse_foreach_6(_Config) ->
"do item.id as zid, timestamp as t "
"from \"t/#\" ",
{ok, Res} = emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql,
+ context =>
+ #{payload => <<"[{\"id\": 5},{\"id\": 15}]">>,
+ topic => <<"t/a">>}}),
[#{<<"t">> := Ts1, <<"zid">> := Zid1},
#{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res,
?assertEqual(true, is_integer(Ts1)),
@@ -1727,10 +899,10 @@ t_sqlparse_foreach_7(_Config) ->
Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": {\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>,
?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}, #{<<"msg_type">> := <<"2">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => Payload,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context =>
+ #{payload => Payload,
+ topic => <<"t/a">>}})),
Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info "
"do info.cmd as msg_type, info.name as name "
"incase is_not_null(info.cmd) "
@@ -1738,10 +910,10 @@ t_sqlparse_foreach_7(_Config) ->
"where s.page = '3' ",
?assertMatch({error, nomatch},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> =>
- #{<<"payload">> => Payload,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql2,
+ context =>
+ #{payload => Payload,
+ topic => <<"t/a">>}})).
t_sqlparse_foreach_8(_Config) ->
%% Verify foreach-do-incase and cascaded AS
@@ -1753,10 +925,10 @@ t_sqlparse_foreach_8(_Config) ->
Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": {\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>,
?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> =>
- #{<<"payload">> => Payload,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context =>
+ #{payload => Payload,
+ topic => <<"t/a">>}})),
Sql3 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, sublist(2,1,c.info) as info "
"do info.cmd as msg_type, info.name as name "
@@ -1764,10 +936,10 @@ t_sqlparse_foreach_8(_Config) ->
"where s.page = '2' ",
[?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => SqlN,
- <<"ctx">> =>
- #{<<"payload">> => Payload,
- <<"topic">> => <<"t/a">>}}))
+ #{sql => SqlN,
+ context =>
+ #{payload => Payload,
+ topic => <<"t/a">>}}))
|| SqlN <- [Sql3]].
t_sqlparse_case_when_1(_Config) ->
@@ -1779,25 +951,25 @@ t_sqlparse_case_when_1(_Config) ->
" end as y "
"from \"t/#\" ",
?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 0}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": -1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 7}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 8}">>,
+ topic => <<"t/a">>}})),
ok.
t_sqlparse_case_when_2(_Config) ->
@@ -1809,25 +981,25 @@ t_sqlparse_case_when_2(_Config) ->
" end as y "
"from \"t/#\" ",
?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 2}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 4}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 7}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 8}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_case_when_3(_Config) ->
%% case-when clause
@@ -1837,29 +1009,29 @@ t_sqlparse_case_when_3(_Config) ->
" end as y "
"from \"t/#\" ",
?assertMatch({ok, #{}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 5}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 0}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": -1}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 7}">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 8}">>,
+ topic => <<"t/a">>}})),
ok.
t_sqlparse_array_index_1(_Config) ->
@@ -1869,38 +1041,38 @@ t_sqlparse_array_index_1(_Config) ->
" p[1] as a "
"from \"t/#\" ",
?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"[{\"x\": 1}]">>,
+ topic => <<"t/a">>}})),
?assertMatch({ok, #{}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql,
+ context => #{payload => <<"{\"x\": 1}">>,
+ topic => <<"t/a">>}})),
%% index get without 'as'
Sql2 = "select "
" payload.x[2] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]},
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql2,
+ context => #{payload => #{<<"x">> => [1,3,4]},
+ topic => <<"t/a">>}})),
%% index get without 'as' again
Sql3 = "select "
" payload.x[2].y "
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]},
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql3,
+ context => #{payload => #{<<"x">> => [1,#{y => 3},4]},
+ topic => <<"t/a">>}})),
%% index get with 'as'
Sql4 = "select "
" payload.x[2].y as b "
"from \"t/#\" ",
?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql4,
- <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]},
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql4,
+ context => #{payload => #{<<"x">> => [1,#{y => 3},4]},
+ topic => <<"t/a">>}})).
t_sqlparse_array_index_2(_Config) ->
%% array get with negative index
@@ -1908,9 +1080,9 @@ t_sqlparse_array_index_2(_Config) ->
" payload.x[-2].y as b "
"from \"t/#\" ",
?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]},
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql1,
+ context => #{payload => #{<<"x">> => [1,#{y => 3},4]},
+ topic => <<"t/a">>}})),
%% array append to head or tail of a list:
Sql2 = "select "
" payload.x as b, "
@@ -1919,9 +1091,9 @@ t_sqlparse_array_index_2(_Config) ->
" b as c[0] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0},
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql2,
+ context => #{payload => #{<<"x">> => 0},
+ topic => <<"t/a">>}})),
%% construct an empty list:
Sql3 = "select "
" [] as c, "
@@ -1930,9 +1102,9 @@ t_sqlparse_array_index_2(_Config) ->
" 0 as c[0] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql3,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql3,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}})),
%% construct a list:
Sql4 = "select "
" [payload.a, \"topic\", 'c'] as c, "
@@ -1941,9 +1113,9 @@ t_sqlparse_array_index_2(_Config) ->
" 0 as c[0] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql4,
- <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>,
- <<"topic">> => <<"t/a">>
+ #{sql => Sql4,
+ context => #{payload => <<"{\"a\":11}">>,
+ topic => <<"t/a">>
}})).
t_sqlparse_array_index_3(_Config) ->
@@ -1954,25 +1126,25 @@ t_sqlparse_array_index_3(_Config) ->
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql0,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql0,
+ context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
+ topic => <<"t/a">>}})),
%% same as above but don't select payload:
Sql1 = "select "
"payload.x[2].y as b "
"from \"t/#\" ",
?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql1,
+ context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
+ topic => <<"t/a">>}})),
%% same as above but add 'as' clause:
Sql2 = "select "
"payload.x[2].y as b.c "
"from \"t/#\" ",
?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql2,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql2,
+ context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_array_index_4(_Config) ->
%% array with json string payload:
@@ -1981,9 +1153,9 @@ t_sqlparse_array_index_4(_Config) ->
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql0,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql0,
+ context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
+ topic => <<"t/a">>}})),
%% array with json string payload, and also select payload.x:
Sql1 = "select "
"payload.x, "
@@ -1991,9 +1163,9 @@ t_sqlparse_array_index_4(_Config) ->
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql1,
+ context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_array_index_5(_Config) ->
Sql00 = "select "
@@ -2001,9 +1173,9 @@ t_sqlparse_array_index_5(_Config) ->
"from \"t/#\" ",
{ok, Res00} =
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}}),
?assert(lists:any(fun({_K, V}) ->
V =:= [1,2,3,4]
end, maps:to_list(Res00))).
@@ -2015,17 +1187,17 @@ t_sqlparse_select_matadata_1(_Config) ->
"from \"t/#\" ",
?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql0,
- <<"ctx">> => #{<<"payload">> => <<"abc">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql0,
+ context => #{payload => <<"abc">>,
+ topic => <<"t/a">>}})),
Sql1 = "select "
"payload, metadata "
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> => #{<<"payload">> => <<"abc">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql1,
+ context => #{payload => <<"abc">>,
+ topic => <<"t/a">>}})).
t_sqlparse_array_range_1(_Config) ->
%% get a range of list
@@ -2033,19 +1205,19 @@ t_sqlparse_array_range_1(_Config) ->
" payload.a[1..4] as c "
"from \"t/#\" ",
?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql0,
- <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql0,
+ context => #{payload => <<"{\"a\":[0,1,2,3,4,5]}">>,
+ topic => <<"t/a">>}})),
%% get a range from non-list data
Sql02 = "select "
" payload.a[1..4] as c "
"from \"t/#\" ",
?assertThrow({select_and_transform_error, {error,{range_get,non_list_data},_}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql02,
- <<"ctx">> =>
- #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>,
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql02,
+ context =>
+ #{payload => <<"{\"x\":[0,1,2,3,4,5]}">>,
+ topic => <<"t/a">>}})),
%% construct a range:
Sql1 = "select "
" [1..4] as c, "
@@ -2054,9 +1226,9 @@ t_sqlparse_array_range_1(_Config) ->
" 0 as c[0] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql1,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql1,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}})).
t_sqlparse_array_range_2(_Config) ->
%% construct a range without 'as'
@@ -2065,9 +1237,9 @@ t_sqlparse_array_range_2(_Config) ->
"from \"t/#\" ",
{ok, Res00} =
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}}),
?assert(lists:any(fun({_K, V}) ->
V =:= [1,2,3,4]
end, maps:to_list(Res00))),
@@ -2077,17 +1249,17 @@ t_sqlparse_array_range_2(_Config) ->
"from \"t/#\" ",
?assertMatch({ok, #{<<"a">> := [2,3,4]}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql01,
- <<"ctx">> => #{<<"a">> => [1,2,3,4,5],
- <<"topic">> => <<"t/a">>}})),
+ #{sql => Sql01,
+ context => #{<<"a">> => [1,2,3,4,5],
+ topic => <<"t/a">>}})),
%% get a range of list without 'as'
Sql02 = "select "
" payload.a[1..4] "
"from \"t/#\" ",
?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql02,
- <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>,
- <<"topic">> => <<"t/a">>}})).
+ #{sql => Sql02,
+ context => #{payload => <<"{\"a\":[0,1,2,3,4,5]}">>,
+ topic => <<"t/a">>}})).
t_sqlparse_true_false(_Config) ->
%% construct a range without 'as'
@@ -2097,9 +1269,9 @@ t_sqlparse_true_false(_Config) ->
"from \"t/#\" ",
{ok, Res00} =
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}}),
?assertMatch(#{<<"a">> := true, <<"b">> := false,
<<"x">> := #{<<"y">> := false},
<<"c">> := [true]
@@ -2113,9 +1285,9 @@ t_sqlparse_new_map(_Config) ->
"from \"t/#\" ",
{ok, Res00} =
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => <<"">>,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => <<"">>,
+ topic => <<"t/a">>}}),
?assertMatch(#{<<"a">> := #{}, <<"b">> := #{},
<<"x">> := #{<<"y">> := #{}},
<<"c">> := [#{}]
@@ -2129,9 +1301,9 @@ t_sqlparse_payload_as(_Config) ->
"FROM \"t/#\" ",
Payload1 = <<"{ \"msgId\": 1002, \"params\": { \"convertTemp\": 20, \"engineSpeed\": 42, \"hydOilTem\": 30 } }">>,
{ok, Res01} = emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => Payload1,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => Payload1,
+ topic => <<"t/a">>}}),
?assertMatch(#{
<<"payload">> := #{
<<"params">> := #{
@@ -2145,9 +1317,9 @@ t_sqlparse_payload_as(_Config) ->
Payload2 = <<"{ \"msgId\": 1002, \"params\": { \"convertTemp\": 20, \"engineSpeed\": 42 } }">>,
{ok, Res02} = emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql00,
- <<"ctx">> => #{<<"payload">> => Payload2,
- <<"topic">> => <<"t/a">>}}),
+ #{sql => Sql00,
+ context => #{payload => Payload2,
+ topic => <<"t/a">>}}),
?assertMatch(#{
<<"payload">> := #{
<<"params">> := #{
@@ -2164,139 +1336,53 @@ t_sqlparse_nested_get(_Config) ->
"from \"t/#\" ",
?assertMatch({ok,#{<<"c">> := 0}},
emqx_rule_sqltester:test(
- #{<<"rawsql">> => Sql,
- <<"ctx">> => #{
- <<"topic">> => <<"t/1">>,
- <<"payload">> => <<"{\"a\": {\"b\": 0}}">>
+ #{sql => Sql,
+ context => #{
+ topic => <<"t/1">>,
+ payload => <<"{\"a\": {\"b\": 0}}">>
}})).
%%------------------------------------------------------------------------------
%% Internal helpers
%%------------------------------------------------------------------------------
-make_simple_rule(RuleId) when is_binary(RuleId) ->
- #rule{id = RuleId,
- rawsql = <<"select * from \"simple/topic\"">>,
- for = [<<"simple/topic">>],
- fields = [<<"*">>],
- is_foreach = false,
- conditions = {},
- actions = [{'inspect', #{}}],
- description = <<"simple rule">>}.
+republish_output(Topic) ->
+ republish_output(Topic, <<"${payload}">>).
+republish_output(Topic, Payload) ->
+ #{type => builtin, target => republish,
+ args => #{<<"payload">> => Payload, <<"topic">> => Topic, <<"qos">> => 0}}.
make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) ->
- #rule{id = RuleId,
- rawsql = <<"select * from \"simple/topic\"">>,
- for = [<<"simple/topic">>],
- fields = [<<"*">>],
- is_foreach = false,
- conditions = {},
- actions = [{'inspect', #{}}],
- created_at = Ts,
- description = <<"simple rule">>}.
+ SQL = <<"select * from \"simple/topic\"">>,
+ Topics = [<<"simple/topic">>],
+ make_simple_rule(RuleId, SQL, Topics, Ts).
-make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) ->
- #rule{id = RuleId,
- rawsql = SQL,
- for = ForTopics,
- fields = [<<"*">>],
- is_foreach = false,
- conditions = {},
- actions = [{'inspect', #{}}],
- description = <<"simple rule">>}.
+make_simple_rule(RuleId) when is_binary(RuleId) ->
+ SQL = <<"select * from \"simple/topic\"">>,
+ Topics = [<<"simple/topic">>],
+ make_simple_rule(RuleId, SQL, Topics).
-create_simple_repub_rule(TargetTopic, SQL) ->
- create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>).
+make_simple_rule(RuleId, SQL, Topics) when is_binary(RuleId) ->
+ make_simple_rule(RuleId, SQL, Topics, erlang:system_time(millisecond)).
-create_simple_repub_rule(TargetTopic, SQL, Template) ->
- {ok, Rule} = emqx_rule_engine:create_rule(
- #{rawsql => SQL,
- actions => [#{name => 'republish',
- args => #{<<"target_topic">> => TargetTopic,
- <<"target_qos">> => -1,
- <<"payload_tmpl">> => Template}
- }],
- description => <<"simple repub rule">>}),
- Rule.
+make_simple_rule(RuleId, SQL, Topics, Ts) when is_binary(RuleId) ->
+ #rule{
+ id = RuleId,
+ info = #{
+ sql => SQL,
+ from => Topics,
+ fields => [<<"*">>],
+ is_foreach => false,
+ conditions => {},
+ ouputs => [#{type => builtin, target => console}],
+ description => <<"simple rule">>
+ },
+ created_at = Ts
+ }.
-make_simple_action(ActionName) when is_atom(ActionName) ->
- #action{name = ActionName, app = ?APP,
- module = ?MODULE, on_create = simple_action_inspect, params_spec = #{},
- title = #{en => <<"Simple inspect action">>},
- description = #{en => <<"Simple inspect action">>}}.
-make_simple_action(ActionName, Hook) when is_atom(ActionName) ->
- #action{name = ActionName, app = ?APP, for = Hook,
- module = ?MODULE, on_create = simple_action_inspect, params_spec = #{},
- title = #{en => <<"Simple inspect action">>},
- description = #{en => <<"Simple inspect action with hook">>}}.
-
-simple_action_inspect(Params) ->
- fun(Data) ->
- io:format("Action InputData: ~p, Action InitParams: ~p~n", [Data, Params])
- end.
-
-make_simple_resource(ResId) ->
- #resource{id = ResId,
- type = simple_resource_type,
- config = #{},
- description = <<"Simple Resource">>}.
-
-make_simple_resource_type(ResTypeName) ->
- #resource_type{name = ResTypeName, provider = ?APP,
- params_spec = #{},
- on_create = {?MODULE, on_simple_resource_type_create},
- on_destroy = {?MODULE, on_simple_resource_type_destroy},
- on_status = {?MODULE, on_simple_resource_type_status},
- title = #{en => <<"Simple Resource Type">>},
- description = #{en => <<"Simple Resource Type">>}}.
-
-on_simple_resource_type_create(_Id, #{}) -> #{}.
-on_simple_resource_type_destroy(_Id, #{}) -> ok.
-on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}.
-
-hook_metrics_action(_Id, _Params) ->
- fun(Data = #{event := EventName}, _Envs) ->
- ct:pal("applying hook_metrics_action: ~p", [Data]),
- ets:insert(events_record_tab, {EventName, Data})
- end.
-
-mfa_action(Id, _Params) ->
- persistent_term:put(Id, 0),
- {?MODULE, mfa_action_do, [Id]}.
-
-mfa_action_do(_Data, _Envs, K) ->
- persistent_term:put(K, 1).
-
-failure_action(_Id, _Params) ->
- fun(Data, _Envs) ->
- ct:pal("applying crash action, Data: ~p", [Data]),
- {badact, intentional_failure}
- end.
-
-crash_action(_Id, _Params) ->
- fun(Data, _Envs) ->
- ct:pal("applying crash action, Data: ~p", [Data]),
- error(crash)
- end.
-
-simple_action_2_create(_Id, _Params) ->
- ets:insert(simple_action_2, {created, erlang:timestamp()}),
- fun(_Data, _Envs) -> ok end.
-
-simple_action_2_destroy(_Id, _Params) ->
- ets:insert(simple_action_2, {destroyed, erlang:timestamp()}),
- fun(_Data, _Envs) -> ok end.
-
-init_plus_by_one_action() ->
- ets:new(plus_by_one_action, [named_table, set, public]),
- ets:insert(plus_by_one_action, {num, 0}).
-
-plus_by_one_action(_Id, #{}) ->
- fun(Data, _Envs) ->
- ct:pal("applying plus_by_one_action, Data: ~p", [Data]),
- Num = ets:lookup_element(plus_by_one_action, num, 2),
- ets:insert(plus_by_one_action, {num, Num + 1})
- end.
+output_record_triggered_events(Data = #{event := EventName}, _Envs, _Args) ->
+ ct:pal("applying output_record_triggered_events: ~p", [Data]),
+ ets:insert(events_record_tab, {EventName, Data}).
verify_event(EventName) ->
ct:sleep(50),
@@ -2561,74 +1647,3 @@ deps_path(App, RelativePath) ->
local_path(RelativePath) ->
deps_path(emqx_rule_engine, RelativePath).
-set_special_configs(emqx_rule_engine) ->
- application:set_env(emqx_rule_engine, ignore_sys_message, true),
- application:set_env(emqx_rule_engine, events,
- [{'client.connected',on,1},
- {'client.disconnected',on,1},
- {'session.subscribed',on,1},
- {'session.unsubscribed',on,1},
- {'message.acked',on,1},
- {'message.dropped',on,1},
- {'message.delivered',on,1}
- ]),
- ok;
-set_special_configs(_App) ->
- ok.
-
-mock_print() ->
- catch meck:unload(emqx_ctl),
- meck:new(emqx_ctl, [non_strict, passthrough]),
- meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end),
- meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end),
- meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end),
- meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end).
-
-unmock_print() ->
- meck:unload(emqx_ctl).
-
-t_load_providers(_) ->
- error('TODO').
-
-t_unload_providers(_) ->
- error('TODO').
-
-t_delete_rule(_) ->
- error('TODO').
-
-t_start_resource(_) ->
- error('TODO').
-
-t_test_resource(_) ->
- error('TODO').
-
-t_get_resource_status(_) ->
- error('TODO').
-
-t_get_resource_params(_) ->
- error('TODO').
-
-t_delete_resource(_) ->
- error('TODO').
-
-t_refresh_resources(_) ->
- error('TODO').
-
-t_refresh_rules(_) ->
- error('TODO').
-
-t_refresh_resource_status(_) ->
- error('TODO').
-
-t_init_resource(_) ->
- error('TODO').
-
-t_init_action(_) ->
- error('TODO').
-
-t_clear_resource(_) ->
- error('TODO').
-
-t_clear_action(_) ->
- error('TODO').
-
diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl
index e7c543c91..ff654ba94 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl
@@ -31,11 +31,8 @@ suite() ->
groups() ->
[{metrics, [sequence],
- [ t_action
- , t_rule
- , t_clear
+ [ t_rule
, t_no_creation_1
- , t_no_creation_2
]},
{speed, [sequence],
[ rule_speed
@@ -55,59 +52,27 @@ end_per_suite(_Config) ->
init_per_testcase(_, Config) ->
catch emqx_rule_metrics:stop(),
{ok, _} = emqx_rule_metrics:start_link(),
- [emqx_metrics:set(M, 0) || M <- emqx_rule_metrics:overall_metrics()],
Config.
end_per_testcase(_, _Config) ->
ok.
t_no_creation_1(_) ->
- ?assertEqual(ok, emqx_rule_metrics:inc_rules_matched(<<"rule1">>)).
-
-t_no_creation_2(_) ->
- ?assertEqual(ok, emqx_rule_metrics:inc_actions_taken(<<"action:0">>)).
-
-t_action(_) ->
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)),
- ?assertEqual(0, emqx_rule_metrics:get_actions_exception(<<"action:1">>)),
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:2">>)),
- ok = emqx_rule_metrics:create_metrics(<<"action:1">>),
- ok = emqx_rule_metrics:create_metrics(<<"action:2">>),
- ok = emqx_rule_metrics:inc_actions_taken(<<"action:1">>),
- ok = emqx_rule_metrics:inc_actions_exception(<<"action:1">>),
- ok = emqx_rule_metrics:inc_actions_taken(<<"action:2">>),
- ok = emqx_rule_metrics:inc_actions_taken(<<"action:2">>),
- ?assertEqual(1, emqx_rule_metrics:get_actions_taken(<<"action:1">>)),
- ?assertEqual(1, emqx_rule_metrics:get_actions_exception(<<"action:1">>)),
- ?assertEqual(2, emqx_rule_metrics:get_actions_taken(<<"action:2">>)),
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:3">>)),
- ?assertEqual(3, emqx_rule_metrics:get_overall('actions.taken')),
- ?assertEqual(1, emqx_rule_metrics:get_overall('actions.exception')),
- ok = emqx_rule_metrics:clear_metrics(<<"action:1">>),
- ok = emqx_rule_metrics:clear_metrics(<<"action:2">>),
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)),
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:2">>)).
+ ?assertEqual(ok, emqx_rule_metrics:inc(<<"rule1">>, 'rules.matched')).
t_rule(_) ->
- ok = emqx_rule_metrics:create_rule_metrics(<<"rule:1">>),
+ ok = emqx_rule_metrics:create_rule_metrics(<<"rule1">>),
ok = emqx_rule_metrics:create_rule_metrics(<<"rule2">>),
- ok = emqx_rule_metrics:inc(<<"rule:1">>, 'rules.matched'),
+ ok = emqx_rule_metrics:inc(<<"rule1">>, 'rules.matched'),
ok = emqx_rule_metrics:inc(<<"rule2">>, 'rules.matched'),
ok = emqx_rule_metrics:inc(<<"rule2">>, 'rules.matched'),
- ?assertEqual(1, emqx_rule_metrics:get(<<"rule:1">>, 'rules.matched')),
+ ct:pal("----couters: ---~p", [persistent_term:get(emqx_rule_metrics)]),
+ ?assertEqual(1, emqx_rule_metrics:get(<<"rule1">>, 'rules.matched')),
?assertEqual(2, emqx_rule_metrics:get(<<"rule2">>, 'rules.matched')),
?assertEqual(0, emqx_rule_metrics:get(<<"rule3">>, 'rules.matched')),
- ?assertEqual(3, emqx_rule_metrics:get_overall('rules.matched')),
- ok = emqx_rule_metrics:clear_rule_metrics(<<"rule:1">>),
+ ok = emqx_rule_metrics:clear_rule_metrics(<<"rule1">>),
ok = emqx_rule_metrics:clear_rule_metrics(<<"rule2">>).
-t_clear(_) ->
- ok = emqx_rule_metrics:create_metrics(<<"action:1">>),
- ok = emqx_rule_metrics:inc_actions_taken(<<"action:1">>),
- ?assertEqual(1, emqx_rule_metrics:get_actions_taken(<<"action:1">>)),
- ok = emqx_rule_metrics:clear_metrics(<<"action:1">>),
- ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)).
-
rule_speed(_) ->
ok = emqx_rule_metrics:create_rule_metrics(<<"rule1">>),
ok = emqx_rule_metrics:create_rule_metrics(<<"rule:2">>),
@@ -119,51 +84,11 @@ rule_speed(_) ->
?LET(#{max := Max, current := Current}, emqx_rule_metrics:get_rule_speed(<<"rule1">>),
{?assert(Max =< 2),
?assert(Current =< 2)}),
- ct:pal("===== Speed: ~p~n", [emqx_rule_metrics:get_overall_rule_speed()]),
- ?LET(#{max := Max, current := Current}, emqx_rule_metrics:get_overall_rule_speed(),
- {?assert(Max =< 3),
- ?assert(Current =< 3)}),
ct:sleep(2100),
?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_rule_speed(<<"rule1">>),
{?assert(Max =< 2),
?assert(Current == 0),
?assert(Last5Min =< 0.67)}),
- ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_overall_rule_speed(),
- {?assert(Max =< 3),
- ?assert(Current == 0),
- ?assert(Last5Min =< 1)}),
ct:sleep(3000),
- ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_overall_rule_speed(),
- {?assert(Max =< 3),
- ?assert(Current == 0),
- ?assert(Last5Min == 0)}),
ok = emqx_rule_metrics:clear_rule_metrics(<<"rule1">>),
ok = emqx_rule_metrics:clear_rule_metrics(<<"rule:2">>).
-
-% t_create(_) ->
-% error('TODO').
-
-% t_get(_) ->
-% error('TODO').
-
-% t_get_overall(_) ->
-% error('TODO').
-
-% t_get_rule_speed(_) ->
-% error('TODO').
-
-% t_get_overall_rule_speed(_) ->
-% error('TODO').
-
-% t_get_rule_metrics(_) ->
-% error('TODO').
-
-% t_get_action_metrics(_) ->
-% error('TODO').
-
-% t_inc(_) ->
-% error('TODO').
-
-% t_overall_metrics(_) ->
-% error('TODO').
-
diff --git a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl
deleted file mode 100644
index 62f538f43..000000000
--- a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl
+++ /dev/null
@@ -1,109 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_monitor_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("emqx_rule_engine/include/rule_engine.hrl").
--include_lib("emqx/include/emqx.hrl").
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
-all() ->
- [ {group, resource}
- ].
-
-suite() ->
- [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}].
-
-groups() ->
- [{resource, [sequence],
- [ t_restart_resource
- ]}
- ].
-
-init_per_suite(Config) ->
- application:load(emqx_machine),
- ok = ekka_mnesia:start(),
- ok = emqx_rule_registry:mnesia(boot),
- Config.
-
-end_per_suite(_Config) ->
- ok.
-
-init_per_testcase(t_restart_resource, Config) ->
- Opts = [public, named_table, set, {read_concurrency, true}],
- _ = ets:new(?RES_PARAMS_TAB, [{keypos, #resource_params.id}|Opts]),
- ets:new(t_restart_resource, [named_table, public]),
- ets:insert(t_restart_resource, {failed_count, 0}),
- ets:insert(t_restart_resource, {succ_count, 0}),
- Config;
-
-init_per_testcase(_, Config) ->
- Config.
-
-end_per_testcase(t_restart_resource, Config) ->
- ets:delete(t_restart_resource),
- Config;
-end_per_testcase(_, Config) ->
- Config.
-
-t_restart_resource(_) ->
- {ok, _} = emqx_rule_monitor:start_link(),
- emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc,1000),
- ok = emqx_rule_registry:register_resource_types(
- [#resource_type{
- name = test_res_1,
- provider = ?APP,
- params_spec = #{},
- on_create = {?MODULE, on_resource_create},
- on_destroy = {?MODULE, on_resource_destroy},
- on_status = {?MODULE, on_get_resource_status},
- title = #{en => <<"Test Resource">>},
- description = #{en => <<"Test Resource">>}}]),
- ok = emqx_rule_engine:load_providers(),
- {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource(
- #{type => test_res_1,
- config => #{},
- description => <<"debug resource">>}),
- [{_, 1}] = ets:lookup(t_restart_resource, failed_count),
- [{_, 0}] = ets:lookup(t_restart_resource, succ_count),
- ct:pal("monitor: ~p", [whereis(emqx_rule_monitor)]),
- emqx_rule_monitor:ensure_resource_retrier(ResId, 100),
- timer:sleep(1000),
- [{_, 5}] = ets:lookup(t_restart_resource, failed_count),
- [{_, 1}] = ets:lookup(t_restart_resource, succ_count),
- #{retryers := Pids} = sys:get_state(whereis(emqx_rule_monitor)),
- ?assertEqual(0, map_size(Pids)),
- ok = emqx_rule_engine:unload_providers(),
- emqx_rule_registry:remove_resource(ResId),
- emqx_rule_monitor:stop(),
- ok.
-
-on_resource_create(Id, _) ->
- case ets:lookup(t_restart_resource, failed_count) of
- [{_, 5}] ->
- ets:insert(t_restart_resource, {succ_count, 1}),
- #{};
- [{_, N}] ->
- ets:insert(t_restart_resource, {failed_count, N+1}),
- error({incorrect_params, Id})
- end.
-on_resource_destroy(_Id, _) -> ok.
-on_get_resource_status(_Id, _) -> #{}.
diff --git a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl
deleted file mode 100644
index cbd69c878..000000000
--- a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl
+++ /dev/null
@@ -1,148 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_registry_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
-
-all() -> emqx_ct:all(?MODULE).
-
-init_per_testcase(_TestCase, Config) ->
- Config.
-
-end_per_testcase(_TestCase, Config) ->
- Config.
-
-% t_mnesia(_) ->
-% error('TODO').
-
-% t_dump(_) ->
-% error('TODO').
-
-% t_start_link(_) ->
-% error('TODO').
-
-% t_get_rules_for(_) ->
-% error('TODO').
-
-% t_add_rules(_) ->
-% error('TODO').
-
-% t_remove_rules(_) ->
-% error('TODO').
-
-% t_add_action(_) ->
-% error('TODO').
-
-% t_remove_action(_) ->
-% error('TODO').
-
-% t_remove_actions(_) ->
-% error('TODO').
-
-% t_init(_) ->
-% error('TODO').
-
-% t_handle_call(_) ->
-% error('TODO').
-
-% t_handle_cast(_) ->
-% error('TODO').
-
-% t_handle_info(_) ->
-% error('TODO').
-
-% t_terminate(_) ->
-% error('TODO').
-
-% t_code_change(_) ->
-% error('TODO').
-
-% t_get_resource_types(_) ->
-% error('TODO').
-
-% t_get_resources_by_type(_) ->
-% error('TODO').
-
-% t_get_actions_for(_) ->
-% error('TODO').
-
-% t_get_actions(_) ->
-% error('TODO').
-
-% t_get_action_instance_params(_) ->
-% error('TODO').
-
-% t_remove_action_instance_params(_) ->
-% error('TODO').
-
-% t_remove_resource_params(_) ->
-% error('TODO').
-
-% t_add_action_instance_params(_) ->
-% error('TODO').
-
-% t_add_resource_params(_) ->
-% error('TODO').
-
-% t_find_action(_) ->
-% error('TODO').
-
-% t_get_rules(_) ->
-% error('TODO').
-
-% t_get_resources(_) ->
-% error('TODO').
-
-% t_remove_resource(_) ->
-% error('TODO').
-
-% t_find_resource_params(_) ->
-% error('TODO').
-
-% t_add_resource(_) ->
-% error('TODO').
-
-% t_find_resource_type(_) ->
-% error('TODO').
-
-% t_remove_rule(_) ->
-% error('TODO').
-
-% t_add_rule(_) ->
-% error('TODO').
-
-% t_register_resource_types(_) ->
-% error('TODO').
-
-% t_add_actions(_) ->
-% error('TODO').
-
-% t_unregister_resource_types_of(_) ->
-% error('TODO').
-
-% t_remove_actions_of(_) ->
-% error('TODO').
-
-% t_get_rule(_) ->
-% error('TODO').
-
-% t_find_resource(_) ->
-% error('TODO').
-
diff --git a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl
deleted file mode 100644
index fdd7857d4..000000000
--- a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl
+++ /dev/null
@@ -1,191 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_rule_validator_SUITE).
-
--compile(nowarn_export_all).
--compile(export_all).
-
--include_lib("eunit/include/eunit.hrl").
-
--define(VALID_SPEC,
- #{
- string_required => #{
- type => string,
- required => true
- },
- string_optional_with_default => #{
- type => string,
- required => false,
- default => <<"a/b">>
- },
- string_optional_without_default_0 => #{
- type => string,
- required => false
- },
- string_optional_without_default_1 => #{
- type => string
- },
- type_number => #{
- type => number,
- required => true
- },
- type_boolean => #{
- type => boolean,
- required => true
- },
- type_enum_number => #{
- type => number,
- enum => [-1, 0, 1, 2],
- required => true
- },
- type_file => #{
- type => file,
- required => true
- },
- type_object => #{
- type => object,
- required => true,
- schema => #{
- string_required => #{
- type => string,
- required => true
- },
- type_number => #{
- type => number,
- required => true
- }
- }
- },
- type_array => #{
- type => array,
- required => true,
- items => #{
- type => string,
- required => true
- }
- }
- }).
-
-all() -> emqx_ct:all(?MODULE).
-
-t_validate_spec_the_complex(_) ->
- ok = emqx_rule_validator:validate_spec(?VALID_SPEC).
-
-t_validate_spec_invalid_1(_) ->
- ?assertThrow({required_field_missing, {type, _}},
- emqx_rule_validator:validate_spec(#{
- type_enum_number => #{
- required => true
- }
- })).
-
-t_validate_spec_invalid_2(_) ->
- ?assertThrow({required_field_missing, {schema, _}},
- emqx_rule_validator:validate_spec(#{
- type_enum_number => #{
- type => object
- }
- })).
-
-t_validate_spec_invalid_3(_) ->
- ?assertThrow({required_field_missing, {items, _}},
- emqx_rule_validator:validate_spec(#{
- type_enum_number => #{
- type => array
- }
- })).
-
-t_validate_params_0(_) ->
- Params = #{<<"eee">> => <<"eee">>},
- Specs = #{<<"eee">> => #{
- type => string,
- required => true
- }},
- ?assertEqual(Params,
- emqx_rule_validator:validate_params(Params, Specs)).
-
-t_validate_params_1(_) ->
- Params = #{<<"eee">> => 1},
- Specs = #{<<"eee">> => #{
- type => string,
- required => true
- }},
- ?assertThrow({invalid_data_type, {string, 1}},
- emqx_rule_validator:validate_params(Params, Specs)).
-
-t_validate_params_2(_) ->
- ?assertThrow({required_field_missing, <<"eee">>},
- emqx_rule_validator:validate_params(
- #{<<"abc">> => 1},
- #{<<"eee">> => #{
- type => string,
- required => true
- }})).
-
-t_validate_params_format(_) ->
- Params = #{<<"eee">> => <<"abc">>},
- Params1 = #{<<"eee">> => <<"http://abc:8080">>},
- Params2 = #{<<"eee">> => <<"http://abc">>},
- Specs = #{<<"eee">> => #{
- type => string,
- format => url,
- required => true
- }},
- ?assertThrow({invalid_data_type, {string, <<"abc">>}},
- emqx_rule_validator:validate_params(Params, Specs)),
- ?assertEqual(Params1,
- emqx_rule_validator:validate_params(Params1, Specs)),
- ?assertEqual(Params2,
- emqx_rule_validator:validate_params(Params2, Specs)).
-
-t_validate_params_fill_default(_) ->
- Params = #{<<"abc">> => 1},
- Specs = #{<<"eee">> => #{
- type => string,
- required => false,
- default => <<"hello">>
- }},
- ?assertMatch(#{<<"abc">> := 1, <<"eee">> := <<"hello">>},
- emqx_rule_validator:validate_params(Params, Specs)).
-
-t_validate_params_the_complex(_) ->
- Params = #{
- <<"string_required">> => <<"hello">>,
- <<"type_number">> => 1,
- <<"type_boolean">> => true,
- <<"type_enum_number">> => 2,
- <<"type_file">> => <<"">>,
- <<"type_object">> => #{
- <<"string_required">> => <<"hello2">>,
- <<"type_number">> => 1.3
- },
- <<"type_array">> => [<<"ok">>, <<"no">>]
- },
- ?assertMatch(
- #{ <<"string_required">> := <<"hello">>,
- <<"string_optional_with_default">> := <<"a/b">>,
- <<"type_number">> := 1,
- <<"type_boolean">> := true,
- <<"type_enum_number">> := 2,
- <<"type_file">> := <<"">>,
- <<"type_object">> := #{
- <<"string_required">> := <<"hello2">>,
- <<"type_number">> := 1.3
- },
- <<"type_array">> := [<<"ok">>, <<"no">>]
- },
- emqx_rule_validator:validate_params(Params, ?VALID_SPEC)).
diff --git a/bin/emqx b/bin/emqx
index 6c77afd1b..1be7996d1 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -488,6 +488,8 @@ case "$1" in
logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
# log to user console
echoerr "stop failed, $msg"
+ echo "ERROR: $PID is still around"
+ ps -p "$PID"
exit 1
fi
logger -t "${REL_NAME}[${PID}]" "STOP: OK"
diff --git a/bin/emqx.cmd b/bin/emqx.cmd
index 768e30d2c..fe0d474c9 100644
--- a/bin/emqx.cmd
+++ b/bin/emqx.cmd
@@ -22,14 +22,19 @@
@set script=%~n0
+@set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka
+@set ERL_FLAGS=%EPMD_ARG%
+
:: Discover the release root directory from the directory
:: of this script
@set script_dir=%~dp0
@for %%A in ("%script_dir%\..") do @(
set rel_root_dir=%%~fA
)
+
@set rel_dir=%rel_root_dir%\releases\%rel_vsn%
@set RUNNER_ROOT_DIR=%rel_root_dir%
+@set RUNNER_ETC_DIR=%rel_root_dir%\etc
@set etc_dir=%rel_root_dir%\etc
@set lib_dir=%rel_root_dir%\lib
@@ -46,22 +51,22 @@
@set progname=erl.exe
@set clean_boot_script=%rel_root_dir%\bin\start_clean
@set erlsrv="%bindir%\erlsrv.exe"
-@set epmd="%bindir%\epmd.exe"
@set escript="%bindir%\escript.exe"
@set werl="%bindir%\werl.exe"
@set erl_exe="%bindir%\erl.exe"
@set nodetool="%rel_root_dir%\bin\nodetool"
@set cuttlefish="%rel_root_dir%\bin\cuttlefish"
@set node_type="-name"
+@set schema_mod="emqx_machine_schema"
:: Extract node name from emqx.conf
-@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @(
+@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.name"`) do @(
@call :set_trim node_name %%I
)
:: Extract node cookie from emqx.conf
-@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @(
- @call :set_trim node_cookie= %%I
+@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.cookie"`) do @(
+ @call :set_trim node_cookie %%I
)
:: Write the erl.ini file to set up paths relative to this script
@@ -139,13 +144,23 @@
)
@goto :eof
-:generate_app_config
-@set gen_config_cmd=%escript% %cuttlefish% -i %rel_dir%\emqx.schema -c %etc_dir%\emqx.conf -d %data_dir%\configs generate
-@for /f "delims=" %%A in ('%%gen_config_cmd%%') do @(
- set generated_config_args=%%A
+:: get the current time with hocon
+:get_cur_time
+@for /f "usebackq tokens=1-6 delims=." %%a in (`"%escript% %nodetool% hocon now_time"`) do @(
+ set now_time=%%a.%%b.%%c.%%d.%%e.%%f
)
@goto :eof
+:generate_app_config
+@call :get_cur_time
+%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate
+@set generated_config_args=-config %data_dir%\configs\app.%now_time%.config -args_file %data_dir%\configs\vm.%now_time%.args
+:: create one new line
+@echo.>>%data_dir%\configs\vm.%now_time%.args
+:: write the node type and node name in to vm args file
+@echo %node_type% %node_name%>>%data_dir%\configs\vm.%now_time%.args
+@goto :eof
+
:: set boot_script variable
:set_boot_script_var
@if exist "%rel_dir%\%rel_name%.boot" (
@@ -188,13 +203,11 @@
:: relup and reldown
goto relup
)
-
@goto :eof
:: Uninstall the Windows service
:uninstall
@%erlsrv% remove %service_name%
-@%epmd% -kill
@goto :eof
:: Start the Windows service
@@ -207,7 +220,7 @@
@echo off
cd /d %rel_root_dir%
@echo on
-@start "%rel_name%" %werl% -boot "%boot_script%" %args%
+@start "%rel_name%" %werl% -boot "%boot_script%" -mode embedded %args%
@goto :eof
:: Stop the Windows service
@@ -237,7 +250,7 @@ cd /d %rel_root_dir%
@echo off
cd /d %rel_root_dir%
@echo on
-@start "bin\%rel_name% console" %werl% -boot "%boot_script%" %args%
+@start "bin\%rel_name% console" %werl% -boot "%boot_script%" -mode embedded %args%
@echo emqx is started!
@goto :eof
@@ -262,4 +275,3 @@ cd /d %rel_root_dir%
:set_trim
@set %1=%2
@goto :eof
-
diff --git a/build b/build
index 36eb4d129..22d1f8960 100755
--- a/build
+++ b/build
@@ -16,7 +16,9 @@ PKG_VSN="$(./pkg-vsn.sh)"
export PKG_VSN
if [ "$(uname -s)" = 'Darwin' ]; then
- SYSTEM=macos
+ DIST='macos'
+ VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
+ SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'centos' /etc/*-release; then
DIST='centos'
diff --git a/rebar.config b/rebar.config
index 9b374d107..ca8dd3e22 100644
--- a/rebar.config
+++ b/rebar.config
@@ -18,8 +18,7 @@
%% Check for the mnesia calls forbidden by Ekka:
{xref_queries,
- [ {"E || \"mnesia\":\"dirty_write\"/\".*\" : Fun", [{{emqx_dashboard_collection,flush,2},{mnesia,dirty_write,2}}]}
- , {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []}
+ [ {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []}
, {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []}
, {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []}
, {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []}
@@ -49,15 +48,15 @@
, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}}
- , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
+ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
- , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.4"}}}
+ , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.5"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
, {replayq, "0.3.3"}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}
- , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}}
+ , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
, {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}
diff --git a/rebar.config.erl b/rebar.config.erl
index 4888fafe0..a5386df6b 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -272,7 +272,6 @@ relx_apps(ReleaseType) ->
, emqx_exhook
, emqx_bridge
, emqx_rule_engine
- , emqx_rule_actions
, emqx_modules
, emqx_management
, emqx_dashboard