yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11243 from id/0710-prep-e5.1.1-alpha.1 

prepare for e5.1.1 alpha.1
This commit is contained in:
Ivan Dyachkov 2023-07-10 20:30:48 +02:00 committed by GitHub
parent bb5b2d4bda fb96c1e20d
commit 4d1c7652ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
280 changed files with 6475 additions and 2829 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
.ci/docker-compose-file/docker-compose-hstreamdb.yaml Normal file
View File

@ -0,0 +1,123 @@
version: "3.5"
services:
hserver:
image: hstreamdb/hstream:v0.15.0
container_name: hstreamdb
depends_on:
- zookeeper
- hstore
# ports:
# - "127.0.0.1:6570:6570"
expose:
- 6570
networks:
- emqx_bridge
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /tmp:/tmp
- data_store:/data/store
command:
- bash
- "-c"
- |
set -e
/usr/local/script/wait-for-storage.sh hstore 6440 zookeeper 2181 600 \
/usr/local/bin/hstream-server \
--bind-address 0.0.0.0 --port 6570 \
--internal-port 6571 \
--server-id 100 \
--seed-nodes "$$(hostname -I | awk '{print $$1}'):6571" \
--advertised-address $$(hostname -I | awk '{print $$1}') \
--metastore-uri zk://zookeeper:2181 \
--store-config /data/store/logdevice.conf \
--store-admin-host hstore --store-admin-port 6440 \
--store-log-level warning \
--io-tasks-path /tmp/io/tasks \
--io-tasks-network emqx_bridge
hstore:
image: hstreamdb/hstream:v0.15.0
networks:
- emqx_bridge
volumes:
- data_store:/data/store
command:
- bash
- "-c"
- |
set -ex
# N.B. "enable-dscp-reflection=false" is required for linux kernel which
# doesn't support dscp reflection, e.g. centos7.
/usr/local/bin/ld-dev-cluster --root /data/store \
--use-tcp --tcp-host $$(hostname -I | awk '{print $$1}') \
--user-admin-port 6440 \
--param enable-dscp-reflection=false \
--no-interactive
zookeeper:
image: zookeeper
expose:
- 2181
networks:
- emqx_bridge
volumes:
- data_zk_data:/data
- data_zk_datalog:/datalog
## The three container `hstream-exporter`, `prometheus`, `console`
## is for HStreamDB Web Console
## But HStreamDB Console is not supported in v0.15.0
## because of HStreamApi proto changed
# hstream-exporter:
# depends_on:
# hserver:
# condition: service_completed_successfully
# image: hstreamdb/hstream-exporter
# networks:
# - hstream-quickstart
# command:
# - bash
# - "-c"
# - |
# set -ex
# hstream-exporter --addr hstream://hserver:6570
# prometheus:
# image: prom/prometheus
# expose:
# - 9097
# networks:
# - hstream-quickstart
# ports:
# - "9097:9090"
# volumes:
# - $PWD/prometheus:/etc/prometheus
# console:
# image: hstreamdb/hstream-console
# depends_on:
# - hserver
# expose:
# - 5177
# networks:
# - hstream-quickstart
# environment:
# - SERVER_PORT=5177
# - PROMETHEUS_URL=http://prometheus:9097
# - HSTREAM_PUBLIC_ADDRESS=hstream.example.com
# - HSTREAM_PRIVATE_ADDRESS=hserver:6570
# ports:
# - "5177:5177"
# networks:
# hstream-quickstart:
# name: hstream-quickstart
volumes:
data_store:
name: quickstart_data_store
data_zk_data:
name: quickstart_data_zk_data
data_zk_datalog:
name: quickstart_data_zk_datalog

4
.ci/docker-compose-file/docker-compose-toxiproxy.yaml
View File

@ -43,10 +43,12 @@ services:
- 19000:19000
# S3 TLS
- 19100:19100
# IOTDB
# IOTDB (3 total)
- 14242:4242
- 28080:18080
- 38080:38080
# HStreamDB
- 15670:5670
command:
- "-host=0.0.0.0"
- "-config=/config/toxiproxy.json"

6
.ci/docker-compose-file/toxiproxy.json
View File

@ -155,5 +155,11 @@
"listen": "0.0.0.0:8085",
"upstream": "gcp_emulator:8085",
"enabled": true
},
{
"name": "hstreamdb",
"listen": "0.0.0.0:6570",
"upstream": "hstreamdb:6570",
"enabled": true
}
]

1
.github/actions/package-macos/action.yaml vendored
View File

@ -33,7 +33,6 @@ runs:
HOMEBREW_NO_INSTALL_UPGRADE: 1
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
run: |
brew update
brew install curl zip unzip coreutils openssl@1.1
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH

2
.github/pull_request_template.md vendored
View File

@ -10,7 +10,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev
- [ ] Added tests for the changes
- [ ] Changed lines covered in coverage report
- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-<PR-id>.en.md` files
- [ ] Change log has been added to `changes/(ce|ee)/(feat|perf|fix)-<PR-id>.en.md` files
- [ ] For internal contributor: there is a jira ticket to track this change
- [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up
- [ ] Schema changes are backward compatible

1
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -182,6 +182,7 @@ jobs:
images: |
${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
flavor: |
latest=${{ matrix.elixir == 'no_elixir' }}
suffix=${{ steps.pre-meta.outputs.img_suffix }}
tags: |
type=semver,pattern={{major}}.{{minor}},value=${{ needs.prepare.outputs.VERSION }}

18
.github/workflows/release.yaml vendored
View File

@ -101,3 +101,21 @@ jobs:
push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm"
push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm"
push "el/9" "packages/$PROFILE-$VERSION-el9-arm64.rpm"
rerun-apps-version-check:
runs-on: ubuntu-22.04
if: github.repository_owner == 'emqx' && github.event_name == 'release'
needs:
- upload
permissions:
pull-requests: read
checks: read
actions: write
steps:
- uses: actions/checkout@v3
- name: trigger re-run of app versions check on open PRs
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
python3 scripts/rerun-apps-version-check.py

12
CONTRIBUTING.md
View File

@ -2,7 +2,6 @@
You are welcome to submit any bugs, issues and feature requests on this repository.
## Commit Message Guidelines
We have very precise rules over how our git commit messages can be formatted. This leads to **more readable messages** that are easy to follow when looking through the **project history**.
@ -80,3 +79,14 @@ Just as in the **subject**, use the imperative, present tense: "change" not "cha
The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**.
**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this.
## Changelog
Changes affecting EMQX functionality shall be described in a separate markdown file under `changes` directory.
File name pattern: `changes/(ce|ee)/(feat|perf|fix)-<PR-id>.en.md`, where:
- `ce,ee`: Indicates whether given change affects community and enterprise edition (`ce`), or enterprise edition only (`ee`); for any change only one file is needed as enterprise edition absorbs all changes from the community edition automatically. When in doubts, one could consult [documentation](https://www.emqx.io/docs/en/latest/). Enterprise features have a corresponding "Tip" banner, see for example [here](https://www.emqx.io/docs/en/v5.1/data-integration/data-bridge-influxdb.html).
- `feat|perf|fix`: Whether the change is a new functionality (`feat`), performance improvement (`perf`), or a bug fix (`fix`).
- `PR-id`: Github pull request id. Since pull request id cannot be known before the PR is actually created, it's common to add change log entry in a separate commit.
- `en`: ISO 639-1 language code indicating the language the change log entry is written in. Right now we are only accepting entries in English.

2
README.md
View File

@ -10,7 +10,7 @@
[![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
EMQX is the world's most scalable open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency.
EMQX is the world's most scalable open-source [MQTT broker](https://www.emqx.com/en/blog/the-ultimate-guide-to-mqtt-broker-comparison) with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency.
EMQX supports multiple open standard protocols like MQTT, HTTP, QUIC, and WebSocket. It’s 100% compliant with MQTT 5.0 and 3.x standard, and secures bi-directional communication with MQTT over TLS/SSL and various authentication mechanisms.

14
apps/emqx/include/emqx_access_control.hrl
View File

@ -18,3 +18,17 @@
-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization").
-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization).
-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY, <<"authorization">>).
-define(DEFAULT_ACTION_QOS, 0).
-define(DEFAULT_ACTION_RETAIN, false).
-define(AUTHZ_SUBSCRIBE(QOS), #{action_type => subscribe, qos => QOS}).
-define(AUTHZ_SUBSCRIBE, ?AUTHZ_SUBSCRIBE(?DEFAULT_ACTION_QOS)).
-define(AUTHZ_PUBLISH(QOS, RETAIN), #{action_type => publish, qos => QOS, retain => RETAIN}).
-define(AUTHZ_PUBLISH(QOS), ?AUTHZ_PUBLISH(QOS, ?DEFAULT_ACTION_RETAIN)).
-define(AUTHZ_PUBLISH, ?AUTHZ_PUBLISH(?DEFAULT_ACTION_QOS)).
-define(authz_action(PUBSUB, QOS), #{action_type := PUBSUB, qos := QOS}).
-define(authz_action(PUBSUB), ?authz_action(PUBSUB, _)).
-define(authz_action, ?authz_action(_)).

4
apps/emqx/include/emqx_placeholder.hrl
View File

@ -21,7 +21,7 @@
-define(PH(Type), <<"${", Type/binary, "}">>).
%% action: publish/subscribe/all
%% action: publish/subscribe
-define(PH_ACTION, <<"${action}">>).
%% cert
@ -79,6 +79,7 @@
-define(PH_REASON, <<"${reason}">>).
-define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>).
-define(PH_RETAIN, <<"${retain}">>).
%% sync change these place holder with binary def.
-define(PH_S_ACTION, "${action}").
@ -113,5 +114,6 @@
-define(PH_S_NODE, "${node}").
-define(PH_S_REASON, "${reason}").
-define(PH_S_ENDPOINT_NAME, "${endpoint_name}").
-define(PH_S_RETAIN, "${retain}").
-endif.

4
apps/emqx/include/emqx_release.hrl
View File

@ -32,10 +32,10 @@
%% `apps/emqx/src/bpapi/README.md'
%% Opensource edition
-define(EMQX_RELEASE_CE, "5.1.0").
-define(EMQX_RELEASE_CE, "5.1.1").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.1.0").
-define(EMQX_RELEASE_EE, "5.1.1-alpha.1").
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

2
apps/emqx/rebar.config
View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.5"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.11"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

2
apps/emqx/src/emqx.app.src
View File

@ -2,7 +2,7 @@
{application, emqx, [
{id, "emqx"},
{description, "EMQX Core"},
{vsn, "5.1.1"},
{vsn, "5.1.2"},
{modules, []},
{registered, []},
{applications, [

28
apps/emqx/src/emqx_access_control.erl
View File

@ -77,10 +77,10 @@ authenticate(Credential) ->
%% @doc Check Authorization
-spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) ->
allow | deny.
authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) ->
authorize(ClientInfo, Action, <<"$delayed/", Data/binary>> = RawTopic) ->
case binary:split(Data, <<"/">>) of
[_, Topic] ->
authorize(ClientInfo, PubSub, Topic);
authorize(ClientInfo, Action, Topic);
_ ->
?SLOG(warning, #{
msg => "invalid_delayed_topic_format",
@ -90,39 +90,39 @@ authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) ->
inc_authz_metrics(deny),
deny
end;
authorize(ClientInfo, PubSub, Topic) ->
authorize(ClientInfo, Action, Topic) ->
Result =
case emqx_authz_cache:is_enabled() of
true -> check_authorization_cache(ClientInfo, PubSub, Topic);
false -> do_authorize(ClientInfo, PubSub, Topic)
true -> check_authorization_cache(ClientInfo, Action, Topic);
false -> do_authorize(ClientInfo, Action, Topic)
end,
inc_authz_metrics(Result),
Result.
check_authorization_cache(ClientInfo, PubSub, Topic) ->
case emqx_authz_cache:get_authz_cache(PubSub, Topic) of
check_authorization_cache(ClientInfo, Action, Topic) ->
case emqx_authz_cache:get_authz_cache(Action, Topic) of
not_found ->
AuthzResult = do_authorize(ClientInfo, PubSub, Topic),
emqx_authz_cache:put_authz_cache(PubSub, Topic, AuthzResult),
AuthzResult = do_authorize(ClientInfo, Action, Topic),
emqx_authz_cache:put_authz_cache(Action, Topic, AuthzResult),
AuthzResult;
AuthzResult ->
emqx:run_hook(
'client.check_authz_complete',
[ClientInfo, PubSub, Topic, AuthzResult, cache]
[ClientInfo, Action, Topic, AuthzResult, cache]
),
inc_authz_metrics(cache_hit),
AuthzResult
end.
do_authorize(ClientInfo, PubSub, Topic) ->
do_authorize(ClientInfo, Action, Topic) ->
NoMatch = emqx:get_config([authorization, no_match], allow),
Default = #{result => NoMatch, from => default},
case run_hooks('client.authorize', [ClientInfo, PubSub, Topic], Default) of
case run_hooks('client.authorize', [ClientInfo, Action, Topic], Default) of
AuthzResult = #{result := Result} when Result == allow; Result == deny ->
From = maps:get(from, AuthzResult, unknown),
emqx:run_hook(
'client.check_authz_complete',
[ClientInfo, PubSub, Topic, Result, From]
[ClientInfo, Action, Topic, Result, From]
),
Result;
Other ->
@ -133,7 +133,7 @@ do_authorize(ClientInfo, PubSub, Topic) ->
}),
emqx:run_hook(
'client.check_authz_complete',
[ClientInfo, PubSub, Topic, deny, unknown_return_format]
[ClientInfo, Action, Topic, deny, unknown_return_format]
),
deny
end.

22
apps/emqx/src/emqx_app.erl
View File

@ -26,8 +26,8 @@
get_release/0,
set_config_loader/1,
get_config_loader/0,
set_init_tnx_id/1,
get_init_tnx_id/0
unset_config_loaded/0,
init_load_done/0
]).
-include("logger.hrl").
@ -56,24 +56,22 @@ prep_stop(_State) ->
stop(_State) -> ok.
-define(CONFIG_LOADER, config_loader).
-define(DEFAULT_LOADER, emqx).
%% @doc Call this function to make emqx boot without loading config,
%% in case we want to delegate the config load to a higher level app
%% which manages emqx app.
set_config_loader(Module) when is_atom(Module) ->
application:set_env(emqx, config_loader, Module).
application:set_env(emqx, ?CONFIG_LOADER, Module).
get_config_loader() ->
application:get_env(emqx, config_loader, emqx).
application:get_env(emqx, ?CONFIG_LOADER, ?DEFAULT_LOADER).
%% @doc Set the transaction id from which this node should start applying after boot.
%% The transaction ID is received from the core node which we just copied the latest
%% config from.
set_init_tnx_id(TnxId) ->
application:set_env(emqx, cluster_rpc_init_tnx_id, TnxId).
unset_config_loaded() ->
application:unset_env(emqx, ?CONFIG_LOADER).
%% @doc Get the transaction id from which this node should start applying after boot.
get_init_tnx_id() ->
application:get_env(emqx, cluster_rpc_init_tnx_id, -1).
init_load_done() ->
get_config_loader() =/= ?DEFAULT_LOADER.
maybe_load_config() ->
case get_config_loader() of

5
apps/emqx/src/emqx_authz_cache.erl
View File

@ -16,7 +16,7 @@
-module(emqx_authz_cache).
-include("emqx.hrl").
-include("emqx_access_control.hrl").
-export([
list_authz_cache/0,
@ -159,8 +159,7 @@ dump_authz_cache() ->
map_authz_cache(Fun) ->
[
Fun(R)
|| R = {{SubPub, _T}, _Authz} <- erlang:get(),
SubPub =:= publish orelse SubPub =:= subscribe
|| R = {{?authz_action, _T}, _Authz} <- erlang:get()
].
foreach_authz_cache(Fun) ->
_ = map_authz_cache(Fun),

47
apps/emqx/src/emqx_channel.erl
View File

@ -20,6 +20,7 @@
-include("emqx.hrl").
-include("emqx_channel.hrl").
-include("emqx_mqtt.hrl").
-include("emqx_access_control.hrl").
-include("logger.hrl").
-include("types.hrl").
@ -491,7 +492,7 @@ handle_in(
ok ->
TopicFilters0 = parse_topic_filters(TopicFilters),
TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
TupleTopicFilters0 = check_sub_authzs(SubPkt, TopicFilters1, Channel),
HasAuthzDeny = lists:any(
fun({_TopicFilter, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
@ -1838,14 +1839,34 @@ check_pub_alias(
check_pub_alias(_Packet, _Channel) ->
ok.
%%--------------------------------------------------------------------
%% Authorization action
authz_action(#mqtt_packet{
header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{}
}) ->
?AUTHZ_PUBLISH(QoS, Retain);
authz_action(#mqtt_packet{
header = #mqtt_packet_header{qos = QoS}, variable = #mqtt_packet_subscribe{}
}) ->
?AUTHZ_SUBSCRIBE(QoS);
%% Will message
authz_action(#message{qos = QoS, flags = #{retain := Retain}}) ->
?AUTHZ_PUBLISH(QoS, Retain);
authz_action(#message{qos = QoS}) ->
?AUTHZ_PUBLISH(QoS).
%%--------------------------------------------------------------------
%% Check Pub Authorization
check_pub_authz(
#mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}},
#mqtt_packet{
variable = #mqtt_packet_publish{topic_name = Topic}
} = Packet,
#channel{clientinfo = ClientInfo}
) ->
case emqx_access_control:authorize(ClientInfo, publish, Topic) of
Action = authz_action(Packet),
case emqx_access_control:authorize(ClientInfo, Action, Topic) of
allow -> ok;
deny -> {error, ?RC_NOT_AUTHORIZED}
end.
@ -1868,24 +1889,23 @@ check_pub_caps(
%%--------------------------------------------------------------------
%% Check Sub Authorization
%% TODO: not only check topic filter. Qos chould be checked too.
%% Not implemented yet:
%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
check_sub_authzs(TopicFilters, Channel) ->
check_sub_authzs(TopicFilters, Channel, []).
check_sub_authzs(Packet, TopicFilters, Channel) ->
Action = authz_action(Packet),
check_sub_authzs(Action, TopicFilters, Channel, []).
check_sub_authzs(
Action,
[TopicFilter = {Topic, _} | More],
Channel = #channel{clientinfo = ClientInfo},
Acc
) ->
case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of
case emqx_access_control:authorize(ClientInfo, Action, Topic) of
allow ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
deny ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end;
check_sub_authzs([], _Channel, Acc) ->
check_sub_authzs(_Action, [], _Channel, Acc) ->
lists:reverse(Acc).
%%--------------------------------------------------------------------
@ -2149,7 +2169,8 @@ publish_will_msg(
ClientInfo = #{mountpoint := MountPoint},
Msg = #message{topic = Topic}
) ->
PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow,
Action = authz_action(Msg),
PublishingDisallowed = emqx_access_control:authorize(ClientInfo, Action, Topic) =/= allow,
ClientBanned = emqx_banned:check(ClientInfo),
case PublishingDisallowed orelse ClientBanned of
true ->

12
apps/emqx/src/emqx_listeners.erl
View File

@ -96,7 +96,9 @@ format_list(Listener) ->
do_list_raw() ->
%% GET /listeners from other nodes returns [] when init config is not loaded.
case emqx_app:get_config_loader() =/= emqx of
%% FIXME This is a workaround for the issue:
%% mria:running_nodes() sometime return node which not ready to accept rpc call.
case emqx_app:init_load_done() of
true ->
Key = <<"listeners">>,
Raw = emqx_config:get_raw([Key], #{}),
@ -368,7 +370,7 @@ console_print(_Fmt, _Args) -> ok.
%% Start MQTT/TCP listener
-spec do_start_listener(atom(), atom(), map()) ->
{ok, pid() | {skipped, atom()}} | {error, term()}.
do_start_listener(_Type, _ListenerName, #{enabled := false}) ->
do_start_listener(_Type, _ListenerName, #{enable := false}) ->
{ok, {skipped, listener_disabled}};
do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when
Type == tcp; Type == ssl
@ -499,8 +501,8 @@ post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf
post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) ->
remove_listener(Type, Name, OldConf);
post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
#{enabled := NewEnabled} = NewConf,
#{enabled := OldEnabled} = OldConf,
#{enable := NewEnabled} = NewConf,
#{enable := OldEnabled} = OldConf,
case {NewEnabled, OldEnabled} of
{true, true} ->
ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
@ -810,7 +812,7 @@ has_enabled_listener_conf_by_type(Type) ->
lists:any(
fun({Id, LConf}) when is_map(LConf) ->
{ok, #{type := Type0}} = parse_listener_id(Id),
Type =:= Type0 andalso maps:get(enabled, LConf, true)
Type =:= Type0 andalso maps:get(enable, LConf, true)
end,
list()
).

11
apps/emqx/src/emqx_schema.erl
View File

@ -30,6 +30,7 @@
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("logger.hrl").
-define(MAX_INT_MQTT_PACKET_SIZE, 268435456).
-define(MAX_INT_TIMEOUT_MS, 4294967295).
%% floor(?MAX_INT_TIMEOUT_MS / 1000).
-define(MAX_INT_TIMEOUT_S, 4294967).
@ -45,6 +46,7 @@
-type timeout_duration_s() :: 0..?MAX_INT_TIMEOUT_S.
-type timeout_duration_ms() :: 0..?MAX_INT_TIMEOUT_MS.
-type bytesize() :: integer().
-type mqtt_max_packet_size() :: 1..?MAX_INT_MQTT_PACKET_SIZE.
-type wordsize() :: bytesize().
-type percent() :: float().
-type file() :: string().
@ -71,6 +73,7 @@
-typerefl_from_string({timeout_duration_s/0, emqx_schema, to_timeout_duration_s}).
-typerefl_from_string({timeout_duration_ms/0, emqx_schema, to_timeout_duration_ms}).
-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}).
-typerefl_from_string({mqtt_max_packet_size/0, emqx_schema, to_bytesize}).
-typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}).
-typerefl_from_string({percent/0, emqx_schema, to_percent}).
-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}).
@ -151,6 +154,7 @@
timeout_duration_s/0,
timeout_duration_ms/0,
bytesize/0,
mqtt_max_packet_size/0,
wordsize/0,
percent/0,
file/0,
@ -1746,13 +1750,12 @@ mqtt_listener(Bind) ->
base_listener(Bind) ->
[
{"enabled",
{"enable",
sc(
boolean(),
#{
default => true,
%% TODO(5.2): change field name to 'enable' and keep 'enabled' as an alias
aliases => [enable],
aliases => [enabled],
desc => ?DESC(fields_listener_enabled)
}
)},
@ -3357,7 +3360,7 @@ mqtt_general() ->
)},
{"max_packet_size",
sc(
bytesize(),
mqtt_max_packet_size(),
#{
default => <<"1MB">>,
desc => ?DESC(mqtt_max_packet_size)

8
apps/emqx/src/emqx_types.erl
View File

@ -29,6 +29,7 @@
-export_type([
zone/0,
pubsub/0,
pubsub_action/0,
subid/0
]).
@ -127,7 +128,12 @@
| exactly_once.
-type zone() :: atom().
-type pubsub() :: publish | subscribe.
-type pubsub_action() :: publish | subscribe.
-type pubsub() ::
#{action_type := subscribe, qos := qos()}
| #{action_type := publish, qos := qos(), retain := boolean()}.
-type subid() :: binary() | atom().
-type group() :: binary() | undefined.

17
apps/emqx/test/emqx_access_control_SUITE.erl
View File

@ -19,8 +19,8 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("emqx/include/emqx_access_control.hrl").
-include_lib("eunit/include/eunit.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
@ -44,8 +44,7 @@ t_authenticate(_) ->
?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
t_authorize(_) ->
Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)).
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, <<"t">>)).
t_delayed_authorize(_) ->
RawTopic = <<"$delayed/1/foo/2">>,
@ -54,11 +53,11 @@ t_delayed_authorize(_) ->
ok = emqx_hooks:put('client.authorize', {?MODULE, authz_stub, [Topic]}, ?HP_AUTHZ),
Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, RawTopic)),
Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>),
?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
?assertEqual(
deny, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, InvalidTopic)
),
ok.
t_quick_deny_anonymous(_) ->
@ -96,8 +95,8 @@ t_quick_deny_anonymous(_) ->
%% Helper functions
%%--------------------------------------------------------------------
authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
authz_stub(_Client, _Action, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
authz_stub(_Client, _Action, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) ->
{stop, {error, not_authorized}};

2
apps/emqx/test/emqx_authz_cache_SUITE.erl
View File

@ -43,8 +43,6 @@ t_clean_authz_cache(_) ->
ct:sleep(100),
ClientPid =
case emqx_cm:lookup_channels(<<"emqx_c">>) of
[Pid] when is_pid(Pid) ->
Pid;
Pids when is_list(Pids) ->
lists:last(Pids);
_ ->

3
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -908,7 +908,8 @@ t_check_pub_alias(_) ->
t_check_sub_authzs(_) ->
emqx_config:put_zone_conf(default, [authorization, enable], true),
TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
[{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()).
Subscribe = ?SUBSCRIBE_PACKET(1, [TopicFilter]),
[{TopicFilter, 0}] = emqx_channel:check_sub_authzs(Subscribe, [TopicFilter], channel()).
t_enrich_connack_caps(_) ->
ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),

2
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -618,7 +618,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) ->
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256"
],
enabled => true,
enable => true,
idle_timeout => 15000,
ssl_options => #{
certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"),

14
apps/emqx/test/emqx_cth_cluster.erl
View File

@ -206,11 +206,6 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) ->
base_port := BasePort,
work_dir := WorkDir
} = Spec,
Listeners = [
#{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}}
|| Type <- [tcp, ssl, ws, wss],
Port <- [listener_port(BasePort, Type)]
],
Cluster =
case get_cluster_seeds(Spec) of
[_ | _] = Seeds ->
@ -239,7 +234,7 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) ->
tcp_server_port => gen_rpc_port(BasePort),
port_discovery => manual
},
listeners => lists:foldl(fun maps:merge/2, #{}, Listeners)
listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec)
}
};
default_appspec(_App, _, _) ->
@ -252,6 +247,13 @@ get_cluster_seeds(#{join_to := Node}) ->
get_cluster_seeds(#{core_nodes := CoreNodes}) ->
CoreNodes.
allocate_listener_port(Type, #{base_port := BasePort}) ->
Port = listener_port(BasePort, Type),
#{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}}.
allocate_listener_ports(Types, Spec) ->
lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]).
start_node_init(Spec = #{name := Node}) ->
Node = start_bare_node(Node, Spec),
pong = net_adm:ping(Node),

2
apps/emqx/test/emqx_cth_suite.erl
View File

@ -305,7 +305,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
#{
config => SharedConfig,
% NOTE
% We inform `emqx` of our config loader before starting `emqx_conf` sothat it won't
% We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
% overwrite everything with a default configuration.
before_start => fun() ->
emqx_app:set_config_loader(?MODULE)

2
apps/emqx/test/emqx_listeners_SUITE.erl
View File

@ -229,7 +229,7 @@ t_ssl_password_cert(Config) ->
keyfile => filename:join(DataDir, "server-password.key")
},
LConf = #{
enabled => true,
enable => true,
bind => {{127, 0, 0, 1}, Port},
mountpoint => <<>>,
zone => default,

21
apps/emqx/test/emqx_proper_types.erl
View File

@ -20,6 +20,7 @@
-include_lib("proper/include/proper.hrl").
-include("emqx.hrl").
-include("emqx_access_control.hrl").
%% High level Types
-export([
@ -34,7 +35,8 @@
subopts/0,
nodename/0,
normal_topic/0,
normal_topic_filter/0
normal_topic_filter/0,
pubsub/0
]).
%% Basic Types
@ -482,6 +484,23 @@ normal_topic_filter() ->
end
).
subscribe_action() ->
?LET(
Qos,
qos(),
?AUTHZ_SUBSCRIBE(Qos)
).
publish_action() ->
?LET(
{Qos, Retain},
{qos(), boolean()},
?AUTHZ_PUBLISH(Qos, Retain)
).
pubsub() ->
oneof([publish_action(), subscribe_action()]).
%%--------------------------------------------------------------------
%% Basic Types
%%--------------------------------------------------------------------

2
apps/emqx_authz/etc/acl.conf
View File

@ -20,7 +20,7 @@
%%
%% -type(permission() :: allow | deny).
%%
%% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}).
%% -type(rule() :: {permission(), who(), action(), topics()} | {permission(), all}).
%%--------------------------------------------------------------------
{allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}.

74
apps/emqx_authz/include/emqx_authz.hrl
View File

@ -18,16 +18,6 @@
-define(APP, emqx_authz).
-define(ALLOW_DENY(A),
((A =:= allow) orelse (A =:= <<"allow">>) orelse
(A =:= deny) orelse (A =:= <<"deny">>))
).
-define(PUBSUB(A),
((A =:= subscribe) orelse (A =:= <<"subscribe">>) orelse
(A =:= publish) orelse (A =:= <<"publish">>) orelse
(A =:= all) orelse (A =:= <<"all">>))
).
%% authz_mnesia
-define(ACL_TABLE, emqx_acl).
@ -59,12 +49,12 @@
username => user1,
rules => [
#{
topic => <<"test/toopic/1">>,
topic => <<"test/topic/1">>,
permission => <<"allow">>,
action => <<"publish">>
},
#{
topic => <<"test/toopic/2">>,
topic => <<"test/topic/2">>,
permission => <<"allow">>,
action => <<"subscribe">>
},
@ -72,6 +62,20 @@
topic => <<"eq test/#">>,
permission => <<"deny">>,
action => <<"all">>
},
#{
topic => <<"test/topic/3">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"1">>],
retain => <<"true">>
},
#{
topic => <<"test/topic/4">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"0">>, <<"1">>, <<"2">>],
retain => <<"all">>
}
]
}).
@ -79,12 +83,12 @@
clientid => client1,
rules => [
#{
topic => <<"test/toopic/1">>,
topic => <<"test/topic/1">>,
permission => <<"allow">>,
action => <<"publish">>
},
#{
topic => <<"test/toopic/2">>,
topic => <<"test/topic/2">>,
permission => <<"allow">>,
action => <<"subscribe">>
},
@ -92,18 +96,32 @@
topic => <<"eq test/#">>,
permission => <<"deny">>,
action => <<"all">>
},
#{
topic => <<"test/topic/3">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"1">>],
retain => <<"true">>
},
#{
topic => <<"test/topic/4">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"0">>, <<"1">>, <<"2">>],
retain => <<"all">>
}
]
}).
-define(ALL_RULES_EXAMPLE, #{
rules => [
#{
topic => <<"test/toopic/1">>,
topic => <<"test/topic/1">>,
permission => <<"allow">>,
action => <<"publish">>
},
#{
topic => <<"test/toopic/2">>,
topic => <<"test/topic/2">>,
permission => <<"allow">>,
action => <<"subscribe">>
},
@ -111,9 +129,28 @@
topic => <<"eq test/#">>,
permission => <<"deny">>,
action => <<"all">>
},
#{
topic => <<"test/topic/3">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"1">>],
retain => <<"true">>
},
#{
topic => <<"test/topic/4">>,
permission => <<"allow">>,
action => <<"publish">>,
qos => [<<"0">>, <<"1">>, <<"2">>],
retain => <<"all">>
}
]
}).
-define(USERNAME_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?USERNAME_RULES_EXAMPLE))).
-define(CLIENTID_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?CLIENTID_RULES_EXAMPLE))).
-define(ALL_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?ALL_RULES_EXAMPLE))).
-define(META_EXAMPLE, #{
page => 1,
limit => 100,
@ -121,3 +158,8 @@
}).
-define(RESOURCE_GROUP, <<"emqx_authz">>).
-define(AUTHZ_FEATURES, [rich_actions]).
-define(DEFAULT_RULE_QOS, [0, 1, 2]).
-define(DEFAULT_RULE_RETAIN, all).

2
apps/emqx_authz/src/emqx_authz.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authz, [
{description, "An OTP application"},
{vsn, "0.1.23"},
{vsn, "0.1.24"},
{registered, []},
{mod, {emqx_authz_app, []}},
{applications, [

19
apps/emqx_authz/src/emqx_authz.erl
View File

@ -39,6 +39,11 @@
get_enabled_authzs/0
]).
-export([
feature_available/1,
set_feature_available/2
]).
-export([post_config_update/5, pre_config_update/3]).
-export([acl_conf_file/0]).
@ -519,6 +524,20 @@ read_acl_file(#{<<"path">> := Path} = Source) ->
{ok, Rules} = emqx_authz_file:read_file(Path),
maps:remove(<<"path">>, Source#{<<"rules">> => Rules}).
%%------------------------------------------------------------------------------
%% Extended Features
%%------------------------------------------------------------------------------
-define(DEFAULT_RICH_ACTIONS, true).
-define(FEATURE_KEY(_NAME_), {?MODULE, _NAME_}).
feature_available(rich_actions) ->
persistent_term:get(?FEATURE_KEY(rich_actions), ?DEFAULT_RICH_ACTIONS).
set_feature_available(Feature, Enable) when is_boolean(Enable) ->
persistent_term:put(?FEATURE_KEY(Feature), Enable).
%%------------------------------------------------------------------------------
%% Internal function
%%------------------------------------------------------------------------------

101
apps/emqx_authz/src/emqx_authz_api_mnesia.erl
View File

@ -359,6 +359,22 @@ fields(rule_item) ->
required => true,
example => publish
}
)},
{qos,
mk(
array(emqx_schema:qos()),
#{
desc => ?DESC(qos),
default => ?DEFAULT_RULE_QOS
}
)},
{retain,
mk(
hoconsc:union([all, boolean()]),
#{
desc => ?DESC(retain),
default => ?DEFAULT_RULE_RETAIN
}
)}
];
fields(clientid) ->
@ -434,7 +450,7 @@ users(post, #{body := Body}) when is_list(Body) ->
[] ->
lists:foreach(
fun(#{<<"username">> := Username, <<"rules">> := Rules}) ->
emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules))
emqx_authz_mnesia:store_rules({username, Username}, Rules)
end,
Body
),
@ -470,7 +486,7 @@ clients(post, #{body := Body}) when is_list(Body) ->
[] ->
lists:foreach(
fun(#{<<"clientid">> := ClientID, <<"rules">> := Rules}) ->
emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules))
emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules)
end,
Body
),
@ -489,21 +505,14 @@ user(get, #{bindings := #{username := Username}}) ->
{ok, Rules} ->
{200, #{
username => Username,
rules => [
#{
topic => Topic,
action => Action,
permission => Permission
}
|| {Permission, Action, Topic} <- Rules
]
rules => format_rules(Rules)
}}
end;
user(put, #{
bindings := #{username := Username},
body := #{<<"username">> := Username, <<"rules">> := Rules}
}) ->
emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)),
emqx_authz_mnesia:store_rules({username, Username}, Rules),
{204};
user(delete, #{bindings := #{username := Username}}) ->
case emqx_authz_mnesia:get_rules({username, Username}) of
@ -521,21 +530,14 @@ client(get, #{bindings := #{clientid := ClientID}}) ->
{ok, Rules} ->
{200, #{
clientid => ClientID,
rules => [
#{
topic => Topic,
action => Action,
permission => Permission
}
|| {Permission, Action, Topic} <- Rules
]
rules => format_rules(Rules)
}}
end;
client(put, #{
bindings := #{clientid := ClientID},
body := #{<<"clientid">> := ClientID, <<"rules">> := Rules}
}) ->
emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules)),
emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules),
{204};
client(delete, #{bindings := #{clientid := ClientID}}) ->
case emqx_authz_mnesia:get_rules({clientid, ClientID}) of
@ -552,18 +554,11 @@ all(get, _) ->
{200, #{rules => []}};
{ok, Rules} ->
{200, #{
rules => [
#{
topic => Topic,
action => Action,
permission => Permission
}
|| {Permission, Action, Topic} <- Rules
]
rules => format_rules(Rules)
}}
end;
all(post, #{body := #{<<"rules">> := Rules}}) ->
emqx_authz_mnesia:store_rules(all, format_rules(Rules)),
emqx_authz_mnesia:store_rules(all, Rules),
{204};
all(delete, _) ->
emqx_authz_mnesia:store_rules(all, []),
@ -626,58 +621,20 @@ run_fuzzy_filter(
%%--------------------------------------------------------------------
%% format funcs
%% format rule from api
format_rules(Rules) when is_list(Rules) ->
lists:foldl(
fun(
#{
<<"topic">> := Topic,
<<"action">> := Action,
<<"permission">> := Permission
},
AccIn
) when
?PUBSUB(Action) andalso
?ALLOW_DENY(Permission)
->
AccIn ++ [{atom(Permission), atom(Action), Topic}]
end,
[],
Rules
).
%% format result from mnesia tab
format_result([{username, Username}, {rules, Rules}]) ->
#{
username => Username,
rules => [
#{
topic => Topic,
action => Action,
permission => Permission
}
|| {Permission, Action, Topic} <- Rules
]
rules => format_rules(Rules)
};
format_result([{clientid, ClientID}, {rules, Rules}]) ->
#{
clientid => ClientID,
rules => [
#{
topic => Topic,
action => Action,
permission => Permission
}
|| {Permission, Action, Topic} <- Rules
]
rules => format_rules(Rules)
}.
atom(B) when is_binary(B) ->
try
binary_to_existing_atom(B, utf8)
catch
_Error:_Expection -> binary_to_atom(B)
end;
atom(A) when is_atom(A) -> A.
format_rules(Rules) ->
[emqx_authz_rule_raw:format_rule(Rule) || Rule <- Rules].
%%--------------------------------------------------------------------
%% Internal functions

1
apps/emqx_authz/src/emqx_authz_file.erl
View File

@ -16,7 +16,6 @@
-module(emqx_authz_file).
-include("emqx_authz.hrl").
-include_lib("emqx/include/logger.hrl").
-behaviour(emqx_authz).

35
apps/emqx_authz/src/emqx_authz_http.erl
View File

@ -51,6 +51,11 @@
?PH_CERT_CN_NAME
]).
-define(PLACEHOLDERS_FOR_RICH_ACTIONS, [
?PH_QOS,
?PH_RETAIN
]).
description() ->
"AuthZ with http".
@ -72,7 +77,7 @@ destroy(#{annotations := #{id := Id}}) ->
authorize(
Client,
PubSub,
Action,
Topic,
#{
type := http,
@ -81,7 +86,7 @@ authorize(
request_timeout := RequestTimeout
} = Config
) ->
Request = generate_request(PubSub, Topic, Client, Config),
Request = generate_request(Action, Topic, Client, Config),
case emqx_resource:simple_sync_query(ResourceID, {Method, Request, RequestTimeout}) of
{ok, 204, _Headers} ->
{matched, allow};
@ -139,14 +144,14 @@ parse_config(
method => Method,
base_url => BaseUrl,
headers => Headers,
base_path_templete => emqx_authz_utils:parse_str(Path, ?PLACEHOLDERS),
base_path_templete => emqx_authz_utils:parse_str(Path, placeholders()),
base_query_template => emqx_authz_utils:parse_deep(
cow_qs:parse_qs(to_bin(Query)),
?PLACEHOLDERS
placeholders()
),
body_template => emqx_authz_utils:parse_deep(
maps:to_list(maps:get(body, Conf, #{})),
?PLACEHOLDERS
placeholders()
),
request_timeout => ReqTimeout,
%% pool_type default value `random`
@ -173,7 +178,7 @@ parse_url(Url) ->
end.
generate_request(
PubSub,
Action,
Topic,
Client,
#{
@ -184,7 +189,7 @@ generate_request(
body_template := BodyTemplate
}
) ->
Values = client_vars(Client, PubSub, Topic),
Values = client_vars(Client, Action, Topic),
Path = emqx_authz_utils:render_urlencoded_str(BasePathTemplate, Values),
Query = emqx_authz_utils:render_deep(BaseQueryTemplate, Values),
Body = emqx_authz_utils:render_deep(BodyTemplate, Values),
@ -227,11 +232,9 @@ serialize_body(<<"application/json">>, Body) ->
serialize_body(<<"application/x-www-form-urlencoded">>, Body) ->
query_string(Body).
client_vars(Client, PubSub, Topic) ->
Client#{
action => PubSub,
topic => Topic
}.
client_vars(Client, Action, Topic) ->
Vars = emqx_authz_utils:vars_for_rule_query(Client, Action),
Vars#{topic => Topic}.
to_list(A) when is_atom(A) ->
atom_to_list(A);
@ -243,3 +246,11 @@ to_list(L) when is_list(L) ->
to_bin(B) when is_binary(B) -> B;
to_bin(L) when is_list(L) -> list_to_binary(L);
to_bin(X) -> X.
placeholders() ->
placeholders(emqx_authz:feature_available(rich_actions)).
placeholders(true) ->
?PLACEHOLDERS ++ ?PLACEHOLDERS_FOR_RICH_ACTIONS;
placeholders(false) ->
?PLACEHOLDERS.

28
apps/emqx_authz/src/emqx_authz_mnesia.erl
View File

@ -16,7 +16,6 @@
-module(emqx_authz_mnesia).
-include_lib("emqx/include/emqx.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include_lib("emqx/include/logger.hrl").
@ -202,25 +201,16 @@ record_count() ->
%%--------------------------------------------------------------------
normalize_rules(Rules) ->
lists:map(fun normalize_rule/1, Rules).
lists:flatmap(fun normalize_rule/1, Rules).
normalize_rule({Permission, Action, Topic}) ->
{normalize_permission(Permission), normalize_action(Action), normalize_topic(Topic)};
normalize_rule(Rule) ->
error({invalid_rule, Rule}).
normalize_topic(Topic) when is_list(Topic) -> list_to_binary(Topic);
normalize_topic(Topic) when is_binary(Topic) -> Topic;
normalize_topic(Topic) -> error({invalid_rule_topic, Topic}).
normalize_action(publish) -> publish;
normalize_action(subscribe) -> subscribe;
normalize_action(all) -> all;
normalize_action(Action) -> error({invalid_rule_action, Action}).
normalize_permission(allow) -> allow;
normalize_permission(deny) -> deny;
normalize_permission(Permission) -> error({invalid_rule_permission, Permission}).
normalize_rule(RuleRaw) ->
case emqx_authz_rule_raw:parse_rule(RuleRaw) of
%% For backward compatibility
{ok, {Permission, Action, TopicFilters}} ->
[{Permission, Action, TopicFilter} || TopicFilter <- TopicFilters];
{error, Reason} ->
error(Reason)
end.
do_get_rules(Key) ->
case mnesia:dirty_read(?ACL_TABLE, Key) of

37
apps/emqx_authz/src/emqx_authz_mongodb.erl
View File

@ -68,7 +68,7 @@ destroy(#{annotations := #{id := Id}}) ->
authorize(
Client,
PubSub,
Action,
Topic,
#{
collection := Collection,
@ -77,14 +77,7 @@ authorize(
}
) ->
RenderedFilter = emqx_authz_utils:render_deep(FilterTemplate, Client),
Result =
try
emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}})
catch
error:Error -> {error, Error}
end,
case Result of
case emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}}) of
{error, Reason} ->
?SLOG(error, #{
msg => "query_mongo_error",
@ -94,18 +87,22 @@ authorize(
resource_id => ResourceID
}),
nomatch;
{ok, []} ->
nomatch;
{ok, Rows} ->
Rules = [
emqx_authz_rule:compile({Permission, all, Action, Topics})
|| #{
<<"topics">> := Topics,
<<"permission">> := Permission,
<<"action">> := Action
} <- Rows
],
do_authorize(Client, PubSub, Topic, Rules)
Rules = lists:flatmap(fun parse_rule/1, Rows),
do_authorize(Client, Action, Topic, Rules)
end.
parse_rule(Row) ->
case emqx_authz_rule_raw:parse_rule(Row) of
{ok, {Permission, Action, Topics}} ->
[emqx_authz_rule:compile({Permission, all, Action, Topics})];
{error, Reason} ->
?SLOG(error, #{
msg => "parse_rule_error",
reason => Reason,
row => Row
}),
[]
end.
do_authorize(_Client, _PubSub, _Topic, []) ->

52
apps/emqx_authz/src/emqx_authz_mysql.erl
View File

@ -55,7 +55,7 @@ create(#{query := SQL} = Source0) ->
ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
{ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source),
Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}.
Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}.
update(#{query := SQL} = Source0) ->
{PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
@ -64,7 +64,7 @@ update(#{query := SQL} = Source0) ->
{error, Reason} ->
error({load_config_error, Reason});
{ok, Id} ->
Source#{annotations => #{id => Id, tmpl_oken => TmplToken}}
Source#{annotations => #{id => Id, tmpl_token => TmplToken}}
end.
destroy(#{annotations := #{id := Id}}) ->
@ -72,57 +72,49 @@ destroy(#{annotations := #{id := Id}}) ->
authorize(
Client,
PubSub,
Action,
Topic,
#{
annotations := #{
id := ResourceID,
tmpl_oken := TmplToken
tmpl_token := TmplToken
}
}
) ->
RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Client),
Vars = emqx_authz_utils:vars_for_rule_query(Client, Action),
RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Vars),
case
emqx_resource:simple_sync_query(ResourceID, {prepared_query, ?PREPARE_KEY, RenderParams})
of
{ok, _Columns, []} ->
nomatch;
{ok, Columns, Rows} ->
do_authorize(Client, PubSub, Topic, Columns, Rows);
{ok, ColumnNames, Rows} ->
do_authorize(Client, Action, Topic, ColumnNames, Rows);
{error, Reason} ->
?SLOG(error, #{
msg => "query_mysql_error",
reason => Reason,
tmpl_oken => TmplToken,
tmpl_token => TmplToken,
params => RenderParams,
resource_id => ResourceID
}),
nomatch
end.
do_authorize(_Client, _PubSub, _Topic, _Columns, []) ->
do_authorize(_Client, _Action, _Topic, _ColumnNames, []) ->
nomatch;
do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) ->
case
do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) ->
try
emqx_authz_rule:match(
Client,
PubSub,
Topic,
emqx_authz_rule:compile(format_result(Columns, Row))
Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row)
)
of
{matched, Permission} -> {matched, Permission};
nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail)
nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail)
catch
error:Reason ->
?SLOG(error, #{
msg => "match_rule_error",
reason => Reason,
rule => Row
}),
do_authorize(Client, Action, Topic, ColumnNames, Tail)
end.
format_result(Columns, Row) ->
Permission = lists:nth(index(<<"permission">>, Columns), Row),
Action = lists:nth(index(<<"action">>, Columns), Row),
Topic = lists:nth(index(<<"topic">>, Columns), Row),
{Permission, all, Action, [Topic]}.
index(Elem, List) ->
index(Elem, List, 1).
index(_Elem, [], _Index) -> {error, not_found};
index(Elem, [Elem | _List], Index) -> Index;
index(Elem, [_ | List], Index) -> index(Elem, List, Index + 1).

53
apps/emqx_authz/src/emqx_authz_postgresql.erl
View File

@ -21,6 +21,8 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("epgsql/include/epgsql.hrl").
-behaviour(emqx_authz).
%% AuthZ Callbacks
@ -77,7 +79,7 @@ destroy(#{annotations := #{id := Id}}) ->
authorize(
Client,
PubSub,
Action,
Topic,
#{
annotations := #{
@ -86,14 +88,13 @@ authorize(
}
}
) ->
RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Client),
Vars = emqx_authz_utils:vars_for_rule_query(Client, Action),
RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Vars),
case
emqx_resource:simple_sync_query(ResourceID, {prepared_query, ResourceID, RenderedParams})
of
{ok, _Columns, []} ->
nomatch;
{ok, Columns, Rows} ->
do_authorize(Client, PubSub, Topic, Columns, Rows);
do_authorize(Client, Action, Topic, column_names(Columns), Rows);
{error, Reason} ->
?SLOG(error, #{
msg => "query_postgresql_error",
@ -104,33 +105,29 @@ authorize(
nomatch
end.
do_authorize(_Client, _PubSub, _Topic, _Columns, []) ->
do_authorize(_Client, _Action, _Topic, _ColumnNames, []) ->
nomatch;
do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) ->
case
do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) ->
try
emqx_authz_rule:match(
Client,
PubSub,
Topic,
emqx_authz_rule:compile(format_result(Columns, Row))
Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row)
)
of
{matched, Permission} -> {matched, Permission};
nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail)
nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail)
catch
error:Reason:Stack ->
?SLOG(error, #{
msg => "match_rule_error",
reason => Reason,
rule => Row,
stack => Stack
}),
do_authorize(Client, Action, Topic, ColumnNames, Tail)
end.
format_result(Columns, Row) ->
Permission = lists:nth(index(<<"permission">>, 2, Columns), erlang:tuple_to_list(Row)),
Action = lists:nth(index(<<"action">>, 2, Columns), erlang:tuple_to_list(Row)),
Topic = lists:nth(index(<<"topic">>, 2, Columns), erlang:tuple_to_list(Row)),
{Permission, all, Action, [Topic]}.
index(Key, N, TupleList) when is_integer(N) ->
Tuple = lists:keyfind(Key, N, TupleList),
index(Tuple, TupleList, 1);
index(_Tuple, [], _Index) ->
{error, not_found};
index(Tuple, [Tuple | _TupleList], Index) ->
Index;
index(Tuple, [_ | TupleList], Index) ->
index(Tuple, TupleList, Index + 1).
column_names(Columns) ->
lists:map(
fun(#column{name = Name}) -> Name end,
Columns
).

63
apps/emqx_authz/src/emqx_authz_redis.erl
View File

@ -70,19 +70,18 @@ destroy(#{annotations := #{id := Id}}) ->
authorize(
Client,
PubSub,
Action,
Topic,
#{
cmd_template := CmdTemplate,
annotations := #{id := ResourceID}
}
) ->
Cmd = emqx_authz_utils:render_deep(CmdTemplate, Client),
Vars = emqx_authz_utils:vars_for_rule_query(Client, Action),
Cmd = emqx_authz_utils:render_deep(CmdTemplate, Vars),
case emqx_resource:simple_sync_query(ResourceID, {cmd, Cmd}) of
{ok, []} ->
nomatch;
{ok, Rows} ->
do_authorize(Client, PubSub, Topic, Rows);
do_authorize(Client, Action, Topic, Rows);
{error, Reason} ->
?SLOG(error, #{
msg => "query_redis_error",
@ -93,21 +92,63 @@ authorize(
nomatch
end.
do_authorize(_Client, _PubSub, _Topic, []) ->
do_authorize(_Client, _Action, _Topic, []) ->
nomatch;
do_authorize(Client, PubSub, Topic, [TopicFilter, Action | Tail]) ->
case
do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) ->
try
emqx_authz_rule:match(
Client,
PubSub,
Action,
Topic,
emqx_authz_rule:compile({allow, all, Action, [TopicFilter]})
compile_rule(RuleEncoded, TopicFilterRaw)
)
of
{matched, Permission} -> {matched, Permission};
nomatch -> do_authorize(Client, PubSub, Topic, Tail)
nomatch -> do_authorize(Client, Action, Topic, Tail)
catch
error:Reason:Stack ->
?SLOG(error, #{
msg => "match_rule_error",
reason => Reason,
rule_encoded => RuleEncoded,
topic_filter_raw => TopicFilterRaw,
stacktrace => Stack
}),
do_authorize(Client, Action, Topic, Tail)
end.
compile_rule(RuleBin, TopicFilterRaw) ->
RuleRaw =
maps:merge(
#{
<<"permission">> => <<"allow">>,
<<"topic">> => TopicFilterRaw
},
parse_rule(RuleBin)
),
case emqx_authz_rule_raw:parse_rule(RuleRaw) of
{ok, {Permission, Action, Topics}} ->
emqx_authz_rule:compile({Permission, all, Action, Topics});
{error, Reason} ->
error(Reason)
end.
tokens(Query) ->
Tokens = binary:split(Query, <<" ">>, [global]),
[Token || Token <- Tokens, size(Token) > 0].
parse_rule(<<"publish">>) ->
#{<<"action">> => <<"publish">>};
parse_rule(<<"subscribe">>) ->
#{<<"action">> => <<"subscribe">>};
parse_rule(<<"all">>) ->
#{<<"action">> => <<"all">>};
parse_rule(Bin) when is_binary(Bin) ->
case emqx_utils_json:safe_decode(Bin, [return_maps]) of
{ok, Map} when is_map(Map) ->
maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map);
{ok, _} ->
error({invalid_topic_rule, Bin, notamap});
{error, Error} ->
error({invalid_topic_rule, Bin, Error})
end.

183
apps/emqx_authz/src/emqx_authz_rule.erl
View File

@ -16,9 +16,9 @@
-module(emqx_authz_rule).
-include("emqx_authz.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-include("emqx_authz.hrl").
-ifdef(TEST).
-compile(export_all).
@ -32,50 +32,123 @@
compile/1
]).
-type ipaddress() ::
{ipaddr, esockd_cidr:cidr_string()}
| {ipaddrs, list(esockd_cidr:cidr_string())}.
-type permission() :: allow | deny.
-type username() :: {username, binary()}.
-type clientid() :: {clientid, binary()}.
-type who() ::
-type who_condition() ::
ipaddress()
| username()
| clientid()
| {'and', [ipaddress() | username() | clientid()]}
| {'or', [ipaddress() | username() | clientid()]}
| all.
-type ipaddress() ::
{ipaddr, esockd_cidr:cidr_string()}
| {ipaddrs, list(esockd_cidr:cidr_string())}.
-type username() :: {username, binary()}.
-type clientid() :: {clientid, binary()}.
-type action() :: subscribe | publish | all.
-type permission() :: allow | deny.
-type action_condition() ::
subscribe
| publish
| #{action_type := subscribe, qos := qos_condition()}
| #{action_type := publish | all, qos := qos_condition(), retain := retain_condition()}
| all.
-type qos_condition() :: [qos()].
-type retain_condition() :: retain() | all.
-type rule() :: {permission(), who(), action(), list(emqx_types:topic())}.
-type topic_condition() :: list(emqx_types:topic() | {eq, emqx_types:topic()}).
-type rule() :: {permission(), who_condition(), action_condition(), topic_condition()}.
-type qos() :: emqx_types:qos().
-type retain() :: boolean().
-type action() ::
#{action_type := subscribe, qos := qos()}
| #{action_type := publish, qos := qos(), retain := retain()}.
-export_type([
action/0,
permission/0
permission/0,
who_condition/0,
action_condition/0,
topic_condition/0
]).
-define(IS_PERMISSION(Permission), (Permission =:= allow orelse Permission =:= deny)).
compile({Permission, all}) when
?ALLOW_DENY(Permission)
?IS_PERMISSION(Permission)
->
{Permission, all, all, [compile_topic(<<"#">>)]};
compile({Permission, Who, Action, TopicFilters}) when
?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(TopicFilters)
?IS_PERMISSION(Permission) andalso is_list(TopicFilters)
->
{atom(Permission), compile_who(Who), atom(Action), [
{Permission, compile_who(Who), compile_action(Action), [
compile_topic(Topic)
|| Topic <- TopicFilters
]};
compile({Permission, _Who, _Action, _TopicFilter}) when not ?ALLOW_DENY(Permission) ->
compile({Permission, _Who, _Action, _TopicFilter}) when not ?IS_PERMISSION(Permission) ->
throw({invalid_authorization_permission, Permission});
compile({_Permission, _Who, Action, _TopicFilter}) when not ?PUBSUB(Action) ->
throw({invalid_authorization_action, Action});
compile(BadRule) ->
throw({invalid_authorization_rule, BadRule}).
compile_action(Action) ->
compile_action(emqx_authz:feature_available(rich_actions), Action).
-define(IS_ACTION_WITH_RETAIN(Action), (Action =:= publish orelse Action =:= all)).
compile_action(_RichActionsOn, subscribe) ->
subscribe;
compile_action(_RichActionsOn, Action) when ?IS_ACTION_WITH_RETAIN(Action) ->
Action;
compile_action(true = _RichActionsOn, {subscribe, Opts}) when is_list(Opts) ->
#{
action_type => subscribe,
qos => qos_from_opts(Opts)
};
compile_action(true = _RichActionsOn, {Action, Opts}) when
?IS_ACTION_WITH_RETAIN(Action) andalso is_list(Opts)
->
#{
action_type => Action,
qos => qos_from_opts(Opts),
retain => retain_from_opts(Opts)
};
compile_action(_RichActionsOn, Action) ->
throw({invalid_authorization_action, Action}).
qos_from_opts(Opts) ->
try
case proplists:get_all_values(qos, Opts) of
[] ->
?DEFAULT_RULE_QOS;
QoSs ->
lists:flatmap(
fun
(QoS) when is_integer(QoS) ->
[validate_qos(QoS)];
(QoS) when is_list(QoS) ->
lists:map(fun validate_qos/1, QoS)
end,
QoSs
)
end
catch
bad_qos ->
throw({invalid_authorization_qos, Opts})
end.
validate_qos(QoS) when is_integer(QoS), QoS >= 0, QoS =< 2 ->
QoS;
validate_qos(_) ->
throw(bad_qos).
retain_from_opts(Opts) ->
case proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN) of
all -> all;
Retain when is_boolean(Retain) -> Retain;
_ -> throw({invalid_authorization_retain, Opts})
end.
compile_who(all) ->
all;
compile_who({user, Username}) ->
@ -99,8 +172,12 @@ compile_who({ipaddrs, CIDRs}) ->
compile_who({'and', L}) when is_list(L) ->
{'and', [compile_who(Who) || Who <- L]};
compile_who({'or', L}) when is_list(L) ->
{'or', [compile_who(Who) || Who <- L]}.
{'or', [compile_who(Who) || Who <- L]};
compile_who(Who) ->
throw({invalid_who, Who}).
compile_topic("eq " ++ Topic) ->
{eq, emqx_topic:words(bin(Topic))};
compile_topic(<<"eq ", Topic/binary>>) ->
{eq, emqx_topic:words(Topic)};
compile_topic({eq, Topic}) ->
@ -117,45 +194,65 @@ compile_topic(Topic) ->
Tokens -> {pattern, Tokens}
end.
atom(B) when is_binary(B) ->
try
binary_to_existing_atom(B, utf8)
catch
_E:_S -> binary_to_atom(B)
end;
atom(A) when is_atom(A) -> A.
bin(L) when is_list(L) ->
list_to_binary(L);
bin(B) when is_binary(B) ->
B.
-spec matches(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), [rule()]) ->
-spec matches(emqx_types:clientinfo(), action(), emqx_types:topic(), [rule()]) ->
{matched, allow} | {matched, deny} | nomatch.
matches(_Client, _PubSub, _Topic, []) ->
matches(_Client, _Action, _Topic, []) ->
nomatch;
matches(Client, PubSub, Topic, [{Permission, Who, Action, TopicFilters} | Tail]) ->
case match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) of
nomatch -> matches(Client, PubSub, Topic, Tail);
matches(Client, Action, Topic, [{Permission, WhoCond, ActionCond, TopicCond} | Tail]) ->
case match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) of
nomatch -> matches(Client, Action, Topic, Tail);
Matched -> Matched
end.
-spec match(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), rule()) ->
-spec match(emqx_types:clientinfo(), action(), emqx_types:topic(), rule()) ->
{matched, allow} | {matched, deny} | nomatch.
match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) ->
match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) ->
case
match_action(PubSub, Action) andalso
match_who(Client, Who) andalso
match_topics(Client, Topic, TopicFilters)
match_action(Action, ActionCond) andalso
match_who(Client, WhoCond) andalso
match_topics(Client, Topic, TopicCond)
of
true -> {matched, Permission};
_ -> nomatch
end.
match_action(publish, publish) -> true;
match_action(subscribe, subscribe) -> true;
match_action(_, all) -> true;
match_action(_, _) -> false.
-spec match_action(action(), action_condition()) -> boolean().
match_action(#{action_type := publish}, PubSubCond) when is_atom(PubSubCond) ->
match_pubsub(publish, PubSubCond);
match_action(
#{action_type := publish, qos := QoS, retain := Retain}, #{
action_type := publish, qos := QoSCond, retain := RetainCond
}
) ->
match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond);
match_action(#{action_type := publish, qos := QoS, retain := Retain}, #{
action_type := all, qos := QoSCond, retain := RetainCond
}) ->
match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond);
match_action(#{action_type := subscribe}, PubSubCond) when is_atom(PubSubCond) ->
match_pubsub(subscribe, PubSubCond);
match_action(#{action_type := subscribe, qos := QoS}, #{action_type := subscribe, qos := QoSCond}) ->
match_qos(QoS, QoSCond);
match_action(#{action_type := subscribe, qos := QoS}, #{action_type := all, qos := QoSCond}) ->
match_qos(QoS, QoSCond);
match_action(_, _) ->
false.
match_pubsub(publish, publish) -> true;
match_pubsub(subscribe, subscribe) -> true;
match_pubsub(_, all) -> true;
match_pubsub(_, _) -> false.
match_qos(QoS, QoSs) -> lists:member(QoS, QoSs).
match_retain(_, all) -> true;
match_retain(Retain, Retain) when is_boolean(Retain) -> true;
match_retain(_, _) -> false.
match_who(_, all) ->
true;

197
apps/emqx_authz/src/emqx_authz_rule_raw.erl Normal file
View File

@ -0,0 +1,197 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% @doc
%% This module converts authz rule fields obtained from
%% external sources like database or API to the format
%% accepted by emqx_authz_rule module.
%%--------------------------------------------------------------------
-module(emqx_authz_rule_raw).
-export([parse_rule/1, format_rule/1]).
-include("emqx_authz.hrl").
-type rule_raw() :: #{binary() => binary() | [binary()]}.
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
-spec parse_rule(rule_raw()) ->
{ok, {
emqx_authz_rule:permission(),
emqx_authz_rule:action_condition(),
emqx_authz_rule:topic_condition()
}}
| {error, term()}.
parse_rule(
#{
<<"permission">> := PermissionRaw,
<<"action">> := ActionTypeRaw
} = RuleRaw
) ->
try
Topics = validate_rule_topics(RuleRaw),
Permission = validate_rule_permission(PermissionRaw),
ActionType = validate_rule_action_type(ActionTypeRaw),
Action = validate_rule_action(ActionType, RuleRaw),
{ok, {Permission, Action, Topics}}
catch
throw:ValidationError ->
{error, ValidationError}
end;
parse_rule(RuleRaw) ->
{error, {invalid_rule, RuleRaw}}.
-spec format_rule({
emqx_authz_rule:permission(),
emqx_authz_rule:action_condition(),
emqx_authz_rule:topic_condition()
}) -> map().
format_rule({Permission, Action, Topics}) when is_list(Topics) ->
maps:merge(
#{
topic => lists:map(fun format_topic/1, Topics),
permission => Permission
},
format_action(Action)
);
format_rule({Permission, Action, Topic}) ->
maps:merge(
#{
topic => format_topic(Topic),
permission => Permission
},
format_action(Action)
).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
validate_rule_topics(#{<<"topic">> := TopicRaw}) when is_binary(TopicRaw) ->
[validate_rule_topic(TopicRaw)];
validate_rule_topics(#{<<"topics">> := TopicsRaw}) when is_list(TopicsRaw) ->
lists:map(fun validate_rule_topic/1, TopicsRaw);
validate_rule_topics(RuleRaw) ->
throw({invalid_topics, RuleRaw}).
validate_rule_topic(<<"eq ", TopicRaw/binary>>) ->
{eq, validate_rule_topic(TopicRaw)};
validate_rule_topic(TopicRaw) when is_binary(TopicRaw) -> TopicRaw.
validate_rule_permission(<<"allow">>) -> allow;
validate_rule_permission(<<"deny">>) -> deny;
validate_rule_permission(PermissionRaw) -> throw({invalid_permission, PermissionRaw}).
validate_rule_action_type(<<"publish">>) -> publish;
validate_rule_action_type(<<"subscribe">>) -> subscribe;
validate_rule_action_type(<<"all">>) -> all;
validate_rule_action_type(ActionRaw) -> throw({invalid_action, ActionRaw}).
validate_rule_action(ActionType, RuleRaw) ->
validate_rule_action(emqx_authz:feature_available(rich_actions), ActionType, RuleRaw).
%% rich_actions disabled
validate_rule_action(false, ActionType, _RuleRaw) ->
ActionType;
%% rich_actions enabled
validate_rule_action(true, publish, RuleRaw) ->
Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)),
Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)),
{publish, [{qos, Qos}, {retain, Retain}]};
validate_rule_action(true, subscribe, RuleRaw) ->
Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)),
{subscribe, [{qos, Qos}]};
validate_rule_action(true, all, RuleRaw) ->
Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)),
Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)),
{all, [{qos, Qos}, {retain, Retain}]}.
validate_rule_qos(QosInt) when is_integer(QosInt) andalso QosInt >= 0 andalso QosInt =< 2 ->
[QosInt];
validate_rule_qos(QosBin) when is_binary(QosBin) ->
try
QosRawList = binary:split(QosBin, <<",">>, [global]),
lists:map(fun validate_rule_qos_atomic/1, QosRawList)
catch
_:_ ->
throw({invalid_qos, QosBin})
end;
validate_rule_qos(QosList) when is_list(QosList) ->
try
lists:map(fun validate_rule_qos_atomic/1, QosList)
catch
invalid_qos ->
throw({invalid_qos, QosList})
end;
validate_rule_qos(undefined) ->
?DEFAULT_RULE_QOS;
validate_rule_qos(null) ->
?DEFAULT_RULE_QOS;
validate_rule_qos(QosRaw) ->
throw({invalid_qos, QosRaw}).
validate_rule_qos_atomic(<<"0">>) -> 0;
validate_rule_qos_atomic(<<"1">>) -> 1;
validate_rule_qos_atomic(<<"2">>) -> 2;
validate_rule_qos_atomic(0) -> 0;
validate_rule_qos_atomic(1) -> 1;
validate_rule_qos_atomic(2) -> 2;
validate_rule_qos_atomic(_) -> throw(invalid_qos).
validate_rule_retain(<<"0">>) -> false;
validate_rule_retain(<<"1">>) -> true;
validate_rule_retain(0) -> false;
validate_rule_retain(1) -> true;
validate_rule_retain(<<"true">>) -> true;
validate_rule_retain(<<"false">>) -> false;
validate_rule_retain(true) -> true;
validate_rule_retain(false) -> false;
validate_rule_retain(undefined) -> ?DEFAULT_RULE_RETAIN;
validate_rule_retain(null) -> ?DEFAULT_RULE_RETAIN;
validate_rule_retain(<<"all">>) -> ?DEFAULT_RULE_RETAIN;
validate_rule_retain(Retain) -> throw({invalid_retain, Retain}).
format_action(Action) ->
format_action(emqx_authz:feature_available(rich_actions), Action).
%% rich_actions disabled
format_action(false, Action) when is_atom(Action) ->
#{
action => Action
};
format_action(false, {ActionType, _Opts}) ->
#{
action => ActionType
};
%% rich_actions enabled
format_action(true, Action) when is_atom(Action) ->
#{
action => Action
};
format_action(true, {ActionType, Opts}) ->
#{
action => ActionType,
qos => proplists:get_value(qos, Opts, ?DEFAULT_RULE_QOS),
retain => proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN)
}.
format_topic({eq, Topic}) when is_binary(Topic) ->
<<"eq ", Topic/binary>>;
format_topic(Topic) when is_binary(Topic) ->
Topic.

30
apps/emqx_authz/src/emqx_authz_utils.erl
View File

@ -31,7 +31,10 @@
parse_sql/3,
render_deep/2,
render_str/2,
render_sql_params/2
render_sql_params/2,
client_vars/1,
vars_for_rule_query/2,
parse_rule_from_row/2
]).
-export([
@ -43,6 +46,8 @@
start_after_created => false
}).
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -171,6 +176,24 @@ content_type(Headers) when is_list(Headers) ->
<<"application/json">>
).
-define(RAW_RULE_KEYS, [<<"permission">>, <<"action">>, <<"topic">>, <<"qos">>, <<"retain">>]).
parse_rule_from_row(ColumnNames, Row) ->
RuleRaw = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))),
case emqx_authz_rule_raw:parse_rule(RuleRaw) of
{ok, {Permission, Action, Topics}} ->
emqx_authz_rule:compile({Permission, all, Action, Topics});
{error, Reason} ->
error(Reason)
end.
vars_for_rule_query(Client, ?authz_action(PubSub, Qos) = Action) ->
Client#{
action => PubSub,
qos => Qos,
retain => maps:get(retain, Action, false)
}.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
@ -208,3 +231,8 @@ handle_sql_var(_Name, Value) ->
bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
bin(L) when is_list(L) -> list_to_binary(L);
bin(X) -> X.
to_list(Tuple) when is_tuple(Tuple) ->
tuple_to_list(Tuple);
to_list(List) when is_list(List) ->
List.

6
apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl
View File

@ -96,7 +96,7 @@ t_api(_) ->
<<"hasnext">> := false
}
} = emqx_utils_json:decode(Request1),
?assertEqual(3, length(Rules1)),
?assertEqual(?USERNAME_RULES_EXAMPLE_COUNT, length(Rules1)),
{ok, 200, Request1_1} =
request(
@ -204,7 +204,7 @@ t_api(_) ->
} =
emqx_utils_json:decode(Request4),
#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5),
?assertEqual(3, length(Rules3)),
?assertEqual(?CLIENTID_RULES_EXAMPLE_COUNT, length(Rules3)),
{ok, 204, _} =
request(
@ -253,7 +253,7 @@ t_api(_) ->
[]
),
#{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7),
?assertEqual(3, length(Rules5)),
?assertEqual(?ALL_RULES_EXAMPLE_COUNT, length(Rules5)),
{ok, 204, _} =
request(

75
apps/emqx_authz/test/emqx_authz_file_SUITE.erl
View File

@ -38,34 +38,26 @@ all() ->
groups() ->
[].
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
ok.
init_per_testcase(TestCase, Config) ->
Apps = emqx_cth_suite:start(
[{emqx_conf, "authorization.no_match = deny"}, emqx_authz],
[
{emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"},
emqx_authz
],
#{work_dir => filename:join(?config(priv_dir, Config), TestCase)}
),
[{tc_apps, Apps} | Config].
end_per_testcase(_TestCase, Config) ->
emqx_cth_suite:stop(?config(tc_apps, Config)).
emqx_cth_suite:stop(?config(tc_apps, Config)),
_ = emqx_authz:set_feature_available(rich_actions, true).
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_ok(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo = emqx_authz_test_lib:base_client_info(),
ok = setup_config(?RAW_SOURCE#{
<<"rules">> => <<"{allow, {user, \"username\"}, publish, [\"t\"]}.">>
@ -73,23 +65,52 @@ t_ok(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>)
).
t_rich_actions(_Config) ->
ClientInfo = emqx_authz_test_lib:base_client_info(),
ok = setup_config(?RAW_SOURCE#{
<<"rules">> =>
<<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">>
}),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>)
),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(0, false), <<"t">>)
),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>)
).
t_no_rich_actions(_Config) ->
_ = emqx_authz:set_feature_available(rich_actions, false),
?assertMatch(
{error, {pre_config_update, emqx_authz, {invalid_authorization_action, _}}},
emqx_authz:update(?CMD_REPLACE, [
?RAW_SOURCE#{
<<"rules">> =>
<<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">>
}
])
).
t_superuser(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
is_superuser => true,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo =
emqx_authz_test_lib:client_info(#{is_superuser => true}),
%% no rules apply to superuser
ok = setup_config(?RAW_SOURCE#{
@ -98,12 +119,12 @@ t_superuser(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>)
).
t_invalid_file(_Config) ->

93
apps/emqx_authz/test/emqx_authz_http_SUITE.erl
View File

@ -65,6 +65,7 @@ init_per_testcase(_Case, Config) ->
Config.
end_per_testcase(_Case, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
try
ok = emqx_authz_http_test_server:stop()
catch
@ -97,7 +98,7 @@ t_response_handling(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
%% Not OK, get, no body
@ -109,7 +110,7 @@ t_response_handling(_Config) ->
#{}
),
deny = emqx_access_control:authorize(ClientInfo, publish, <<"t">>),
deny = emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>),
%% OK, get, 204
ok = setup_handler_and_config(
@ -122,7 +123,7 @@ t_response_handling(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
%% Not OK, get, 400
@ -136,7 +137,7 @@ t_response_handling(_Config) ->
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
%% Not OK, get, 400 + body & headers
@ -155,7 +156,7 @@ t_response_handling(_Config) ->
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
%% the server cannot be reached; should skip to the next
@ -165,7 +166,7 @@ t_response_handling(_Config) ->
?check_trace(
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
fun(Trace) ->
?assertMatch(
@ -200,7 +201,9 @@ t_query_params(_Config) ->
proto_name := <<"MQTT">>,
mountpoint := <<"MOUNTPOINT">>,
topic := <<"t/1">>,
action := <<"publish">>
action := <<"publish">>,
qos := <<"1">>,
retain := <<"false">>
} = cowboy_req:match_qs(
[
username,
@ -209,7 +212,9 @@ t_query_params(_Config) ->
proto_name,
mountpoint,
topic,
action
action,
qos,
retain
],
Req0
),
@ -224,7 +229,9 @@ t_query_params(_Config) ->
"proto_name=${proto_name}&"
"mountpoint=${mountpoint}&"
"topic=${topic}&"
"action=${action}"
"action=${action}&"
"qos=${qos}&"
"retain=${retain}"
>>
}
),
@ -241,7 +248,7 @@ t_query_params(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>)
).
t_path(_Config) ->
@ -256,7 +263,9 @@ t_path(_Config) ->
"MQTT/"
"MOUNTPOINT/"
"t%2F1/"
"publish"
"publish/"
"1/"
"false"
>>,
cowboy_req:path(Req0)
),
@ -271,7 +280,9 @@ t_path(_Config) ->
"${proto_name}/"
"${mountpoint}/"
"${topic}/"
"${action}"
"${action}/"
"${qos}/"
"${retain}"
>>
}
),
@ -288,7 +299,7 @@ t_path(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>)
).
t_json_body(_Config) ->
@ -309,7 +320,9 @@ t_json_body(_Config) ->
<<"proto_name">> := <<"MQTT">>,
<<"mountpoint">> := <<"MOUNTPOINT">>,
<<"topic">> := <<"t">>,
<<"action">> := <<"publish">>
<<"action">> := <<"publish">>,
<<"qos">> := <<"1">>,
<<"retain">> := <<"false">>
},
emqx_utils_json:decode(RawBody, [return_maps])
),
@ -324,7 +337,9 @@ t_json_body(_Config) ->
<<"proto_name">> => <<"${proto_name}">>,
<<"mountpoint">> => <<"${mountpoint}">>,
<<"topic">> => <<"${topic}">>,
<<"action">> => <<"${action}">>
<<"action">> => <<"${action}">>,
<<"qos">> => <<"${qos}">>,
<<"retain">> => <<"${retain}">>
}
}
),
@ -341,7 +356,45 @@ t_json_body(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>)
).
t_no_rich_actions(_Config) ->
_ = emqx_authz:set_feature_available(rich_actions, false),
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
<<"/authz/users/">>,
cowboy_req:path(Req0)
),
{ok, RawBody, Req1} = cowboy_req:read_body(Req0),
%% No interpolation if rich_actions is disabled
?assertMatch(
#{
<<"qos">> := <<"${qos}">>,
<<"retain">> := <<"${retain}">>
},
emqx_utils_json:decode(RawBody, [return_maps])
),
{ok, ?AUTHZ_HTTP_RESP(allow, Req1), State}
end,
#{
<<"method">> => <<"post">>,
<<"body">> => #{
<<"qos">> => <<"${qos}">>,
<<"retain">> => <<"${retain}">>
}
}
),
ClientInfo = emqx_authz_test_lib:base_client_info(),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>)
).
t_placeholder_and_body(_Config) ->
@ -401,7 +454,7 @@ t_placeholder_and_body(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
).
t_no_value_for_placeholder(_Config) ->
@ -441,7 +494,7 @@ t_no_value_for_placeholder(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
).
t_create_replace(_Config) ->
@ -466,7 +519,7 @@ t_create_replace(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
%% Changing to valid config
@ -485,7 +538,7 @@ t_create_replace(_Config) ->
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
).
%%------------------------------------------------------------------------------

5
apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl
View File

@ -22,6 +22,7 @@
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("emqx_authn/include/emqx_authn.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_access_control.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -341,12 +342,12 @@ t_check_undefined_expire(_Config) ->
?assertMatch(
{matched, allow},
emqx_authz_client_info:authorize(Client, subscribe, <<"a/b">>, undefined)
emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/b">>, undefined)
),
?assertMatch(
{matched, deny},
emqx_authz_client_info:authorize(Client, subscribe, <<"a/bar">>, undefined)
emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/bar">>, undefined)
).
%%------------------------------------------------------------------------------

189
apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
View File

@ -18,6 +18,8 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -44,6 +46,7 @@ init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok = emqx_authz_mnesia:purge_rules().
set_special_configs(emqx_authz) ->
@ -54,51 +57,135 @@ set_special_configs(_) ->
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_username_topic_rules(_Config) ->
ok = test_topic_rules(username).
t_clientid_topic_rules(_Config) ->
ok = test_topic_rules(clientid).
t_authz(_Config) ->
ClientInfo = emqx_authz_test_lib:base_client_info(),
t_all_topic_rules(_Config) ->
ok = test_topic_rules(all).
test_authz(
allow,
allow,
{all, #{
<<"permission">> => <<"allow">>, <<"action">> => <<"subscribe">>, <<"topic">> => <<"t">>
}},
{ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>}
),
test_authz(
allow,
allow,
{{username, <<"username">>}, #{
<<"permission">> => <<"allow">>,
<<"action">> => <<"subscribe">>,
<<"topic">> => <<"t/${username}">>
}},
{ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>}
),
test_authz(
allow,
allow,
{{username, <<"username">>}, #{
<<"permission">> => <<"allow">>,
<<"action">> => <<"subscribe">>,
<<"topic">> => <<"eq t/${username}">>
}},
{ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>}
),
test_authz(
deny,
deny,
{{username, <<"username">>}, #{
<<"permission">> => <<"allow">>,
<<"action">> => <<"subscribe">>,
<<"topic">> => <<"eq t/${username}">>
}},
{ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>}
),
test_authz(
allow,
allow,
{{clientid, <<"clientid">>}, #{
<<"permission">> => <<"allow">>,
<<"action">> => <<"subscribe">>,
<<"topic">> => <<"eq t/${username}">>
}},
{ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>}
),
test_authz(
allow,
allow,
{
{clientid, <<"clientid">>},
#{
<<"permission">> => <<"allow">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"t">>,
<<"qos">> => <<"1,2">>,
<<"retain">> => <<"true">>
}
},
{ClientInfo, ?AUTHZ_PUBLISH(1, true), <<"t">>}
),
test_authz(
deny,
allow,
{
{clientid, <<"clientid">>},
#{
<<"permission">> => <<"allow">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"t">>,
<<"qos">> => <<"1,2">>,
<<"retain">> => <<"true">>
}
},
{ClientInfo, ?AUTHZ_PUBLISH(0, true), <<"t">>}
),
test_authz(
deny,
allow,
{
{clientid, <<"clientid">>},
#{
<<"permission">> => <<"allow">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"t">>,
<<"qos">> => <<"1,2">>,
<<"retain">> => <<"true">>
}
},
{ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>}
).
test_topic_rules(Key) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
test_authz(Expected, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic}) ->
test_authz_with_rich_actions(true, Expected, {Who, Rule}, {ClientInfo, Action, Topic}),
test_authz_with_rich_actions(
false, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic}
).
SetupSamples = fun(CInfo, Samples) ->
setup_client_samples(CInfo, Samples, Key)
end,
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, SetupSamples),
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, SetupSamples),
ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, SetupSamples).
test_authz_with_rich_actions(
RichActionsEnabled, Expected, {Who, Rule}, {ClientInfo, Action, Topic}
) ->
ct:pal("Test authz rich_actions:~p~nwho:~p~nrule:~p~nattempt:~p~nexpected ~p", [
RichActionsEnabled, Who, Rule, {ClientInfo, Action, Topic}, Expected
]),
try
_ = emqx_authz:set_feature_available(rich_actions, RichActionsEnabled),
ok = emqx_authz_mnesia:store_rules(Who, [Rule]),
?assertEqual(Expected, emqx_access_control:authorize(ClientInfo, Action, Topic))
after
ok = emqx_authz_mnesia:purge_rules()
end.
t_normalize_rules(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo = emqx_authz_test_lib:base_client_info(),
ok = emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{allow, publish, "t"}]
[#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}]
),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
?assertException(
@ -106,25 +193,31 @@ t_normalize_rules(_Config) ->
{invalid_rule, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[[allow, publish, <<"t">>]]
[[<<"allow">>, <<"publish">>, <<"t">>]]
)
),
?assertException(
error,
{invalid_rule_action, _},
{invalid_action, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{allow, pub, <<"t">>}]
[#{<<"permission">> => <<"allow">>, <<"action">> => <<"pub">>, <<"topic">> => <<"t">>}]
)
),
?assertException(
error,
{invalid_rule_permission, _},
{invalid_permission, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{accept, publish, <<"t">>}]
[
#{
<<"permission">> => <<"accept">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"t">>
}
]
)
).
@ -138,27 +231,5 @@ raw_mnesia_authz_config() ->
<<"type">> => <<"built_in_database">>
}.
setup_client_samples(ClientInfo, Samples, Key) ->
ok = emqx_authz_mnesia:purge_rules(),
Rules = lists:flatmap(
fun(#{topics := Topics, permission := Permission, action := Action}) ->
lists:map(
fun(Topic) ->
{binary_to_atom(Permission), binary_to_atom(Action), Topic}
end,
Topics
)
end,
Samples
),
#{username := Username, clientid := ClientId} = ClientInfo,
Who =
case Key of
username -> {username, Username};
clientid -> {clientid, ClientId};
all -> all
end,
ok = emqx_authz_mnesia:store_rules(Who, Rules).
setup_config() ->
emqx_authz_test_lib:setup_config(raw_mnesia_authz_config(), #{}).

516
apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
View File

@ -28,10 +28,10 @@
-define(MONGO_CLIENT, 'emqx_authz_mongo_SUITE_client').
all() ->
emqx_common_test_helpers:all(?MODULE).
emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()).
groups() ->
[].
emqx_authz_test_lib:table_groups(t_run_case, cases()).
init_per_suite(Config) ->
ok = stop_apps([emqx_resource]),
@ -57,12 +57,18 @@ set_special_configs(emqx_authz) ->
set_special_configs(_) ->
ok.
init_per_group(Group, Config) ->
[{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].
end_per_group(_Group, _Config) ->
ok.
init_per_testcase(_TestCase, Config) ->
{ok, _} = mc_worker_api:connect(mongo_config()),
ok = emqx_authz_test_lib:reset_authorizers(),
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok = reset_samples(),
ok = mc_worker_api:disconnect(?MONGO_CLIENT).
@ -70,233 +76,313 @@ end_per_testcase(_TestCase, _Config) ->
%% Testcases
%%------------------------------------------------------------------------------
t_topic_rules(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
t_run_case(Config) ->
Case = ?config(test_case, Config),
ok = setup_source_data(Case),
ok = setup_authz_source(Case),
ok = emqx_authz_test_lib:run_checks(Case).
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2),
%%------------------------------------------------------------------------------
%% Cases
%%------------------------------------------------------------------------------
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2).
t_complex_filter(_) ->
%% atom and string values also supported
ClientInfo = #{
clientid => clientid,
username => "username",
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
Samples = [
cases() ->
[
#{
<<"x">> => #{
<<"u">> => <<"username">>,
<<"c">> => [#{<<"c">> => <<"clientid">>}],
<<"y">> => 1
},
<<"permission">> => <<"allow">>,
<<"action">> => <<"publish">>,
<<"topics">> => [<<"t">>]
}
],
ok = setup_samples(Samples),
ok = setup_config(
#{
<<"filter">> => #{
<<"x">> => #{
<<"u">> => <<"${username}">>,
<<"c">> => [#{<<"c">> => <<"${clientid}">>}],
<<"y">> => 1
name => base_publish,
records => [
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"subscribe">>,
<<"topic">> => <<"b">>,
<<"permission">> => <<"allow">>
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"all">>,
<<"topics">> => [<<"c">>, <<"d">>],
<<"permission">> => <<"allow">>
}
}
],
filter => #{<<"username">> => <<"${username}">>},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"b">>},
{allow, ?AUTHZ_PUBLISH, <<"c">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"c">>},
{allow, ?AUTHZ_PUBLISH, <<"d">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"d">>}
]
},
#{
name => filter_works,
records => [
#{
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
filter => #{<<"username">> => <<"${username}">>},
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_rich_rules,
features => [rich_actions],
records => [
#{
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>,
<<"qos">> => <<"1,2,3">>
},
#{
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>,
<<"retain">> => <<"yes">>
}
],
filter => #{},
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_rules,
records => [
#{
<<"action">> => <<"publis">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
filter => #{},
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => rule_by_clientid_cn_dn_peerhost,
records => [
#{
<<"cn">> => <<"cn">>,
<<"dn">> => <<"dn">>,
<<"clientid">> => <<"clientid">>,
<<"peerhost">> => <<"127.0.0.1">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
client_info => #{
cn => <<"cn">>,
dn => <<"dn">>
},
filter => #{
<<"cn">> => <<"${cert_common_name}">>,
<<"dn">> => <<"${cert_subject}">>,
<<"clientid">> => <<"${clientid}">>,
<<"peerhost">> => <<"${peerhost}">>
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => topics_literal_wildcard_variable,
records => [
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topics">> => [
<<"t/${username}">>,
<<"t/${clientid}">>,
<<"t1/#">>,
<<"t2/+">>,
<<"eq t3/${username}">>
]
}
],
filter => #{<<"username">> => <<"${username}">>},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>},
{allow, ?AUTHZ_PUBLISH, <<"t2/a">>},
{allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>},
{deny, ?AUTHZ_PUBLISH, <<"t3/username">>}
]
},
#{
name => qos_retain_in_query_result,
features => [rich_actions],
records => [
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"a">>,
<<"qos">> => 1,
<<"retain">> => true
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"b">>,
<<"qos">> => <<"1">>,
<<"retain">> => <<"true">>
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"c">>,
<<"qos">> => <<"1,2">>,
<<"retain">> => 1
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"d">>,
<<"qos">> => [1, 2],
<<"retain">> => <<"1">>
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"e">>,
<<"qos">> => [1, 2],
<<"retain">> => <<"all">>
},
#{
<<"username">> => <<"username">>,
<<"action">> => <<"publish">>,
<<"permission">> => <<"allow">>,
<<"topic">> => <<"f">>,
<<"qos">> => null,
<<"retain">> => null
}
],
filter => #{<<"username">> => <<"${username}">>},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"a">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"a">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"b">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"b">>},
{deny, ?AUTHZ_PUBLISH(2, false), <<"b">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"c">>},
{deny, ?AUTHZ_PUBLISH(2, false), <<"c">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"c">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"d">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"d">>},
{allow, ?AUTHZ_PUBLISH(1, false), <<"e">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"e">>},
{deny, ?AUTHZ_PUBLISH(0, false), <<"e">>},
{allow, ?AUTHZ_PUBLISH, <<"f">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"f">>}
]
},
#{
name => nonbin_values_in_client_info,
records => [
#{
<<"username">> => <<"username">>,
<<"clientid">> => <<"clientid">>,
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
client_info => #{
username => "username",
clientid => clientid
},
filter => #{<<"username">> => <<"${username}">>, <<"clientid">> => <<"${clientid}">>},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_query,
records => [
#{
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
filter => #{<<"$in">> => #{<<"a">> => 1}},
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => complex_query,
records => [
#{
<<"a">> => #{<<"u">> => <<"clientid">>, <<"c">> => [<<"cn">>, <<"dn">>]},
<<"action">> => <<"publish">>,
<<"topic">> => <<"a">>,
<<"permission">> => <<"allow">>
}
],
client_info => #{
cn => <<"cn">>,
dn => <<"dn">>
},
filter => #{
<<"a">> => #{
<<"u">> => <<"${clientid}">>,
<<"c">> => [<<"${cert_common_name}">>, <<"${cert_subject}">>]
}
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[{allow, publish, <<"t">>}]
).
t_mongo_error(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = setup_samples([]),
ok = setup_config(
#{<<"filter">> => #{<<"$badoperator">> => <<"$badoperator">>}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[{deny, publish, <<"t">>}]
).
t_lookups(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ByClientid = #{
<<"clientid">> => <<"clientid">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByClientid]),
ok = setup_config(
#{<<"filter">> => #{<<"clientid">> => <<"${clientid}">>}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByPeerhost = #{
<<"peerhost">> => <<"127.0.0.1">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByPeerhost]),
ok = setup_config(
#{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByCN = #{
<<"CN">> => <<"cn">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByCN]),
ok = setup_config(
#{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByDN = #{
<<"DN">> => <<"dn">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByDN]),
ok = setup_config(
#{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
t_bad_filter(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = setup_config(
#{<<"filter">> => #{<<"$in">> => #{<<"a">> => 1}}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{deny, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
].
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
populate_records(AclRecords, AdditionalData) ->
[maps:merge(Record, AdditionalData) || Record <- AclRecords].
setup_samples(AclRecords) ->
ok = reset_samples(),
{{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, AclRecords),
ok.
setup_client_samples(ClientInfo, Samples) ->
#{username := Username} = ClientInfo,
Records = lists:map(
fun(Sample) ->
#{
topics := Topics,
permission := Permission,
action := Action
} = Sample,
#{
<<"topics">> => Topics,
<<"permission">> => Permission,
<<"action">> => Action,
<<"username">> => Username
}
end,
Samples
),
setup_samples(Records),
setup_config(#{<<"filter">> => #{<<"username">> => <<"${username}">>}}).
reset_samples() ->
{true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"acl">>, #{}),
ok.
setup_source_data(#{records := Records}) ->
{{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, Records),
ok.
setup_authz_source(#{filter := Filter}) ->
setup_config(
#{
<<"filter">> => Filter
}
).
setup_config(SpecialParams) ->
emqx_authz_test_lib:setup_config(
raw_mongo_authz_config(),

586
apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
View File

@ -27,10 +27,10 @@
-define(MYSQL_RESOURCE, <<"emqx_authz_mysql_SUITE">>).
all() ->
emqx_common_test_helpers:all(?MODULE).
emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()).
groups() ->
[].
emqx_authz_test_lib:table_groups(t_run_case, cases()).
init_per_suite(Config) ->
ok = stop_apps([emqx_resource]),
@ -41,13 +41,7 @@ init_per_suite(Config) ->
fun set_special_configs/1
),
ok = start_apps([emqx_resource]),
{ok, _} = emqx_resource:create_local(
?MYSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_mysql,
mysql_config(),
#{}
),
ok = create_mysql_resource(),
Config;
false ->
{skip, no_mysql}
@ -59,9 +53,18 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource]),
ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
init_per_group(Group, Config) ->
[{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].
end_per_group(_Group, _Config) ->
ok.
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok = drop_table(),
ok.
set_special_configs(emqx_authz) ->
ok = emqx_authz_test_lib:reset_authorizers();
@ -72,189 +75,11 @@ set_special_configs(_) ->
%% Testcases
%%------------------------------------------------------------------------------
t_topic_rules(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2).
t_lookups(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
%% by clientid
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = ${clientid}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by peerhost
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(peerhost, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE peerhost = ${peerhost}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by cn
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(cn, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE cn = ${cert_common_name}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by dn
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(dn, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE dn = ${cert_subject}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% strip double quote support
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = \"${clientid}\""
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
t_mysql_error(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = setup_config(
#{<<"query">> => <<"SOME INVALID STATEMENT">>}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[{deny, subscribe, <<"a">>}]
).
t_run_case(Config) ->
Case = ?config(test_case, Config),
ok = setup_source_data(Case),
ok = setup_authz_source(Case),
ok = emqx_authz_test_lib:run_checks(Case).
t_create_invalid(_Config) ->
BadConfig = maps:merge(
@ -265,45 +90,307 @@ t_create_invalid(_Config) ->
[_] = emqx_authz:lookup().
t_nonbinary_values(_Config) ->
ClientInfo = #{
clientid => clientid,
username => "username",
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
%%------------------------------------------------------------------------------
%% Cases
%%------------------------------------------------------------------------------
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(clientid, username, topic, permission, action)"
"VALUES(?, ?, ?, ?, ?)"
>>,
[<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
cases() ->
[
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = ${clientid} AND username = ${username}"
>>
}
),
name => base_publish,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
client_info => #{username => <<"username">>},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"a">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"b">>}
]
},
#{
name => rule_by_clientid_cn_dn_peerhost,
setup => [
"CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255),"
" peerhost VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))",
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
"INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)"
" VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')"
],
query =>
"SELECT permission, action, topic FROM acl WHERE"
" clientid = ${clientid} AND cn = ${cert_common_name}"
" AND dn = ${cert_subject} AND peerhost = ${peerhost}",
client_info => #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
peerhost => {127, 0, 0, 1}
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>}
]
},
#{
name => topics_literal_wildcard_variable,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/${username}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/${clientid}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 'eq t/${username}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/#', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't1/+', 'allow', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{allow, ?AUTHZ_PUBLISH, <<"t/${username}">>},
{allow, ?AUTHZ_PUBLISH, <<"t/1/2">>},
{allow, ?AUTHZ_PUBLISH, <<"t1/1">>},
{deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>},
{deny, ?AUTHZ_PUBLISH, <<"abc">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>}
]
},
#{
name => qos_retain_in_query_result,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255),"
"qos_s VARCHAR(255), retain_s VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't1', 'allow', 'publish', '1', 'true')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't2', 'allow', 'publish', '2', 'false')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't4', 'allow', 'subscribe', '1', null)",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)"
],
query =>
"SELECT permission, action, topic, qos_s as qos, retain_s as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>},
{allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>},
{allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>},
{deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>},
{allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>},
{allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>},
{allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>}
]
},
#{
name => qos_retain_in_query_result_as_integer,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255),"
"qos_i VARCHAR(255), retain_i VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)"
" VALUES('username', 't1', 'allow', 'publish', 1, 1)"
],
query =>
"SELECT permission, action, topic, qos_i as qos, retain_i as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}
]
},
#{
name => retain_in_query_result_as_boolean,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255),"
" action VARCHAR(255), retain_b BOOLEAN)",
"INSERT INTO acl(username, topic, permission, action, retain_b)"
" VALUES('username', 't1', 'allow', 'publish', true)",
"INSERT INTO acl(username, topic, permission, action, retain_b)"
" VALUES('username', 't2', 'allow', 'publish', false)"
],
query =>
"SELECT permission, action, topic, retain_b as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>}
]
},
#{
name => nonbin_values_in_client_info,
setup => [
"CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255),"
" action VARCHAR(255))",
"INSERT INTO acl(who, topic, permission, action)"
" VALUES('username', 't/${username}', 'allow', 'publish')",
"INSERT INTO acl(who, topic, permission, action)"
" VALUES('clientid', 't/${clientid}', 'allow', 'publish')"
],
query =>
"SELECT permission, action, topic"
" FROM acl WHERE who = ${username} OR who = ${clientid}",
client_info => #{
%% string, not a binary
username => "username",
%% atom, not a binary
clientid => clientid
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{deny, ?AUTHZ_PUBLISH, <<"t/foo">>}
]
},
#{
name => null_retain_qos,
features => [rich_actions],
setup => [
"CREATE TABLE acl(qos VARCHAR(255), retain VARCHAR(255),"
" topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(qos, retain, topic, permission, action)"
" VALUES(NULL, NULL, 'tp', 'allow', 'publish')"
],
query =>
"SELECT permission, action, topic, qos FROM acl",
checks => [
{allow, ?AUTHZ_PUBLISH(0, false), <<"tp">>},
{allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"tp">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"xxx">>}
]
},
#{
name => strip_double_quote,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"",
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_query,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))"
],
query => "SELECT permission, action, topic FRO",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => runtime_error,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))"
],
query =>
"SELECT permission, action, topic FROM acl WHERE username = ${username}",
checks => [
fun() ->
_ = q("DROP TABLE IF EXISTS acl"),
{deny, ?AUTHZ_PUBLISH, <<"t">>}
end
]
},
#{
name => invalid_rule,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
%% 'permit' is invalid value for action
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
}
].
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
setup_source_data(#{setup := Queries}) ->
lists:foreach(
fun(Query) ->
_ = q(Query)
end,
Queries
).
setup_authz_source(#{query := Query}) ->
setup_config(
#{
<<"query">> => Query
}
).
raw_mysql_authz_config() ->
#{
<<"enable">> => <<"true">>,
@ -332,52 +419,9 @@ q(Sql, Params) ->
{sql, Sql, Params}
).
init_table() ->
ok = drop_table(),
ok = q(
"CREATE TABLE acl(\n"
" username VARCHAR(255),\n"
" clientid VARCHAR(255),\n"
" peerhost VARCHAR(255),\n"
" cn VARCHAR(255),\n"
" dn VARCHAR(255),\n"
" topic VARCHAR(255),\n"
" permission VARCHAR(255),\n"
" action VARCHAR(255))"
).
drop_table() ->
ok = q("DROP TABLE IF EXISTS acl").
setup_client_samples(ClientInfo, Samples) ->
#{username := Username} = ClientInfo,
ok = init_table(),
ok = lists:foreach(
fun(#{topics := Topics, permission := Permission, action := Action}) ->
lists:foreach(
fun(Topic) ->
q(
<<
"INSERT INTO acl(username, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[Username, Topic, Permission, Action]
)
end,
Topics
)
end,
Samples
),
setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE username = ${username}"
>>
}
).
setup_config(SpecialParams) ->
emqx_authz_test_lib:setup_config(
raw_mysql_authz_config(),
@ -400,3 +444,13 @@ start_apps(Apps) ->
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).
create_mysql_resource() ->
{ok, _} = emqx_resource:create_local(
?MYSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_mysql,
mysql_config(),
#{}
),
ok.

595
apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
View File

@ -27,10 +27,10 @@
-define(PGSQL_RESOURCE, <<"emqx_authz_pgsql_SUITE">>).
all() ->
emqx_common_test_helpers:all(?MODULE).
emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()).
groups() ->
[].
emqx_authz_test_lib:table_groups(t_run_case, cases()).
init_per_suite(Config) ->
ok = stop_apps([emqx_resource]),
@ -41,13 +41,7 @@ init_per_suite(Config) ->
fun set_special_configs/1
),
ok = start_apps([emqx_resource]),
{ok, _} = emqx_resource:create_local(
?PGSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_pgsql,
pgsql_config(),
#{}
),
{ok, _} = create_pgsql_resource(),
Config;
false ->
{skip, no_pgsql}
@ -59,9 +53,18 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource]),
ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
init_per_group(Group, Config) ->
[{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].
end_per_group(_Group, _Config) ->
ok.
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok = drop_table(),
ok.
set_special_configs(emqx_authz) ->
ok = emqx_authz_test_lib:reset_authorizers();
@ -72,194 +75,11 @@ set_special_configs(_) ->
%% Testcases
%%------------------------------------------------------------------------------
t_topic_rules(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2).
t_lookups(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
%% by clientid
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = ${clientid}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by peerhost
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(peerhost, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE peerhost = ${peerhost}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by cn
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(cn, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE cn = ${cert_common_name}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% by dn
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(dn, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE dn = ${cert_subject}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% strip double quote support
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = \"${clientid}\""
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
t_pgsql_error(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = ${username}"
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[{deny, subscribe, <<"a">>}]
).
t_run_case(Config) ->
Case = ?config(test_case, Config),
ok = setup_source_data(Case),
ok = setup_authz_source(Case),
ok = emqx_authz_test_lib:run_checks(Case).
t_create_invalid(_Config) ->
BadConfig = maps:merge(
@ -270,45 +90,304 @@ t_create_invalid(_Config) ->
[_] = emqx_authz:lookup().
t_nonbinary_values(_Config) ->
ClientInfo = #{
clientid => clientid,
username => "username",
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
%%------------------------------------------------------------------------------
%% Cases
%%------------------------------------------------------------------------------
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(clientid, username, topic, permission, action)"
"VALUES($1, $2, $3, $4, $5)"
>>,
[<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
cases() ->
[
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = ${clientid} AND username = ${username}"
>>
}
),
name => base_publish,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
client_info => #{username => <<"username">>},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"a">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"b">>}
]
},
#{
name => rule_by_clientid_cn_dn_peerhost,
setup => [
"CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255),"
" peerhost VARCHAR(255), topic VARCHAR(255),"
" permission VARCHAR(255), action VARCHAR(255))",
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
"INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)"
" VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')"
],
query =>
"SELECT permission, action, topic FROM acl WHERE"
" clientid = ${clientid} AND cn = ${cert_common_name}"
" AND dn = ${cert_subject} AND peerhost = ${peerhost}",
client_info => #{
clientid => <<"clientid">>,
cn => <<"cn">>,
dn => <<"dn">>,
peerhost => {127, 0, 0, 1}
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>}
]
},
#{
name => topics_literal_wildcard_variable,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/${username}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/${clientid}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 'eq t/${username}', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't/#', 'allow', 'publish')",
"INSERT INTO acl(username, topic, permission, action) "
"VALUES('username', 't1/+', 'allow', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{allow, ?AUTHZ_PUBLISH, <<"t/${username}">>},
{allow, ?AUTHZ_PUBLISH, <<"t/1/2">>},
{allow, ?AUTHZ_PUBLISH, <<"t1/1">>},
{deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>},
{deny, ?AUTHZ_PUBLISH, <<"abc">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>}
]
},
#{
name => qos_retain_in_query_result,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255),"
"qos_s VARCHAR(255), retain_s VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't1', 'allow', 'publish', '1', 'true')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't2', 'allow', 'publish', '2', 'false')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't4', 'allow', 'subscribe', '1', null)",
"INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)"
" VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)"
],
query =>
"SELECT permission, action, topic, qos_s as qos, retain_s as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>},
{allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>},
{allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>},
{allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>},
{deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>},
{allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>},
{allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>},
{allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>}
]
},
#{
name => qos_retain_in_query_result_as_integer,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255),"
"qos_i VARCHAR(255), retain_i VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)"
" VALUES('username', 't1', 'allow', 'publish', 1, 1)"
],
query =>
"SELECT permission, action, topic, qos_i as qos, retain_i as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}
]
},
#{
name => retain_in_query_result_as_boolean,
features => [rich_actions],
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255),"
" action VARCHAR(255), retain_b BOOLEAN)",
"INSERT INTO acl(username, topic, permission, action, retain_b)"
" VALUES('username', 't1', 'allow', 'publish', true)",
"INSERT INTO acl(username, topic, permission, action, retain_b)"
" VALUES('username', 't2', 'allow', 'publish', false)"
],
query =>
"SELECT permission, action, topic, retain_b as retain"
" FROM acl WHERE username = ${username}",
client_info => #{
username => <<"username">>
},
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>},
{allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>},
{deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>}
]
},
#{
name => nonbin_values_in_client_info,
setup => [
"CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255),"
" action VARCHAR(255))",
"INSERT INTO acl(who, topic, permission, action)"
" VALUES('username', 't/${username}', 'allow', 'publish')",
"INSERT INTO acl(who, topic, permission, action)"
" VALUES('clientid', 't/${clientid}', 'allow', 'publish')"
],
query =>
"SELECT permission, action, topic"
" FROM acl WHERE who = ${username} OR who = ${clientid}",
client_info => #{
%% string, not a binary
username => "username",
%% atom, not a binary
clientid => clientid
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{deny, ?AUTHZ_PUBLISH, <<"t/foo">>}
]
},
#{
name => array_null_qos,
features => [rich_actions],
setup => [
"CREATE TABLE acl(qos INTEGER[], "
" topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(qos, topic, permission, action)"
" VALUES('{1,2}', 'tp', 'allow', 'publish')",
"INSERT INTO acl(qos, topic, permission, action)"
" VALUES(NULL, 'ts', 'allow', 'subscribe')"
],
query =>
"SELECT permission, action, topic, qos FROM acl",
checks => [
{allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>},
{allow, ?AUTHZ_PUBLISH(2, false), <<"tp">>},
{deny, ?AUTHZ_PUBLISH(3, false), <<"tp">>},
{allow, ?AUTHZ_SUBSCRIBE(1), <<"ts">>},
{allow, ?AUTHZ_SUBSCRIBE(2), <<"ts">>}
]
},
#{
name => strip_double_quote,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"",
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_query,
setup => [],
query => "SELECT permission, action, topic FROM acl WHER",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => pgsql_error,
setup => [],
query =>
"SELECT permission, action, topic FROM table_not_exists WHERE username = ${username}",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"t">>}
]
},
#{
name => invalid_rule,
setup => [
"CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), "
"permission VARCHAR(255), action VARCHAR(255))",
%% 'permit' is invalid value for action
"INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')"
],
query => "SELECT permission, action, topic FROM acl WHERE username = ${username}",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
}
%% TODO: add case for unknown variables after fixing EMQX-10400
].
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
setup_source_data(#{setup := Queries}) ->
lists:foreach(
fun(Query) ->
_ = q(Query)
end,
Queries
).
setup_authz_source(#{query := Query}) ->
setup_config(
#{
<<"query">> => Query
}
).
raw_pgsql_authz_config() ->
#{
<<"enable">> => <<"true">>,
@ -331,61 +410,10 @@ q(Sql) ->
{query, Sql}
).
insert(Sql, Params) ->
{ok, _} = emqx_resource:simple_sync_query(
?PGSQL_RESOURCE,
{query, Sql, Params}
),
ok.
init_table() ->
ok = drop_table(),
{ok, _, _} = q(
"CREATE TABLE acl(\n"
" username VARCHAR(255),\n"
" clientid VARCHAR(255),\n"
" peerhost VARCHAR(255),\n"
" cn VARCHAR(255),\n"
" dn VARCHAR(255),\n"
" topic VARCHAR(255),\n"
" permission VARCHAR(255),\n"
" action VARCHAR(255))"
),
ok.
drop_table() ->
{ok, _, _} = q("DROP TABLE IF EXISTS acl"),
ok.
setup_client_samples(ClientInfo, Samples) ->
#{username := Username} = ClientInfo,
ok = init_table(),
ok = lists:foreach(
fun(#{topics := Topics, permission := Permission, action := Action}) ->
lists:foreach(
fun(Topic) ->
insert(
<<
"INSERT INTO acl(username, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[Username, Topic, Permission, Action]
)
end,
Topics
)
end,
Samples
),
setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE username = ${username}"
>>
}
).
setup_config(SpecialParams) ->
emqx_authz_test_lib:setup_config(
raw_pgsql_authz_config(),
@ -403,6 +431,15 @@ pgsql_config() ->
ssl => #{enable => false}
}.
create_pgsql_resource() ->
emqx_resource:create_local(
?PGSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_pgsql,
pgsql_config(),
#{}
).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).

363
apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
View File

@ -28,10 +28,10 @@
-define(REDIS_RESOURCE, <<"emqx_authz_redis_SUITE">>).
all() ->
emqx_common_test_helpers:all(?MODULE).
emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()).
groups() ->
[].
emqx_authz_test_lib:table_groups(t_run_case, cases()).
init_per_suite(Config) ->
ok = stop_apps([emqx_resource]),
@ -42,13 +42,7 @@ init_per_suite(Config) ->
fun set_special_configs/1
),
ok = start_apps([emqx_resource]),
{ok, _} = emqx_resource:create_local(
?REDIS_RESOURCE,
?RESOURCE_GROUP,
emqx_redis,
redis_config(),
#{}
),
ok = create_redis_resource(),
Config;
false ->
{skip, no_redis}
@ -60,9 +54,18 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource]),
ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
init_per_group(Group, Config) ->
[{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].
end_per_group(_Group, _Config) ->
ok.
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
_ = cleanup_redis(),
ok.
set_special_configs(emqx_authz) ->
ok = emqx_authz_test_lib:reset_authorizers();
@ -73,93 +76,11 @@ set_special_configs(_) ->
%% Tests
%%------------------------------------------------------------------------------
t_topic_rules(_Config) ->
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2),
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2).
t_lookups(_Config) ->
ClientInfo = #{
clientid => <<"client id">>,
cn => <<"cn">>,
dn => <<"dn">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ByClientid = #{
<<"mqtt_user:client id">> =>
#{<<"a">> => <<"all">>}
},
ok = setup_sample(ByClientid),
ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${clientid}">>}),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByPeerhost = #{
<<"mqtt_user:127.0.0.1">> =>
#{<<"a">> => <<"all">>}
},
ok = setup_sample(ByPeerhost),
ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${peerhost}">>}),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByCN = #{
<<"mqtt_user:cn">> =>
#{<<"a">> => <<"all">>}
},
ok = setup_sample(ByCN),
ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_common_name}">>}),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByDN = #{
<<"mqtt_user:dn">> =>
#{<<"a">> => <<"all">>}
},
ok = setup_sample(ByDN),
ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_subject}">>}),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
).
t_run_case(Config) ->
Case = ?config(test_case, Config),
ok = setup_source_data(Case),
ok = setup_authz_source(Case),
ok = emqx_authz_test_lib:run_checks(Case).
%% should still succeed to create even if the config will not work,
%% because it's not a part of the schema check
@ -181,7 +102,7 @@ t_create_with_config_values_wont_work(_Config) ->
InvalidConfigs
).
%% creating without a require field should return error
%% creating without a required field should return error
t_create_invalid_config(_Config) ->
AuthzConfig = raw_redis_authz_config(),
C = maps:without([<<"server">>], AuthzConfig),
@ -196,54 +117,211 @@ t_create_invalid_config(_Config) ->
t_redis_error(_Config) ->
ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}),
ClientInfo = #{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo = emqx_authz_test_lib:base_client_info(),
deny = emqx_access_control:authorize(ClientInfo, subscribe, <<"a">>).
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"a">>)
).
%%------------------------------------------------------------------------------
%% Cases
%%------------------------------------------------------------------------------
cases() ->
[
#{
name => base_publish,
setup => [
[
"HMSET",
"acl:username",
"a",
"publish",
"b",
"subscribe",
"d",
"all"
]
],
cmd => "HGETALL acl:${username}",
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"b">>},
{allow, ?AUTHZ_PUBLISH, <<"d">>},
{allow, ?AUTHZ_SUBSCRIBE, <<"d">>}
]
},
#{
name => invalid_rule,
setup => [
[
"HMSET",
"acl:username",
"a",
"[]",
"b",
"{invalid:json}",
"c",
"pub",
"d",
emqx_utils_json:encode(#{qos => 1, retain => true})
]
],
cmd => "HGETALL acl:${username}",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>},
{deny, ?AUTHZ_PUBLISH, <<"b">>},
{deny, ?AUTHZ_PUBLISH, <<"c">>},
{deny, ?AUTHZ_PUBLISH(1, true), <<"d">>}
]
},
#{
name => rule_by_clientid_cn_dn_peerhost,
setup => [
["HMSET", "acl:clientid:cn:dn:127.0.0.1", "a", "publish"]
],
cmd => "HGETALL acl:${clientid}:${cert_common_name}:${cert_subject}:${peerhost}",
client_info => #{
cn => <<"cn">>,
dn => <<"dn">>
},
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => topics_literal_wildcard_variable,
setup => [
[
"HMSET",
"acl:username",
"t/${username}",
"publish",
"t/${clientid}",
"publish",
"t1/#",
"publish",
"t2/+",
"publish",
"eq t3/${username}",
"publish"
]
],
cmd => "HGETALL acl:${username}",
checks => [
{allow, ?AUTHZ_PUBLISH, <<"t/username">>},
{allow, ?AUTHZ_PUBLISH, <<"t/clientid">>},
{allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>},
{allow, ?AUTHZ_PUBLISH, <<"t2/a">>},
{allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>},
{deny, ?AUTHZ_PUBLISH, <<"t3/username">>}
]
},
#{
name => qos_retain_in_query_result,
features => [rich_actions],
setup => [
[
"HMSET",
"acl:username",
"a",
emqx_utils_json:encode(#{action => <<"publish">>, qos => 1, retain => true}),
"b",
emqx_utils_json:encode(#{
action => <<"publish">>, qos => <<"1">>, retain => <<"true">>
}),
"c",
emqx_utils_json:encode(#{action => <<"publish">>, qos => <<"1,2">>, retain => 1}),
"d",
emqx_utils_json:encode(#{
action => <<"publish">>, qos => [1, 2], retain => <<"1">>
}),
"e",
emqx_utils_json:encode(#{
action => <<"publish">>, qos => [1, 2], retain => <<"all">>
}),
"f",
emqx_utils_json:encode(#{action => <<"publish">>, qos => null, retain => null})
]
],
cmd => "HGETALL acl:${username}",
checks => [
{allow, ?AUTHZ_PUBLISH(1, true), <<"a">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"a">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"b">>},
{deny, ?AUTHZ_PUBLISH(1, false), <<"b">>},
{deny, ?AUTHZ_PUBLISH(2, false), <<"b">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"c">>},
{deny, ?AUTHZ_PUBLISH(2, false), <<"c">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"c">>},
{allow, ?AUTHZ_PUBLISH(2, true), <<"d">>},
{deny, ?AUTHZ_PUBLISH(0, true), <<"d">>},
{allow, ?AUTHZ_PUBLISH(1, false), <<"e">>},
{allow, ?AUTHZ_PUBLISH(1, true), <<"e">>},
{deny, ?AUTHZ_PUBLISH(0, false), <<"e">>},
{allow, ?AUTHZ_PUBLISH, <<"f">>},
{deny, ?AUTHZ_SUBSCRIBE, <<"f">>}
]
},
#{
name => nonbin_values_in_client_info,
setup => [
[
"HMSET",
"acl:username:clientid",
"a",
"publish"
]
],
client_info => #{
username => "username",
clientid => clientid
},
cmd => "HGETALL acl:${username}:${clientid}",
checks => [
{allow, ?AUTHZ_PUBLISH, <<"a">>}
]
},
#{
name => invalid_query,
setup => [
["SET", "acl:username", 1]
],
cmd => "HGETALL acl:${username}",
checks => [
{deny, ?AUTHZ_PUBLISH, <<"a">>}
]
}
].
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
setup_sample(AuthzData) ->
{ok, _} = q(["FLUSHDB"]),
ok = lists:foreach(
fun({Key, Values}) ->
lists:foreach(
fun({TopicFilter, Action}) ->
q(["HSET", Key, TopicFilter, Action])
end,
maps:to_list(Values)
)
setup_source_data(#{setup := Queries}) ->
lists:foreach(
fun(Query) ->
_ = q(Query)
end,
maps:to_list(AuthzData)
Queries
).
setup_client_samples(ClientInfo, Samples) ->
#{username := Username} = ClientInfo,
Key = <<"mqtt_user:", Username/binary>>,
lists:foreach(
fun(Sample) ->
#{
topics := Topics,
permission := <<"allow">>,
action := Action
} = Sample,
lists:foreach(
fun(Topic) ->
q(["HSET", Key, Topic, Action])
end,
Topics
)
end,
Samples
),
setup_config(#{}).
setup_authz_source(#{cmd := Cmd}) ->
setup_config(
#{
<<"cmd">> => Cmd
}
).
setup_config(SpecialParams) ->
Config = maps:merge(raw_redis_authz_config(), SpecialParams),
@ -261,6 +339,9 @@ raw_redis_authz_config() ->
<<"server">> => <<?REDIS_HOST>>
}.
cleanup_redis() ->
q([<<"FLUSHALL">>]).
q(Command) ->
emqx_resource:simple_sync_query(
?REDIS_RESOURCE,
@ -283,3 +364,13 @@ start_apps(Apps) ->
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).
create_redis_resource() ->
{ok, _} = emqx_resource:create_local(
?REDIS_RESOURCE,
?RESOURCE_GROUP,
emqx_redis,
redis_config(),
#{}
),
ok.

696
apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
View File

@ -18,24 +18,17 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-define(SOURCE1, {deny, all}).
-define(SOURCE2, {allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}).
-define(SOURCE3, {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}).
-define(SOURCE4, {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}).
-define(SOURCE5,
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
).
-define(SOURCE6, {allow, {username, "test"}, publish, ["t/foo${username}boo"]}).
-define(CLIENT_INFO_BASE, #{
clientid => <<"test">>,
username => <<"test">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
}).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -59,6 +52,12 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
ok.
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok.
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
@ -68,11 +67,11 @@ set_special_configs(_App) ->
ok.
t_compile(_) ->
?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile(?SOURCE1)),
?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})),
?assertEqual(
{allow, {ipaddr, {{127, 0, 0, 1}, {127, 0, 0, 1}, 32}}, all, [{eq, ['#']}, {eq, ['+']}]},
emqx_authz_rule:compile(?SOURCE2)
emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]})
),
?assertEqual(
@ -82,14 +81,18 @@ t_compile(_) ->
{{192, 168, 1, 0}, {192, 168, 1, 255}, 24}
]},
subscribe, [{pattern, [{var, [<<"clientid">>]}]}]},
emqx_authz_rule:compile(?SOURCE3)
emqx_authz_rule:compile(
{allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}
)
),
?assertMatch(
?assertEqual(
{allow, {'and', [{clientid, {eq, <<"test">>}}, {username, {eq, <<"test">>}}]}, publish, [
[<<"topic">>, <<"test">>]
]},
emqx_authz_rule:compile(?SOURCE4)
emqx_authz_rule:compile(
{allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}
)
),
?assertMatch(
@ -101,240 +104,643 @@ t_compile(_) ->
publish, [
{pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]}
]},
emqx_authz_rule:compile(?SOURCE5)
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, publish, [
{pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]}
]},
emqx_authz_rule:compile(?SOURCE6)
emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]})
),
?assertEqual(
{allow, {username, {eq, <<"test">>}},
#{action_type => publish, qos => [0, 1, 2], retain => all}, [[<<"topic">>, <<"test">>]]},
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, #{action_type => publish, qos => [1], retain => true},
[
[<<"topic">>, <<"test">>]
]},
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1, 2]}, [
[<<"topic">>, <<"test">>]
]},
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, [{qos, 1}, {qos, 2}]}, ["topic/test"]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1]}, [
[<<"topic">>, <<"test">>]
]},
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, #{action_type => all, qos => [2], retain => true}, [
[<<"topic">>, <<"test">>]
]},
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
),
ok.
t_compile_ce(_Config) ->
_ = emqx_authz:set_feature_available(rich_actions, false),
?assertThrow(
{invalid_authorization_action, _},
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
),
?assertEqual(
{allow, {username, {eq, <<"test">>}}, all, [[<<"topic">>, <<"test">>]]},
emqx_authz_rule:compile(
{allow, {username, "test"}, all, ["topic/test"]}
)
).
t_match(_) ->
ClientInfo1 = #{
clientid => <<"test">>,
username => <<"test">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo2 = #{
clientid => <<"test">>,
username => <<"test">>,
peerhost => {192, 168, 1, 10},
zone => default,
listener => {tcp, default}
},
ClientInfo3 = #{
clientid => <<"test">>,
username => <<"fake">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
ClientInfo4 = #{
clientid => <<"fake">>,
username => <<"test">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
},
?assertEqual(
{matched, deny},
emqx_authz_rule:match(
client_info(),
#{action_type => subscribe, qos => 0},
<<"#">>,
emqx_authz_rule:compile({deny, all})
)
),
?assertEqual(
{matched, deny},
emqx_authz_rule:match(
ClientInfo1,
subscribe,
<<"#">>,
emqx_authz_rule:compile(?SOURCE1)
)
),
?assertEqual(
{matched, deny},
emqx_authz_rule:match(
ClientInfo2,
subscribe,
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => subscribe, qos => 0},
<<"+">>,
emqx_authz_rule:compile(?SOURCE1)
emqx_authz_rule:compile({deny, all})
)
),
?assertEqual(
{matched, deny},
emqx_authz_rule:match(
ClientInfo3,
subscribe,
client_info(#{username => <<"fake">>}),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE1)
emqx_authz_rule:compile({deny, all})
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo1,
subscribe,
client_info(),
#{action_type => subscribe, qos => 0},
<<"#">>,
emqx_authz_rule:compile(?SOURCE2)
emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo1,
subscribe,
client_info(),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE2)
emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo2,
subscribe,
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => subscribe, qos => 0},
<<"#">>,
emqx_authz_rule:compile(?SOURCE2)
emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]})
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo1,
subscribe,
client_info(),
#{action_type => subscribe, qos => 0},
<<"test">>,
emqx_authz_rule:compile(?SOURCE3)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo2,
subscribe,
<<"test">>,
emqx_authz_rule:compile(?SOURCE3)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo2,
subscribe,
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE3)
emqx_authz_rule:compile(
{allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo1,
publish,
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE4)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo2,
publish,
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE4)
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => subscribe, qos => 0},
<<"test">>,
emqx_authz_rule:compile(
{allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo3,
publish,
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE4)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo4,
publish,
<<"topic/test">>,
emqx_authz_rule:compile(?SOURCE4)
emqx_authz_rule:compile(
{allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo1,
publish,
<<"test">>,
emqx_authz_rule:compile(?SOURCE5)
client_info(),
#{action_type => publish, qos => 0, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo2,
publish,
<<"test">>,
emqx_authz_rule:compile(?SOURCE5)
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => publish, qos => 0, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{username => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo3,
publish,
client_info(),
#{action_type => publish, qos => 0, retain => false},
<<"test">>,
emqx_authz_rule:compile(?SOURCE5)
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo3,
publish,
client_info(#{peerhost => {192, 168, 1, 10}}),
#{action_type => publish, qos => 0, retain => false},
<<"test">>,
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{username => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"test">>,
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{username => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"fake">>,
emqx_authz_rule:compile(?SOURCE5)
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo4,
publish,
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"test">>,
emqx_authz_rule:compile(?SOURCE5)
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo4,
publish,
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"fake">>,
emqx_authz_rule:compile(?SOURCE5)
emqx_authz_rule:compile(
{allow,
{'or', [
{username, {re, "^test"}},
{clientid, {re, "test?"}}
]},
publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
ClientInfo1,
publish,
client_info(),
#{action_type => publish, qos => 0, retain => false},
<<"t/foo${username}boo">>,
emqx_authz_rule:compile(?SOURCE6)
emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]})
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
ClientInfo4,
publish,
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => false},
<<"t/footestboo">>,
emqx_authz_rule:compile(?SOURCE6)
emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]})
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 1, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 1, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 0, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 1, retain => false},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, []}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, []}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 1},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 0},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 1, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 2, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
)
),
?assertEqual(
{matched, allow},
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 2, retain => true},
<<"topic/test">>,
emqx_authz_rule:compile({allow, all, publish, ["#"]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile({allow, all, publish, ["#"]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{username => undefined, peerhost => undefined}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile({allow, {username, "user"}, all, ["#"]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{username => undefined, peerhost => undefined}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, ["#"]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{username => undefined, peerhost => undefined}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile({allow, {ipaddrs, []}, all, ["#"]})
)
),
?assertEqual(
nomatch,
emqx_authz_rule:match(
client_info(#{clientid => <<"fake">>}),
#{action_type => subscribe, qos => 2},
<<"topic/test">>,
emqx_authz_rule:compile({allow, {clientid, {re, "^test"}}, all, ["#"]})
)
),
ok.
t_invalid_rule(_) ->
?assertThrow(
{invalid_authorization_permission, _},
emqx_authz_rule:compile({allawww, all, all, ["topic/test"]})
),
?assertThrow(
{invalid_authorization_rule, _},
emqx_authz_rule:compile(ooops)
),
?assertThrow(
{invalid_authorization_qos, _},
emqx_authz_rule:compile({allow, {username, "test"}, {publish, [{qos, 3}]}, ["topic/test"]})
),
?assertThrow(
{invalid_authorization_retain, _},
emqx_authz_rule:compile(
{allow, {username, "test"}, {publish, [{retain, 'FALSE'}]}, ["topic/test"]}
)
),
?assertThrow(
{invalid_authorization_action, _},
emqx_authz_rule:compile({allow, all, unsubscribe, ["topic/test"]})
),
?assertThrow(
{invalid_who, _},
emqx_authz_rule:compile({allow, who, all, ["topic/test"]})
).
t_matches(_) ->
?assertEqual(
{matched, allow},
emqx_authz_rule:matches(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 2, retain => true},
<<"topic/test">>,
[
emqx_authz_rule:compile(
{allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]}
),
emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
)
]
)
),
Rule = emqx_authz_rule:compile(
{allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]}
),
?assertEqual(
nomatch,
emqx_authz_rule:matches(
client_info(#{clientid => <<"fake">>}),
#{action_type => publish, qos => 1, retain => true},
<<"topic/test">>,
[Rule, Rule, Rule]
)
).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
client_info() ->
?CLIENT_INFO_BASE.
client_info(Overrides) ->
maps:merge(?CLIENT_INFO_BASE, Overrides).

288
apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl Normal file
View File

@ -0,0 +1,288 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_rule_raw_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok.
t_parse_ok(_Config) ->
lists:foreach(
fun({Expected, RuleRaw}) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
?assertEqual({ok, Expected}, emqx_authz_rule_raw:parse_rule(RuleRaw)),
_ = emqx_authz:set_feature_available(rich_actions, false),
?assertEqual({ok, simple_rule(Expected)}, emqx_authz_rule_raw:parse_rule(RuleRaw))
end,
ok_cases()
).
t_parse_error(_Config) ->
emqx_authz:set_feature_available(rich_actions, true),
lists:foreach(
fun(RuleRaw) ->
?assertMatch(
{error, _},
emqx_authz_rule_raw:parse_rule(RuleRaw)
)
end,
error_cases() ++ error_rich_action_cases()
),
%% without rich actions some fields are not parsed, so they are not errors when invalid
_ = emqx_authz:set_feature_available(rich_actions, false),
lists:foreach(
fun(RuleRaw) ->
?assertMatch(
{error, _},
emqx_authz_rule_raw:parse_rule(RuleRaw)
)
end,
error_cases()
),
lists:foreach(
fun(RuleRaw) ->
?assertMatch(
{ok, _},
emqx_authz_rule_raw:parse_rule(RuleRaw)
)
end,
error_rich_action_cases()
).
t_format(_Config) ->
?assertEqual(
#{
action => subscribe,
permission => allow,
qos => [1, 2],
retain => true,
topic => [<<"a/b/c">>]
},
emqx_authz_rule_raw:format_rule(
{allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]}
)
),
?assertEqual(
#{
action => publish,
permission => allow,
topic => [<<"a/b/c">>]
},
emqx_authz_rule_raw:format_rule(
{allow, publish, [<<"a/b/c">>]}
)
).
t_format_no_rich_action(_Config) ->
_ = emqx_authz:set_feature_available(rich_actions, false),
Rule = {allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]},
?assertEqual(
#{action => subscribe, permission => allow, topic => [<<"a/b/c">>]},
emqx_authz_rule_raw:format_rule(Rule)
).
%%--------------------------------------------------------------------
%% Cases
%%--------------------------------------------------------------------
ok_cases() ->
[
{
{allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a/b/c">>]},
#{
<<"permission">> => <<"allow">>,
<<"topic">> => <<"a/b/c">>,
<<"action">> => <<"publish">>
}
},
{
{deny, {subscribe, [{qos, [1, 2]}]}, [{eq, <<"a/b/c">>}]},
#{
<<"permission">> => <<"deny">>,
<<"topic">> => <<"eq a/b/c">>,
<<"action">> => <<"subscribe">>,
<<"retain">> => <<"true">>,
<<"qos">> => <<"1,2">>
}
},
{
{allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a">>, <<"b">>]},
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [<<"a">>, <<"b">>],
<<"action">> => <<"publish">>
}
},
{
{allow, {all, [{qos, [0, 1, 2]}, {retain, all}]}, []},
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"all">>
}
},
%% Retain
{
expected_rule_with_qos_retain([0, 1, 2], true),
rule_with_raw_qos_retain(#{<<"retain">> => <<"true">>})
},
{
expected_rule_with_qos_retain([0, 1, 2], true),
rule_with_raw_qos_retain(#{<<"retain">> => true})
},
{
expected_rule_with_qos_retain([0, 1, 2], false),
rule_with_raw_qos_retain(#{<<"retain">> => false})
},
{
expected_rule_with_qos_retain([0, 1, 2], false),
rule_with_raw_qos_retain(#{<<"retain">> => <<"false">>})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{<<"retain">> => <<"all">>})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{<<"retain">> => undefined})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{<<"retain">> => null})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{})
},
{
expected_rule_with_qos_retain([0, 1, 2], true),
rule_with_raw_qos_retain(#{<<"retain">> => <<"1">>})
},
{
expected_rule_with_qos_retain([0, 1, 2], false),
rule_with_raw_qos_retain(#{<<"retain">> => <<"0">>})
},
%% Qos
{
expected_rule_with_qos_retain([2], all),
rule_with_raw_qos_retain(#{<<"qos">> => <<"2">>})
},
{
expected_rule_with_qos_retain([2], all),
rule_with_raw_qos_retain(#{<<"qos">> => [<<"2">>]})
},
{
expected_rule_with_qos_retain([1, 2], all),
rule_with_raw_qos_retain(#{<<"qos">> => <<"1,2">>})
},
{
expected_rule_with_qos_retain([1, 2], all),
rule_with_raw_qos_retain(#{<<"qos">> => [<<"1">>, <<"2">>]})
},
{
expected_rule_with_qos_retain([1, 2], all),
rule_with_raw_qos_retain(#{<<"qos">> => [1, 2]})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{<<"qos">> => undefined})
},
{
expected_rule_with_qos_retain([0, 1, 2], all),
rule_with_raw_qos_retain(#{<<"qos">> => null})
}
].
error_cases() ->
[
#{
<<"permission">> => <<"allo">>,
<<"topic">> => <<"a/b/c">>,
<<"action">> => <<"publish">>
},
#{
<<"permission">> => <<"allow">>,
<<"topic">> => <<"a/b/c">>,
<<"action">> => <<"publis">>
},
#{
<<"permission">> => <<"allow">>,
<<"topic">> => #{},
<<"action">> => <<"publish">>
},
#{
<<"permission">> => <<"allow">>,
<<"action">> => <<"publish">>
}
].
error_rich_action_cases() ->
[
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"publish">>,
<<"qos">> => 3
},
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"publish">>,
<<"qos">> => <<"three">>
},
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"publish">>,
<<"retain">> => 3
},
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"publish">>,
<<"qos">> => [<<"3">>]
}
].
expected_rule_with_qos_retain(QoS, Retain) ->
{allow, {publish, [{qos, QoS}, {retain, Retain}]}, []}.
rule_with_raw_qos_retain(Overrides) ->
maps:merge(base_raw_rule(), Overrides).
base_raw_rule() ->
#{
<<"permission">> => <<"allow">>,
<<"topics">> => [],
<<"action">> => <<"publish">>
}.
simple_rule({Pemission, {Action, _Opts}, Topics}) ->
{Pemission, Action, Topics}.

270
apps/emqx_authz/test/emqx_authz_test_lib.erl
View File

@ -22,8 +22,6 @@
-compile(nowarn_export_all).
-compile(export_all).
-define(DEFAULT_CHECK_AVAIL_TIMEOUT, 1000).
reset_authorizers() ->
reset_authorizers(deny, false, []).
@ -35,7 +33,7 @@ reset_authorizers(Nomatch, CacheEnabled, Source) ->
[authorization],
#{
<<"no_match">> => atom_to_binary(Nomatch),
<<"cache">> => #{<<"enable">> => atom_to_binary(CacheEnabled)},
<<"cache">> => #{<<"enable">> => CacheEnabled},
<<"sources">> => Source
}
),
@ -53,216 +51,70 @@ setup_config(BaseConfig, SpecialParams) ->
{error, Reason} -> {error, Reason}
end.
test_samples(ClientInfo, Samples) ->
%%--------------------------------------------------------------------
%% Table-based test helpers
%%--------------------------------------------------------------------
all_with_table_case(Mod, TableCase, Cases) ->
(emqx_common_test_helpers:all(Mod) -- [TableCase]) ++
[{group, Name} || Name <- case_names(Cases)].
table_groups(TableCase, Cases) ->
[{Name, [], [TableCase]} || Name <- case_names(Cases)].
case_names(Cases) ->
lists:map(fun(Case) -> maps:get(name, Case) end, Cases).
get_case(Name, Cases) ->
[Case] = [C || C <- Cases, maps:get(name, C) =:= Name],
Case.
setup_default_permission(Case) ->
DefaultPermission = maps:get(default_permission, Case, deny),
emqx_authz_test_lib:reset_authorizers(DefaultPermission, false).
base_client_info() ->
#{
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127, 0, 0, 1},
zone => default,
listener => {tcp, default}
}.
client_info(Overrides) ->
maps:merge(base_client_info(), Overrides).
enable_features(Case) ->
Features = maps:get(features, Case, []),
lists:foreach(
fun({Expected, Action, Topic}) ->
ct:pal(
"client_info: ~p, action: ~p, topic: ~p, expected: ~p",
[ClientInfo, Action, Topic, Expected]
),
?assertEqual(
Expected,
emqx_access_control:authorize(
ClientInfo,
Action,
Topic
)
)
fun(Feature) ->
Enable = lists:member(Feature, Features),
emqx_authz:set_feature_available(Feature, Enable)
end,
Samples
?AUTHZ_FEATURES
).
test_no_topic_rules(ClientInfo, SetupSamples) ->
%% No rules
ok = reset_authorizers(deny, false),
ok = SetupSamples(ClientInfo, []),
ok = test_samples(
ClientInfo,
[
{deny, subscribe, <<"#">>},
{deny, subscribe, <<"subs">>},
{deny, publish, <<"pub">>}
]
run_checks(#{checks := Checks} = Case) ->
_ = setup_default_permission(Case),
_ = enable_features(Case),
ClientInfoOverrides = maps:get(client_info, Case, #{}),
ClientInfo = client_info(ClientInfoOverrides),
lists:foreach(
fun(Check) ->
run_check(ClientInfo, Check)
end,
Checks
).
test_allow_topic_rules(ClientInfo, SetupSamples) ->
Samples = [
#{
topics => [
<<"eq testpub1/${username}">>,
<<"testpub2/${clientid}">>,
<<"testpub3/#">>
],
permission => <<"allow">>,
action => <<"publish">>
},
#{
topics => [
<<"eq testsub1/${username}">>,
<<"testsub2/${clientid}">>,
<<"testsub3/#">>
],
permission => <<"allow">>,
action => <<"subscribe">>
},
#{
topics => [
<<"eq testall1/${username}">>,
<<"testall2/${clientid}">>,
<<"testall3/#">>
],
permission => <<"allow">>,
action => <<"all">>
}
],
ok = reset_authorizers(deny, false),
ok = SetupSamples(ClientInfo, Samples),
ok = test_samples(
ClientInfo,
[
%% Publish rules
{deny, publish, <<"testpub1/username">>},
{allow, publish, <<"testpub1/${username}">>},
{allow, publish, <<"testpub2/clientid">>},
{allow, publish, <<"testpub3/foobar">>},
{deny, publish, <<"testpub2/username">>},
{deny, publish, <<"testpub1/clientid">>},
{deny, subscribe, <<"testpub1/username">>},
{deny, subscribe, <<"testpub2/clientid">>},
{deny, subscribe, <<"testpub3/foobar">>},
%% Subscribe rules
{deny, subscribe, <<"testsub1/username">>},
{allow, subscribe, <<"testsub1/${username}">>},
{allow, subscribe, <<"testsub2/clientid">>},
{allow, subscribe, <<"testsub3/foobar">>},
{allow, subscribe, <<"testsub3/+/foobar">>},
{allow, subscribe, <<"testsub3/#">>},
{deny, subscribe, <<"testsub2/username">>},
{deny, subscribe, <<"testsub1/clientid">>},
{deny, subscribe, <<"testsub4/foobar">>},
{deny, publish, <<"testsub1/username">>},
{deny, publish, <<"testsub2/clientid">>},
{deny, publish, <<"testsub3/foobar">>},
%% All rules
{deny, subscribe, <<"testall1/username">>},
{allow, subscribe, <<"testall1/${username}">>},
{allow, subscribe, <<"testall2/clientid">>},
{allow, subscribe, <<"testall3/foobar">>},
{allow, subscribe, <<"testall3/+/foobar">>},
{allow, subscribe, <<"testall3/#">>},
{deny, publish, <<"testall1/username">>},
{allow, publish, <<"testall1/${username}">>},
{allow, publish, <<"testall2/clientid">>},
{allow, publish, <<"testall3/foobar">>},
{deny, subscribe, <<"testall2/username">>},
{deny, subscribe, <<"testall1/clientid">>},
{deny, subscribe, <<"testall4/foobar">>},
{deny, publish, <<"testall2/username">>},
{deny, publish, <<"testall1/clientid">>},
{deny, publish, <<"testall4/foobar">>}
]
).
test_deny_topic_rules(ClientInfo, SetupSamples) ->
Samples = [
#{
topics => [
<<"eq testpub1/${username}">>,
<<"testpub2/${clientid}">>,
<<"testpub3/#">>
],
permission => <<"deny">>,
action => <<"publish">>
},
#{
topics => [
<<"eq testsub1/${username}">>,
<<"testsub2/${clientid}">>,
<<"testsub3/#">>
],
permission => <<"deny">>,
action => <<"subscribe">>
},
#{
topics => [
<<"eq testall1/${username}">>,
<<"testall2/${clientid}">>,
<<"testall3/#">>
],
permission => <<"deny">>,
action => <<"all">>
}
],
ok = reset_authorizers(allow, false),
ok = SetupSamples(ClientInfo, Samples),
ok = test_samples(
ClientInfo,
[
%% Publish rules
{allow, publish, <<"testpub1/username">>},
{deny, publish, <<"testpub1/${username}">>},
{deny, publish, <<"testpub2/clientid">>},
{deny, publish, <<"testpub3/foobar">>},
{allow, publish, <<"testpub2/username">>},
{allow, publish, <<"testpub1/clientid">>},
{allow, subscribe, <<"testpub1/username">>},
{allow, subscribe, <<"testpub2/clientid">>},
{allow, subscribe, <<"testpub3/foobar">>},
%% Subscribe rules
{allow, subscribe, <<"testsub1/username">>},
{deny, subscribe, <<"testsub1/${username}">>},
{deny, subscribe, <<"testsub2/clientid">>},
{deny, subscribe, <<"testsub3/foobar">>},
{deny, subscribe, <<"testsub3/+/foobar">>},
{deny, subscribe, <<"testsub3/#">>},
{allow, subscribe, <<"testsub2/username">>},
{allow, subscribe, <<"testsub1/clientid">>},
{allow, subscribe, <<"testsub4/foobar">>},
{allow, publish, <<"testsub1/username">>},
{allow, publish, <<"testsub2/clientid">>},
{allow, publish, <<"testsub3/foobar">>},
%% All rules
{allow, subscribe, <<"testall1/username">>},
{deny, subscribe, <<"testall1/${username}">>},
{deny, subscribe, <<"testall2/clientid">>},
{deny, subscribe, <<"testall3/foobar">>},
{deny, subscribe, <<"testall3/+/foobar">>},
{deny, subscribe, <<"testall3/#">>},
{allow, publish, <<"testall1/username">>},
{deny, publish, <<"testall1/${username}">>},
{deny, publish, <<"testall2/clientid">>},
{deny, publish, <<"testall3/foobar">>},
{allow, subscribe, <<"testall2/username">>},
{allow, subscribe, <<"testall1/clientid">>},
{allow, subscribe, <<"testall4/foobar">>},
{allow, publish, <<"testall2/username">>},
{allow, publish, <<"testall1/clientid">>},
{allow, publish, <<"testall4/foobar">>}
]
run_check(ClientInfo, Fun) when is_function(Fun, 0) ->
run_check(ClientInfo, Fun());
run_check(ClientInfo, {ExpectedPermission, Action, Topic}) ->
?assertEqual(
ExpectedPermission,
emqx_access_control:authorize(
ClientInfo,
Action,
Topic
)
).

3
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,13 +1,14 @@
%% -*- mode: erlang -*-
{application, emqx_bridge, [
{description, "EMQX bridges"},
{vsn, "0.1.22"},
{vsn, "0.1.23"},
{registered, [emqx_bridge_sup]},
{mod, {emqx_bridge_app, []}},
{applications, [
kernel,
stdlib,
emqx,
emqx_resource,
emqx_connector
]},
{env, []},

18
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -175,14 +175,14 @@ bridge_info_examples(Method) ->
value => info_example(mqtt, Method)
}
},
ee_bridge_examples(Method)
emqx_enterprise_bridge_examples(Method)
).
-if(?EMQX_RELEASE_EDITION == ee).
ee_bridge_examples(Method) ->
emqx_ee_bridge:examples(Method).
emqx_enterprise_bridge_examples(Method) ->
emqx_bridge_enterprise:examples(Method).
-else.
ee_bridge_examples(_Method) -> #{}.
emqx_enterprise_bridge_examples(_Method) -> #{}.
-endif.
info_example(Type, Method) ->
@ -985,9 +985,13 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
{error, timeout} ->
?SERVICE_UNAVAILABLE(<<"Request timeout">>);
{error, {start_pool_failed, Name, Reason}} ->
?SERVICE_UNAVAILABLE(
bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason]))
);
Msg = bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])),
case Reason of
nxdomain ->
?BAD_REQUEST(Msg);
_ ->
?SERVICE_UNAVAILABLE(Msg)
end;
{error, not_found} ->
BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
?SLOG(warning, #{

8
apps/emqx_bridge/src/emqx_bridge_app.erl
View File

@ -31,7 +31,7 @@
start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_bridge_sup:start_link(),
ok = start_ee_apps(),
ok = ensure_enterprise_schema_loaded(),
ok = emqx_bridge:load(),
ok = emqx_bridge:load_hook(),
ok = emqx_config_handler:add_handler(?LEAF_NODE_HDLR_PATH, ?MODULE),
@ -46,11 +46,11 @@ stop(_State) ->
ok.
-if(?EMQX_RELEASE_EDITION == ee).
start_ee_apps() ->
{ok, _} = application:ensure_all_started(emqx_ee_bridge),
ensure_enterprise_schema_loaded() ->
_ = emqx_bridge_enterprise:module_info(),
ok.
-else.
start_ee_apps() ->
ensure_enterprise_schema_loaded() ->
ok.
-endif.

2
apps/emqx_bridge/src/emqx_bridge_resource.erl
View File

@ -64,7 +64,7 @@ bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
bridge_to_resource_type(webhook) -> emqx_connector_http;
bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType).
bridge_to_resource_type(BridgeType) -> emqx_bridge_enterprise:resource_type(BridgeType).
-else.
bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;

16
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl → apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl
View File

@ -1,7 +1,9 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_bridge).
-module(emqx_bridge_enterprise).
-if(?EMQX_RELEASE_EDITION == ee).
-include_lib("hocon/include/hoconsc.hrl").
-import(hoconsc, [mk/2, enum/1, ref/2]).
@ -30,7 +32,7 @@ api_schemas(Method) ->
api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"),
api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"),
api_ref(emqx_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"),
api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method),
api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method),
api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"),
api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"),
api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"),
@ -54,7 +56,7 @@ schema_modules() ->
[
emqx_bridge_kafka,
emqx_bridge_cassandra,
emqx_ee_bridge_hstreamdb,
emqx_bridge_hstreamdb,
emqx_bridge_gcp_pubsub,
emqx_bridge_influxdb,
emqx_bridge_mongodb,
@ -93,7 +95,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer;
%% to hocon; keeping this as just `kafka' for backwards compatibility.
resource_type(kafka) -> emqx_bridge_kafka_impl_producer;
resource_type(cassandra) -> emqx_bridge_cassandra_connector;
resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
resource_type(hstreamdb) -> emqx_bridge_hstreamdb_connector;
resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_impl_producer;
resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer;
resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector;
@ -123,7 +125,7 @@ fields(bridges) ->
[
{hstreamdb,
mk(
hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")),
hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config")),
#{
desc => <<"HStreamDB Bridge Config">>,
required => false
@ -365,3 +367,7 @@ rabbitmq_structs() ->
api_ref(Module, Type, Method) ->
{Type, ref(Module, Method)}.
-else.
-endif.

33
apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
View File

@ -57,7 +57,7 @@ api_schema(Method) ->
{<<"mqtt">>, emqx_bridge_mqtt_schema}
]
],
EE = ee_api_schemas(Method),
EE = enterprise_api_schemas(Method),
hoconsc:union(bridge_api_union(Broker ++ EE)).
bridge_api_union(Refs) ->
@ -86,36 +86,23 @@ bridge_api_union(Refs) ->
end.
-if(?EMQX_RELEASE_EDITION == ee).
ee_api_schemas(Method) ->
ensure_loaded(emqx_ee_bridge, emqx_ee_bridge),
case erlang:function_exported(emqx_ee_bridge, api_schemas, 1) of
true -> emqx_ee_bridge:api_schemas(Method);
enterprise_api_schemas(Method) ->
case erlang:function_exported(emqx_bridge_enterprise, api_schemas, 1) of
true -> emqx_bridge_enterprise:api_schemas(Method);
false -> []
end.
ee_fields_bridges() ->
ensure_loaded(emqx_ee_bridge, emqx_ee_bridge),
case erlang:function_exported(emqx_ee_bridge, fields, 1) of
true -> emqx_ee_bridge:fields(bridges);
enterprise_fields_bridges() ->
case erlang:function_exported(emqx_bridge_enterprise, fields, 1) of
true -> emqx_bridge_enterprise:fields(bridges);
false -> []
end.
%% must ensure the app is loaded before checking if fn is defined.
ensure_loaded(App, Mod) ->
try
_ = application:load(App),
_ = Mod:module_info(),
ok
catch
_:_ ->
ok
end.
-else.
ee_api_schemas(_) -> [].
enterprise_api_schemas(_) -> [].
ee_fields_bridges() -> [].
enterprise_fields_bridges() -> [].
-endif.
@ -191,7 +178,7 @@ fields(bridges) ->
end
}
)}
] ++ ee_fields_bridges();
] ++ enterprise_fields_bridges();
fields("metrics") ->
[
{"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})},

10
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src
View File

@ -1,8 +1,14 @@
{application, emqx_bridge_cassandra, [
{description, "EMQX Enterprise Cassandra Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [kernel, stdlib, ecql]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
ecql
]},
{env, []},
{modules, []},
{links, []}

2
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
View File

@ -396,7 +396,7 @@ conn_opts([Opt | Opts], Acc) ->
%% prepare
%% XXX: hardcode
%% note: the `cql` param is passed by emqx_ee_bridge_cassa
%% note: the `cql` param is passed by emqx_bridge_cassandra
parse_prepare_cql(#{cql := SQL}) ->
parse_prepare_cql([{send_message, SQL}], #{}, #{});
parse_prepare_cql(_) ->

3
apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl
View File

@ -170,9 +170,8 @@ common_init(Config0) ->
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
_ = application:load(emqx_ee_bridge),
_ = emqx_ee_bridge:module_info(),
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
% Connect to cassnadra directly and create the table
catch connect_and_drop_table(Config0),

4
apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl
View File

@ -56,7 +56,6 @@ init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
{ok, _} = application:ensure_all_started(emqx_ee_connector),
%% keyspace `mqtt` must be created in advance
{ok, Conn} =
ecql:connect([
@ -79,8 +78,7 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector),
_ = application:stop(emqx_ee_connector).
_ = application:stop(emqx_connector).
init_per_testcase(_, Config) ->
Config.

2
apps/emqx_bridge_clickhouse/rebar.config
View File

@ -1,6 +1,6 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}
{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3.1"}}}
, {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}

10
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src
View File

@ -1,8 +1,14 @@
{application, emqx_bridge_clickhouse, [
{description, "EMQX Enterprise ClickHouse Bridge"},
{vsn, "0.2.1"},
{vsn, "0.2.2"},
{registered, []},
{applications, [kernel, stdlib, clickhouse, emqx_resource]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
clickhouse
]},
{env, []},
{modules, []},
{links, []}

2
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
View File

@ -469,7 +469,7 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) ->
reason => ClickhouseErrorResult
}),
case is_recoverable_error(ClickhouseErrorResult) of
%% TODO: The hackeny errors that the clickhouse library forwards are
%% TODO: The hackney errors that the clickhouse library forwards are
%% very loosely defined. We should try to make sure that the following
%% handles all error cases that we need to handle as recoverable_error
true ->

2
apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl
View File

@ -12,7 +12,7 @@
-include_lib("emqx_connector/include/emqx_connector.hrl").
%% See comment in
%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to
%% apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl for how to
%% run this without bringing up the whole CI infrastucture
%%------------------------------------------------------------------------------

10
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src
View File

@ -1,8 +1,14 @@
{application, emqx_bridge_dynamo, [
{description, "EMQX Enterprise Dynamo Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [kernel, stdlib, erlcloud]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
erlcloud
]},
{env, []},
{modules, []},
{links, []}

12
apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl
View File

@ -88,7 +88,7 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, erlcloud]),
ok.
init_per_testcase(TestCase, Config) ->
@ -128,10 +128,12 @@ common_init(ConfigT) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
_ = application:load(emqx_ee_bridge),
_ = emqx_ee_bridge:module_info(),
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([
emqx_conf, emqx_resource, emqx_bridge
]),
_ = application:ensure_all_started(erlcloud),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
% setup dynamo
setup_dynamo(Config0),

4
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src
View File

@ -1,10 +1,12 @@
{application, emqx_bridge_gcp_pubsub, [
{description, "EMQX Enterprise GCP Pub/Sub Bridge"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
ehttpc
]},
{env, []},

2
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl
View File

@ -21,7 +21,7 @@
service_account_json_converter/1
]).
%% emqx_ee_bridge "unofficial" API
%% emqx_bridge_enterprise "unofficial" API
-export([conn_bridge_examples/1]).
-type service_account_json() :: map().

2
lib-ee/emqx_ee_connector/docker-ct → apps/emqx_bridge_hstreamdb/docker-ct
View File

@ -1,2 +1,2 @@
toxiproxy
influxdb
hstreamdb

5
apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl Normal file
View File

@ -0,0 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-define(HSTREAMDB_DEFAULT_PORT, 6570).

4
lib-ee/emqx_ee_connector/rebar.config → apps/emqx_bridge_hstreamdb/rebar.config
View File

@ -1,11 +1,11 @@
%% -*- mode: erlang -*-
{erl_opts, [debug_info]}.
{deps, [
{hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
{hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.3.1+v0.12.0"}}},
{emqx, {path, "../../apps/emqx"}},
{emqx_utils, {path, "../../apps/emqx_utils"}}
]}.
{shell, [
{apps, [emqx_ee_connector]}
{apps, [emqx_bridge_hstreamdb]}
]}.

10
apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src
View File

@ -1,8 +1,14 @@
{application, emqx_bridge_hstreamdb, [
{description, "EMQX Enterprise HStreamDB Bridge"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, []},
{applications, [kernel, stdlib]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
hstreamdb_erl
]},
{env, []},
{modules, []},
{links, []}

109
apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl Normal file
View File

@ -0,0 +1,109 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_hstreamdb).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-import(hoconsc, [mk/2, enum/1, ref/2]).
-export([
conn_bridge_examples/1
]).
-export([
namespace/0,
roots/0,
fields/1,
desc/1
]).
%% -------------------------------------------------------------------------------------------------
%% api
conn_bridge_examples(Method) ->
[
#{
<<"hstreamdb">> => #{
summary => <<"HStreamDB Bridge">>,
value => values(Method)
}
}
].
values(get) ->
values(post);
values(put) ->
values(post);
values(post) ->
#{
type => <<"hstreamdb">>,
name => <<"demo">>,
direction => <<"egress">>,
url => <<"http://127.0.0.1:6570">>,
stream => <<"stream">>,
%% raw HRecord
record_template =>
<<"{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }">>,
pool_size => 8,
%% grpc_timeout => <<"1m">>
resource_opts => #{
query_mode => sync,
batch_size => 100,
batch_time => <<"20ms">>
},
ssl => #{enable => false}
};
values(_) ->
#{}.
%% -------------------------------------------------------------------------------------------------
%% Hocon Schema Definitions
namespace() -> "bridge_hstreamdb".
roots() -> [].
fields("config") ->
hstream_bridge_common_fields() ++
connector_fields();
fields("post") ->
hstream_bridge_common_fields() ++
connector_fields() ++
type_name_fields();
fields("get") ->
hstream_bridge_common_fields() ++
connector_fields() ++
type_name_fields() ++
emqx_bridge_schema:status_fields();
fields("put") ->
hstream_bridge_common_fields() ++
connector_fields().
hstream_bridge_common_fields() ->
emqx_bridge_schema:common_bridge_fields() ++
[
{direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
{local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
{record_template,
mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("record_template")})}
] ++
emqx_resource_schema:fields("resource_opts").
connector_fields() ->
emqx_bridge_hstreamdb_connector:fields(config).
desc("config") ->
?DESC("desc_config");
desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
["Configuration for HStreamDB bridge using `", string:to_upper(Method), "` method."];
desc(_) ->
undefined.
%% -------------------------------------------------------------------------------------------------
%% internal
type_name_fields() ->
[
{type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})},
{name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}
].

212
lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl → apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl
View File

@ -1,11 +1,12 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_connector_hstreamdb).
-module(emqx_bridge_hstreamdb_connector).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-import(hoconsc, [mk/2, enum/1]).
@ -17,6 +18,7 @@
on_start/2,
on_stop/2,
on_query/3,
on_batch_query/3,
on_get_status/2
]).
@ -28,10 +30,15 @@
namespace/0,
roots/0,
fields/1,
desc/1,
connector_examples/1
desc/1
]).
%% Allocatable resources
-define(hstreamdb_client, hstreamdb_client).
-define(DEFAULT_GRPC_TIMEOUT, timer:seconds(30)).
-define(DEFAULT_GRPC_TIMEOUT_RAW, <<"30s">>).
%% -------------------------------------------------------------------------------------------------
%% resource callback
callback_mode() -> always_sync.
@ -39,25 +46,52 @@ callback_mode() -> always_sync.
on_start(InstId, Config) ->
start_client(InstId, Config).
on_stop(InstId, #{client := Client, producer := Producer}) ->
StopClientRes = hstreamdb:stop_client(Client),
StopProducerRes = hstreamdb:stop_producer(Producer),
?SLOG(info, #{
msg => "stop hstreamdb connector",
connector => InstId,
client => Client,
producer => Producer,
stop_client => StopClientRes,
stop_producer => StopProducerRes
}).
on_stop(InstId, _State) ->
case emqx_resource:get_allocated_resources(InstId) of
#{client := Client, producer := Producer} ->
StopClientRes = hstreamdb:stop_client(Client),
StopProducerRes = hstreamdb:stop_producer(Producer),
?SLOG(info, #{
msg => "stop hstreamdb connector",
connector => InstId,
client => Client,
producer => Producer,
stop_client => StopClientRes,
stop_producer => StopProducerRes
});
_ ->
ok
end.
-define(FAILED_TO_APPLY_HRECORD_TEMPLATE,
{error, {unrecoverable_error, failed_to_apply_hrecord_template}}
).
on_query(
_InstId,
{send_message, Data},
#{producer := Producer, ordering_key := OrderingKey, payload := Payload}
_State = #{
producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate
}
) ->
Record = to_record(OrderingKey, Payload, Data),
do_append(Producer, Record).
try to_record(PartitionKey, HRecordTemplate, Data) of
Record -> append_record(Producer, Record)
catch
_:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE
end.
on_batch_query(
_InstId,
BatchList,
_State = #{
producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate
}
) ->
try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of
Records -> append_record(Producer, Records)
catch
_:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE
end.
on_get_status(_InstId, #{client := Client}) ->
case is_alive(Client) of
@ -87,43 +121,16 @@ fields(config) ->
[
{url, mk(binary(), #{required => true, desc => ?DESC("url")})},
{stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})},
{ordering_key, mk(binary(), #{required => false, desc => ?DESC("ordering_key")})},
{pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
];
fields("get") ->
fields("post");
fields("put") ->
fields(config);
fields("post") ->
[
{type, mk(hstreamdb, #{required => true, desc => ?DESC("type")})},
{name, mk(binary(), #{required => true, desc => ?DESC("name")})}
] ++ fields("put").
{partition_key, mk(binary(), #{required => false, desc => ?DESC("partition_key")})},
{pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})},
{grpc_timeout, fun grpc_timeout/1}
] ++ emqx_connector_schema_lib:ssl_fields().
connector_examples(Method) ->
[
#{
<<"hstreamdb">> => #{
summary => <<"HStreamDB Connector">>,
value => values(Method)
}
}
].
values(post) ->
maps:merge(values(put), #{name => <<"connector">>});
values(get) ->
values(post);
values(put) ->
#{
type => hstreamdb,
url => <<"http://127.0.0.1:6570">>,
stream => <<"stream1">>,
ordering_key => <<"some_key">>,
pool_size => 8
};
values(_) ->
#{}.
grpc_timeout(type) -> emqx_schema:timeout_duration_ms();
grpc_timeout(desc) -> ?DESC("grpc_timeout");
grpc_timeout(default) -> ?DEFAULT_GRPC_TIMEOUT_RAW;
grpc_timeout(required) -> false;
grpc_timeout(_) -> undefined.
desc(config) ->
?DESC("config").
@ -168,6 +175,10 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
}),
start_producer(InstId, Client, Config);
_ ->
?tp(
hstreamdb_connector_start_failed,
#{error => client_not_alive}
),
?SLOG(error, #{
msg => "hstreamdb connector: client not alive",
connector => InstId
@ -202,7 +213,7 @@ is_alive(Client) ->
start_producer(
InstId,
Client,
Options = #{stream := Stream, pool_size := PoolSize, egress := #{payload := PayloadBin}}
Options = #{stream := Stream, pool_size := PoolSize}
) ->
%% TODO: change these batch options after we have better disk cache.
BatchSize = maps:get(batch_size, Options, 100),
@ -212,7 +223,8 @@ start_producer(
{callback, {?MODULE, on_flush_result, []}},
{max_records, BatchSize},
{interval, Interval},
{pool_size, PoolSize}
{pool_size, PoolSize},
{grpc_timeout, maps:get(grpc_timeout, Options, ?DEFAULT_GRPC_TIMEOUT)}
],
Name = produce_name(InstId),
?SLOG(info, #{
@ -224,17 +236,18 @@ start_producer(
?SLOG(info, #{
msg => "hstreamdb connector: producer started"
}),
EnableBatch = maps:get(enable_batch, Options, false),
Payload = emqx_placeholder:preproc_tmpl(PayloadBin),
OrderingKeyBin = maps:get(ordering_key, Options, <<"">>),
OrderingKey = emqx_placeholder:preproc_tmpl(OrderingKeyBin),
State = #{
client => Client,
producer => Producer,
enable_batch => EnableBatch,
ordering_key => OrderingKey,
payload => Payload
enable_batch => maps:get(enable_batch, Options, false),
partition_key => emqx_placeholder:preproc_tmpl(
maps:get(partition_key, Options, <<"">>)
),
record_template => record_template(Options)
},
ok = emqx_resource:allocate_resource(InstId, ?hstreamdb_client, #{
client => Client, producer => Producer
}),
{ok, State};
{error, {already_started, Pid}} ->
?SLOG(info, #{
@ -253,47 +266,53 @@ start_producer(
{error, Reason}
end.
to_record(OrderingKeyTmpl, PayloadTmpl, Data) ->
OrderingKey = emqx_placeholder:proc_tmpl(OrderingKeyTmpl, Data),
Payload = emqx_placeholder:proc_tmpl(PayloadTmpl, Data),
to_record(OrderingKey, Payload).
to_record(PartitionKeyTmpl, HRecordTmpl, Data) ->
PartitionKey = emqx_placeholder:proc_tmpl(PartitionKeyTmpl, Data),
RawRecord = emqx_placeholder:proc_tmpl(HRecordTmpl, Data),
to_record(PartitionKey, RawRecord).
to_record(OrderingKey, Payload) when is_binary(OrderingKey) ->
to_record(binary_to_list(OrderingKey), Payload);
to_record(OrderingKey, Payload) ->
hstreamdb:to_record(OrderingKey, raw, Payload).
to_record(PartitionKey, RawRecord) when is_binary(PartitionKey) ->
to_record(binary_to_list(PartitionKey), RawRecord);
to_record(PartitionKey, RawRecord) ->
hstreamdb:to_record(PartitionKey, raw, RawRecord).
do_append(Producer, Record) ->
do_append(false, Producer, Record).
to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) ->
Records0 = lists:map(
fun({send_message, Data}) ->
to_record(PartitionKeyTmpl, HRecordTmpl, Data)
end,
BatchList
),
PartitionKeys = proplists:get_keys(Records0),
[
{PartitionKey, proplists:get_all_values(PartitionKey, Records0)}
|| PartitionKey <- PartitionKeys
].
%% TODO: this append is async, remove or change it after we have better disk cache.
% do_append(true, Producer, Record) ->
% case hstreamdb:append(Producer, Record) of
% ok ->
% ?SLOG(debug, #{
% msg => "hstreamdb producer async append success",
% record => Record
% });
% {error, Reason} = Err ->
% ?SLOG(error, #{
% msg => "hstreamdb producer async append failed",
% reason => Reason,
% record => Record
% }),
% Err
% end;
do_append(false, Producer, Record) ->
%% TODO: this append is sync, but it does not support [Record], can only append one Record.
%% Change it after we have better dick cache.
append_record(Producer, MultiPartsRecords) when is_list(MultiPartsRecords) ->
lists:foreach(fun(Record) -> append_record(Producer, Record) end, MultiPartsRecords);
append_record(Producer, Record) when is_tuple(Record) ->
do_append_records(false, Producer, Record).
%% TODO: only sync request supported. implement async request later.
do_append_records(false, Producer, Record) ->
case hstreamdb:append_flush(Producer, Record) of
{ok, _} ->
{ok, _Result} ->
?tp(
hstreamdb_connector_query_return,
#{result => _Result}
),
?SLOG(debug, #{
msg => "hstreamdb producer sync append success",
msg => "HStreamDB producer sync append success",
record => Record
});
{error, Reason} = Err ->
?tp(
hstreamdb_connector_query_return,
#{error => Reason}
),
?SLOG(error, #{
msg => "hstreamdb producer sync append failed",
msg => "HStreamDB producer sync append failed",
reason => Reason,
record => Record
}),
@ -306,6 +325,11 @@ client_name(InstId) ->
produce_name(ActionId) ->
list_to_atom("producer:" ++ to_string(ActionId)).
record_template(#{record_template := RawHRecordTemplate}) ->
emqx_placeholder:preproc_tmpl(RawHRecordTemplate);
record_template(_) ->
emqx_placeholder:preproc_tmpl(<<"${payload}">>).
to_string(List) when is_list(List) -> List;
to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin);
to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom).

578
apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl Normal file
View File

@ -0,0 +1,578 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_hstreamdb_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_bridge_hstreamdb.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
% SQL definitions
-define(STREAM, "stream").
-define(REPLICATION_FACTOR, 1).
%% in seconds
-define(BACKLOG_RETENTION_SECOND, (24 * 60 * 60)).
-define(SHARD_COUNT, 1).
-define(BRIDGE_NAME, <<"hstreamdb_demo_bridge">>).
-define(RECORD_TEMPLATE,
"{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }"
).
-define(POOL_SIZE, 8).
-define(BATCH_SIZE, 10).
-define(GRPC_TIMEOUT, "1s").
-define(WORKER_POOL_SIZE, 4).
-define(WITH_CLIENT(Process),
Client = connect_direct_hstream(_Name = test_c, Config),
Process,
ok = disconnect(Client)
).
%% How to run it locally (all commands are run in $PROJ_ROOT dir):
%% A: run ct on host
%% 1. Start all deps services
%% ```bash
%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \
%% -f .ci/docker-compose-file/docker-compose-hstreamdb.yaml \
%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
%% up --build
%% ```
%%
%% 2. Run use cases with special environment variables
%% 6570 is toxiproxy exported port.
%% Local:
%% ```bash
%% HSTREAMDB_HOST=$REAL_TOXIPROXY_IP HSTREAMDB_PORT=6570 \
%% PROXY_HOST=$REAL_TOXIPROXY_IP PROXY_PORT=6570 \
%% ./rebar3 as test ct -c -v --readable true --name ct@127.0.0.1 \
%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl
%% ```
%%
%% B: run ct in docker container
%% run script:
%% ```bash
%% ./scripts/ct/run.sh --ci --app apps/emqx_bridge_hstreamdb/ -- \
%% --name 'test@127.0.0.1' -c -v --readable true \
%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl
%% ````
%%------------------------------------------------------------------------------
%% CT boilerplate
%%------------------------------------------------------------------------------
all() ->
[
{group, sync}
].
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
NonBatchCases = [t_write_timeout],
BatchingGroups = [{group, with_batch}, {group, without_batch}],
[
{sync, BatchingGroups},
{with_batch, TCs -- NonBatchCases},
{without_batch, TCs}
].
init_per_group(sync, Config) ->
[{query_mode, sync} | Config];
init_per_group(with_batch, Config0) ->
Config = [{enable_batch, true} | Config0],
common_init(Config);
init_per_group(without_batch, Config0) ->
Config = [{enable_batch, false} | Config0],
common_init(Config);
init_per_group(_Group, Config) ->
Config.
end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
connect_and_delete_stream(Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
ok;
end_per_group(_Group, _Config) ->
ok.
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, hstreamdb_erl]),
ok.
init_per_testcase(t_to_hrecord_failed, Config) ->
meck:new([hstreamdb], [passthrough, no_history, no_link]),
meck:expect(hstreamdb, to_record, fun(_, _, _) -> error(trans_to_hrecord_failed) end),
Config;
init_per_testcase(_Testcase, Config) ->
%% drop stream and will create a new one in common_init/1
%% TODO: create a new stream for each test case
delete_bridge(Config),
snabbkaffe:start_trace(),
Config.
end_per_testcase(t_to_hrecord_failed, _Config) ->
meck:unload([hstreamdb]);
end_per_testcase(_Testcase, Config) ->
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
ok = snabbkaffe:stop(),
delete_bridge(Config),
ok.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_setup_via_config_and_publish(Config) ->
?assertMatch(
{ok, _},
create_bridge(Config)
),
Data = rand_data(),
?check_trace(
begin
?wait_async_action(
?assertEqual(ok, send_message(Config, Data)),
#{?snk_kind := hstreamdb_connector_query_return},
10_000
),
ok
end,
fun(Trace0) ->
Trace = ?of_kind(hstreamdb_connector_query_return, Trace0),
lists:foreach(
fun(EachTrace) ->
?assertMatch(#{result := #{streamName := <<?STREAM>>}}, EachTrace)
end,
Trace
),
ok
end
),
ok.
t_setup_via_http_api_and_publish(Config) ->
BridgeType = ?config(hstreamdb_bridge_type, Config),
Name = ?config(hstreamdb_name, Config),
HStreamDBConfig0 = ?config(hstreamdb_config, Config),
HStreamDBConfig = HStreamDBConfig0#{
<<"name">> => Name,
<<"type">> => BridgeType
},
?assertMatch(
{ok, _},
create_bridge_http(HStreamDBConfig)
),
Data = rand_data(),
?check_trace(
begin
?wait_async_action(
?assertEqual(ok, send_message(Config, Data)),
#{?snk_kind := hstreamdb_connector_query_return},
10_000
),
ok
end,
fun(Trace) ->
?assertMatch(
[#{result := #{streamName := <<?STREAM>>}}],
?of_kind(hstreamdb_connector_query_return, Trace)
)
end
),
ok.
t_get_status(Config) ->
?assertMatch(
{ok, _},
create_bridge(Config)
),
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyName = ?config(proxy_name, Config),
health_check_resource_ok(Config),
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
health_check_resource_down(Config)
end),
ok.
t_create_disconnected(Config) ->
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyName = ?config(proxy_name, Config),
?check_trace(
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
?assertMatch({ok, _}, create_bridge(Config))
end),
fun(Trace) ->
?assertMatch(
[#{error := client_not_alive}],
?of_kind(hstreamdb_connector_start_failed, Trace)
),
ok
end
),
%% TODO: Investigate why reconnection takes at least 5 seconds during ct.
%% While in practical applications, recovers to the 'connected' state
%% within 3 seconds after toxiproxy being enabled.'"
%% timer:sleep(10000),
restart_resource(Config),
health_check_resource_ok(Config),
ok.
t_write_failure(Config) ->
ProxyName = ?config(proxy_name, Config),
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),
QueryMode = ?config(query_mode, Config),
Data = rand_data(),
{{ok, _}, {ok, _}} =
?wait_async_action(
create_bridge(Config),
#{?snk_kind := resource_connected_enter},
20_000
),
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
health_check_resource_down(Config),
case QueryMode of
sync ->
?assertMatch(
{error, {resource_error, #{msg := "call resource timeout", reason := timeout}}},
send_message(Config, Data)
);
async ->
%% TODO: async mode is not supported yet,
%% but it will return ok if calling emqx_resource_buffer_worker:async_query/3,
?assertMatch(
ok,
send_message(Config, Data)
)
end
end),
ok.
t_simple_query(Config) ->
BatchSize = batch_size(Config),
?assertMatch(
{ok, _},
create_bridge(Config)
),
Requests = gen_batch_req(BatchSize),
?check_trace(
begin
?wait_async_action(
lists:foreach(
fun(Request) ->
?assertEqual(ok, query_resource(Config, Request))
end,
Requests
),
#{?snk_kind := hstreamdb_connector_query_return},
10_000
)
end,
fun(Trace0) ->
Trace = ?of_kind(hstreamdb_connector_query_return, Trace0),
lists:foreach(
fun(EachTrace) ->
?assertMatch(#{result := #{streamName := <<?STREAM>>}}, EachTrace)
end,
Trace
),
ok
end
),
ok.
t_to_hrecord_failed(Config) ->
QueryMode = ?config(query_mode, Config),
?assertMatch(
{ok, _},
create_bridge(Config)
),
Result = send_message(Config, #{}),
case QueryMode of
sync ->
?assertMatch(
{error, {unrecoverable_error, failed_to_apply_hrecord_template}},
Result
)
%% TODO: async mode is not supported yet
end,
ok.
%%------------------------------------------------------------------------------
%% Helper fns
%%------------------------------------------------------------------------------
common_init(ConfigT) ->
Host = os:getenv("HSTREAMDB_HOST", "toxiproxy"),
RawPort = os:getenv("HSTREAMDB_PORT", str(?HSTREAMDB_DEFAULT_PORT)),
Port = list_to_integer(RawPort),
URL = "http://" ++ Host ++ ":" ++ RawPort,
Config0 = [
{hstreamdb_host, Host},
{hstreamdb_port, Port},
{hstreamdb_url, URL},
%% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json
{proxy_name, "hstreamdb"},
{batch_size, batch_size(ConfigT)}
| ConfigT
],
BridgeType = proplists:get_value(bridge_type, Config0, <<"hstreamdb">>),
case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
true ->
% Setup toxiproxy
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]),
_ = application:ensure_all_started(hstreamdb_erl),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
% Connect to hstreamdb directly
% drop old stream and then create new one
connect_and_delete_stream(Config0),
connect_and_create_stream(Config0),
{Name, HStreamDBConf} = hstreamdb_config(BridgeType, Config0),
Config =
[
{hstreamdb_config, HStreamDBConf},
{hstreamdb_bridge_type, BridgeType},
{hstreamdb_name, Name},
{proxy_host, ProxyHost},
{proxy_port, ProxyPort}
| Config0
],
Config;
false ->
case os:getenv("IS_CI") of
"yes" ->
throw(no_hstreamdb);
_ ->
{skip, no_hstreamdb}
end
end.
hstreamdb_config(BridgeType, Config) ->
Port = integer_to_list(?config(hstreamdb_port, Config)),
URL = "http://" ++ ?config(hstreamdb_host, Config) ++ ":" ++ Port,
Name = ?BRIDGE_NAME,
BatchSize = batch_size(Config),
ConfigString =
io_lib:format(
"bridges.~s.~s {\n"
" enable = true\n"
" url = ~p\n"
" stream = ~p\n"
" record_template = ~p\n"
" pool_size = ~p\n"
" grpc_timeout = ~p\n"
" resource_opts = {\n"
%% always sync
" query_mode = sync\n"
" request_ttl = 500ms\n"
" batch_size = ~b\n"
" worker_pool_size = ~b\n"
" }\n"
"}",
[
BridgeType,
Name,
URL,
?STREAM,
?RECORD_TEMPLATE,
?POOL_SIZE,
?GRPC_TIMEOUT,
BatchSize,
?WORKER_POOL_SIZE
]
),
{Name, parse_and_check(ConfigString, BridgeType, Name)}.
parse_and_check(ConfigString, BridgeType, Name) ->
{ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
#{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf,
Config.
-define(RPC_OPTIONS, #{pool_size => 4}).
-define(CONN_ATTEMPTS, 10).
default_options(Config) ->
[
{url, ?config(hstreamdb_url, Config)},
{rpc_options, ?RPC_OPTIONS}
].
connect_direct_hstream(Name, Config) ->
client(Name, Config, ?CONN_ATTEMPTS).
client(_Name, _Config, N) when N =< 0 -> error(cannot_connect);
client(Name, Config, N) ->
try
_ = hstreamdb:stop_client(Name),
{ok, Client} = hstreamdb:start_client(Name, default_options(Config)),
{ok, echo} = hstreamdb:echo(Client),
Client
catch
Class:Error ->
ct:print("Error connecting: ~p", [{Class, Error}]),
ct:sleep(timer:seconds(1)),
client(Name, Config, N - 1)
end.
disconnect(Client) ->
hstreamdb:stop_client(Client).
create_bridge(Config) ->
create_bridge(Config, _Overrides = #{}).
create_bridge(Config, Overrides) ->
BridgeType = ?config(hstreamdb_bridge_type, Config),
Name = ?config(hstreamdb_name, Config),
HSDBConfig0 = ?config(hstreamdb_config, Config),
HSDBConfig = emqx_utils_maps:deep_merge(HSDBConfig0, Overrides),
emqx_bridge:create(BridgeType, Name, HSDBConfig).
delete_bridge(Config) ->
BridgeType = ?config(hstreamdb_bridge_type, Config),
Name = ?config(hstreamdb_name, Config),
emqx_bridge:remove(BridgeType, Name).
create_bridge_http(Params) ->
Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
{ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
Error -> Error
end.
send_message(Config, Data) ->
Name = ?config(hstreamdb_name, Config),
BridgeType = ?config(hstreamdb_bridge_type, Config),
BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
emqx_bridge:send_message(BridgeID, Data).
query_resource(Config, Request) ->
Name = ?config(hstreamdb_name, Config),
BridgeType = ?config(hstreamdb_bridge_type, Config),
ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
emqx_resource:query(ResourceID, Request, #{timeout => 1_000}).
restart_resource(Config) ->
BridgeName = ?config(hstreamdb_name, Config),
BridgeType = ?config(hstreamdb_bridge_type, Config),
emqx_bridge:disable_enable(disable, BridgeType, BridgeName),
timer:sleep(200),
emqx_bridge:disable_enable(enable, BridgeType, BridgeName).
resource_id(Config) ->
BridgeName = ?config(hstreamdb_name, Config),
BridgeType = ?config(hstreamdb_bridge_type, Config),
_ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName).
health_check_resource_ok(Config) ->
?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))).
health_check_resource_down(Config) ->
case emqx_resource_manager:health_check(resource_id(Config)) of
{ok, Status} when Status =:= disconnected orelse Status =:= connecting ->
ok;
{error, timeout} ->
ok;
Other ->
?assert(
false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other]))
)
end.
% These funs start and then stop the hstreamdb connection
connect_and_create_stream(Config) ->
?WITH_CLIENT(
_ = hstreamdb:create_stream(
Client, ?STREAM, ?REPLICATION_FACTOR, ?BACKLOG_RETENTION_SECOND, ?SHARD_COUNT
)
),
%% force write to stream to make it created and ready to be written data for rest cases
ProducerOptions = [
{pool_size, 4},
{stream, ?STREAM},
{callback, fun(_) -> ok end},
{max_records, 10},
{interval, 1000}
],
?WITH_CLIENT(
begin
{ok, Producer} = hstreamdb:start_producer(Client, test_producer, ProducerOptions),
_ = hstreamdb:append_flush(Producer, hstreamdb:to_record([], raw, rand_payload())),
_ = hstreamdb:stop_producer(Producer)
end
).
connect_and_delete_stream(Config) ->
?WITH_CLIENT(
_ = hstreamdb:delete_stream(Client, ?STREAM)
).
%%--------------------------------------------------------------------
%% help functions
%%--------------------------------------------------------------------
batch_size(Config) ->
case ?config(enable_batch, Config) of
true -> ?BATCH_SIZE;
false -> 1
end.
rand_data() ->
#{
%% Raw MTTT Payload in binary
payload => rand_payload(),
id => <<"0005F8F84FFFAFB9F44200000D810002">>,
topic => <<"test/topic">>,
qos => 0
}.
rand_payload() ->
emqx_utils_json:encode(#{
temperature => rand:uniform(40), humidity => rand:uniform(100)
}).
gen_batch_req(Count) when
is_integer(Count) andalso Count > 0
->
[{send_message, rand_data()} || _Val <- lists:seq(1, Count)];
gen_batch_req(Count) ->
ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]).
str(List) when is_list(List) ->
unicode:characters_to_list(List, utf8);
str(Bin) when is_binary(Bin) ->
unicode:characters_to_list(Bin, utf8);
str(Num) when is_number(Num) ->
number_to_list(Num).
number_to_list(Int) when is_integer(Int) ->
integer_to_list(Int);
number_to_list(Float) when is_float(Float) ->
float_to_list(Float, [{decimals, 10}, compact]).

10
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src
View File

@ -1,8 +1,14 @@
{application, emqx_bridge_influxdb, [
{description, "EMQX Enterprise InfluxDB Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [kernel, stdlib, influxdb]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
influxdb
]},
{env, []},
{modules, []},
{links, []}

6
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
View File

@ -638,8 +638,10 @@ value_type([UInt, <<"u">>]) when
is_integer(UInt)
->
{uint, UInt};
value_type([Float]) when is_float(Float) ->
Float;
%% write `1`, `1.0`, `-1.0` all as float
%% see also: https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float
value_type([Number]) when is_number(Number) ->
Number;
value_type([<<"t">>]) ->
't';
value_type([<<"T">>]) ->

77
apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl
View File

@ -454,24 +454,26 @@ query_by_clientid(ClientId, Config) ->
{ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}),
DecodedCSV1 = [
[Field || Field <- Line, Field =/= <<>>]
|| Line <- DecodedCSV0,
Line =/= [<<>>]
|| Line <- DecodedCSV0, Line =/= [<<>>]
],
DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []),
DecodedCSV2 = csv_lines_to_maps(DecodedCSV1),
index_by_field(DecodedCSV2).
decode_csv(RawBody) ->
Lines =
[
binary:split(Line, [<<";">>], [global, trim_all])
|| Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all])
],
csv_lines_to_maps(Lines, []).
csv_lines_to_maps([Title | Rest]) ->
csv_lines_to_maps(Rest, Title, _Acc = []);
csv_lines_to_maps([]) ->
[].
csv_lines_to_maps([Fields, Data | Rest], Acc) ->
Map = maps:from_list(lists:zip(Fields, Data)),
csv_lines_to_maps(Rest, [Map | Acc]);
csv_lines_to_maps(_Data, Acc) ->
csv_lines_to_maps([[<<"_result">> | _] = Data | RestData], Title, Acc) ->
Map = maps:from_list(lists:zip(Title, Data)),
csv_lines_to_maps(RestData, Title, [Map | Acc]);
%% ignore the csv title line
%% it's always like this:
%% [<<"result">>,<<"table">>,<<"_start">>,<<"_stop">>,
%% <<"_time">>,<<"_value">>,<<"_field">>,<<"_measurement">>, Measurement],
csv_lines_to_maps([[<<"result">> | _] = _Title | RestData], Title, Acc) ->
csv_lines_to_maps(RestData, Title, Acc);
csv_lines_to_maps([], _Title, Acc) ->
lists:reverse(Acc).
index_by_field(DecodedCSV) ->
@ -768,6 +770,53 @@ t_boolean_variants(Config) ->
),
ok.
t_any_num_as_float(Config) ->
QueryMode = ?config(query_mode, Config),
Const = erlang:system_time(nanosecond),
ConstBin = integer_to_binary(Const),
TsStr = iolist_to_binary(
calendar:system_time_to_rfc3339(Const, [{unit, nanosecond}, {offset, "Z"}])
),
?assertMatch(
{ok, _},
create_bridge(
Config,
#{
<<"write_syntax">> =>
<<"mqtt,clientid=${clientid}", " ",
"float_no_dp=${payload.float_no_dp},float_dp=${payload.float_dp},bar=5i ",
ConstBin/binary>>
}
)
),
ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
Payload = #{
%% no decimal point
float_no_dp => 123,
%% with decimal point
float_dp => 123.0
},
SentData = #{
<<"clientid">> => ClientId,
<<"topic">> => atom_to_binary(?FUNCTION_NAME),
<<"payload">> => Payload,
<<"timestamp">> => erlang:system_time(millisecond)
},
case QueryMode of
sync ->
?assertMatch({ok, 204, _}, send_message(Config, SentData)),
ok;
async ->
?assertMatch(ok, send_message(Config, SentData)),
ct:sleep(500)
end,
PersistedData = query_by_clientid(ClientId, Config),
Expected = #{float_no_dp => <<"123">>, float_dp => <<"123">>},
assert_persisted_data(ClientId, Expected, PersistedData),
TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"float_no_dp">>, PersistedData)),
TimeReturned = pad_zero(TimeReturned0),
?assertEqual(TsStr, TimeReturned).
t_bad_timestamp(Config) ->
InfluxDBType = ?config(influxdb_type, Config),
InfluxDBName = ?config(influxdb_name, Config),

5
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_iotdb, [
{description, "EMQX Enterprise Apache IoTDB Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{modules, [
emqx_bridge_iotdb,
emqx_bridge_iotdb_impl
@ -10,6 +10,9 @@
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
%% for module emqx_connector_http
emqx_connector
]},
{env, []},

2
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl
View File

@ -18,7 +18,7 @@
desc/1
]).
%% emqx_ee_bridge "unofficial" API
%% emqx_bridge_enterprise "unofficial" API
-export([conn_bridge_examples/1]).
%%-------------------------------------------------------------------------------------------------

5
apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
View File

@ -1,10 +1,13 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_kafka, [
{description, "EMQX Enterprise Kafka Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, [emqx_bridge_kafka_consumer_sup]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
telemetry,
wolff,
brod,

2
apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
View File

@ -40,7 +40,7 @@ query_mode(_) ->
callback_mode() -> async_if_possible.
%% @doc Config schema is defined in emqx_ee_bridge_kafka.
%% @doc Config schema is defined in emqx_bridge_kafka.
on_start(InstId, Config) ->
#{
authentication := Auth,

8
apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
View File

@ -73,11 +73,9 @@ wait_until_kafka_is_up(Attempts) ->
end.
init_per_suite(Config) ->
%% ensure loaded
_ = application:load(emqx_ee_bridge),
_ = emqx_ee_bridge:module_info(),
application:load(emqx_bridge),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
%% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
_ = emqx_bridge_enterprise:module_info(),
ok = emqx_connector_test_helpers:start_apps(?APPS),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),

9
apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src
View File

@ -1,8 +1,13 @@
{application, emqx_bridge_matrix, [
{description, "EMQX Enterprise MatrixDB Bridge"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [kernel, stdlib]},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge
]},
{env, []},
{modules, []},
{links, []}

3
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_mongodb, [
{description, "EMQX Enterprise MongoDB Bridge"},
{vsn, "0.2.0"},
{vsn, "0.2.1"},
{registered, []},
{applications, [
kernel,
@ -8,7 +8,6 @@
emqx_connector,
emqx_resource,
emqx_bridge,
emqx_ee_bridge,
emqx_mongodb
]},
{env, []},

2
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl
View File

@ -10,7 +10,7 @@
-behaviour(hocon_schema).
%% emqx_ee_bridge "callbacks"
%% emqx_bridge_enterprise "callbacks"
-export([
conn_bridge_examples/1
]).

2
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl
View File

@ -58,7 +58,7 @@ on_query(InstanceId, {send_message, Message0}, State) ->
},
Message = render_message(PayloadTemplate, Message0),
Res = emqx_mongodb:on_query(InstanceId, {send_message, Message}, NewConnectorState),
?tp(mongo_ee_connector_on_query_return, #{result => Res}),
?tp(mongo_bridge_connector_on_query_return, #{result => Res}),
Res;
on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) ->
emqx_mongodb:on_query(InstanceId, Request, ConnectorState).

13
apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl
View File

@ -116,7 +116,7 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf, emqx_rule_engine]),
ok = emqx_common_test_helpers:stop_apps([emqx_mongodb, emqx_bridge, emqx_rule_engine, emqx_conf]),
ok.
init_per_testcase(_Testcase, Config) ->
@ -146,9 +146,8 @@ start_apps() ->
]).
ensure_loaded() ->
_ = application:load(emqx_ee_bridge),
_ = application:load(emqtt),
_ = emqx_ee_bridge:module_info(),
_ = emqx_bridge_enterprise:module_info(),
ok.
mongo_type(Config) ->
@ -354,7 +353,7 @@ t_setup_via_config_and_publish(Config) ->
{ok, {ok, _}} =
?wait_async_action(
send_message(Config, #{key => Val}),
#{?snk_kind := mongo_ee_connector_on_query_return},
#{?snk_kind := mongo_bridge_connector_on_query_return},
5_000
),
?assertMatch(
@ -379,7 +378,7 @@ t_setup_via_http_api_and_publish(Config) ->
{ok, {ok, _}} =
?wait_async_action(
send_message(Config, #{key => Val}),
#{?snk_kind := mongo_ee_connector_on_query_return},
#{?snk_kind := mongo_bridge_connector_on_query_return},
5_000
),
?assertMatch(
@ -395,7 +394,7 @@ t_payload_template(Config) ->
{ok, {ok, _}} =
?wait_async_action(
send_message(Config, #{key => Val, clientid => ClientId}),
#{?snk_kind := mongo_ee_connector_on_query_return},
#{?snk_kind := mongo_bridge_connector_on_query_return},
5_000
),
?assertMatch(
@ -421,7 +420,7 @@ t_collection_template(Config) ->
clientid => ClientId,
mycollectionvar => <<"mycol">>
}),
#{?snk_kind := mongo_ee_connector_on_query_return},
#{?snk_kind := mongo_bridge_connector_on_query_return},
5_000
),
?assertMatch(

11
apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src
View File

@ -1,8 +1,15 @@
{application, emqx_bridge_mysql, [
{description, "EMQX Enterprise MySQL Bridge"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_mysql]},
{applications, [
kernel,
stdlib,
emqx_connector,
emqx_resource,
emqx_bridge,
emqx_mysql
]},
{env, []},
{modules, []},
{links, []}

5
apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl
View File

@ -142,10 +142,9 @@ common_init(Config0) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
_ = application:load(emqx_ee_bridge),
_ = emqx_ee_bridge:module_info(),
% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, emqx_rule_engine]),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
% Connect to mysql directly and create the table
connect_and_create_table(Config0),

4
apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src
View File

@ -1,10 +1,12 @@
{application, emqx_bridge_opents, [
{description, "EMQX Enterprise OpenTSDB Bridge"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
opentsdb
]},
{env, []},

12
apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl
View File

@ -53,7 +53,7 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
ok = emqx_common_test_helpers:stop_apps([opentsdb, emqx_bridge, emqx_resource, emqx_conf]),
ok.
init_per_testcase(_Testcase, Config) ->
@ -91,10 +91,12 @@ common_init(ConfigT) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
_ = application:load(emqx_ee_bridge),
_ = emqx_ee_bridge:module_info(),
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([
emqx_conf, emqx_resource, emqx_bridge
]),
_ = application:ensure_all_started(opentsdb),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
{Name, OpenTSConf} = opents_config(BridgeType, Config0),
Config =

4
apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src
View File

@ -1,10 +1,12 @@
{application, emqx_bridge_oracle, [
{description, "EMQX Enterprise Oracle Database Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [
kernel,
stdlib,
emqx_resource,
emqx_bridge,
emqx_oracle
]},
{env, []},

5
apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl
View File

@ -83,8 +83,9 @@ common_init_per_group() ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
application:load(emqx_bridge),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
%% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
_ = emqx_bridge_enterprise:module_info(),
ok = emqx_connector_test_helpers:start_apps(?APPS),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),

Some files were not shown because too many files have changed in this diff Show More