yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into release-53

This commit is contained in:
Zaiming (Stone) Shi 2023-12-01 16:55:22 +01:00
parent e5e8384515 956b25cc4c
commit 4fed23cc06
588 changed files with 38291 additions and 4668 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.ci/docker-compose-file/credentials.env Normal file
View File

@ -0,0 +1,7 @@
MONGO_USERNAME=emqx
MONGO_PASSWORD=passw0rd
MONGO_AUTHSOURCE=admin
# See "Environment Variables" @ https://hub.docker.com/_/mongo
MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}

3
.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml
View File

@ -9,6 +9,9 @@ services:
- emqx_bridge
ports:
- "27017:27017"
env_file:
- .env
- credentials.env
command:
--ipv6
--bind_ip_all

1
.ci/docker-compose-file/docker-compose.yaml
View File

@ -5,6 +5,7 @@ services:
container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
env_file:
- credentials.env
- conf.env
environment:
GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}

8
.ci/docker-compose-file/redis/sentinel-tcp/sentinel-base.conf
View File

@ -1,7 +1,7 @@
sentinel resolve-hostnames yes
bind :: 0.0.0.0
sentinel monitor mymaster redis-sentinel-master 6379 1
sentinel auth-pass mymaster public
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 20000
sentinel monitor mytcpmaster redis-sentinel-master 6379 1
sentinel auth-pass mytcpmaster public
sentinel down-after-milliseconds mytcpmaster 10000
sentinel failover-timeout mytcpmaster 20000

8
.ci/docker-compose-file/redis/sentinel-tls/sentinel-base.conf
View File

@ -8,7 +8,7 @@ tls-key-file /etc/certs/key.pem
tls-ca-cert-file /etc/certs/cacert.pem
tls-auth-clients no
sentinel monitor mymaster redis-sentinel-tls-master 6389 1
sentinel auth-pass mymaster public
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 20000
sentinel monitor mytlsmaster redis-sentinel-tls-master 6389 1
sentinel auth-pass mytlsmaster public
sentinel down-after-milliseconds mytlsmaster 10000
sentinel failover-timeout mytlsmaster 20000

4
.github/ISSUE_TEMPLATE/bug-report.yaml vendored
View File

@ -61,10 +61,6 @@ body:
# paste output here
$ uname -a
# paste output here
# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```
</details>

3
.github/pull_request_template.md vendored
View File

@ -1,9 +1,8 @@
Fixes <issue-or-jira-number>
<!-- Make sure to target release-52 branch if this PR is intended to fix the issues for the release candidate. -->
Release version: v/e5.?
## Summary
copilot:summary
## PR Checklist
Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked:

54
.github/workflows/build_packages.yaml vendored
View File

@ -65,58 +65,6 @@ on:
default: '5.2-3'
jobs:
windows:
runs-on: windows-2019
if: inputs.profile == 'emqx'
strategy:
fail-fast: false
matrix:
profile: # for now only CE for windows
- emqx
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.inputs.ref }}
fetch-depth: 0
- uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1.16.0
with:
otp-version: 25.3.2
- name: build
env:
PYTHON: python
DIAGNOSTIC: 1
run: |
# ensure crypto app (openssl)
erl -eval "erlang:display(crypto:info_lib())" -s init stop
make ${{ matrix.profile }}-tgz
- name: run emqx
timeout-minutes: 5
run: |
$ErrorActionPreference = "Stop"
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
Start-Sleep -s 10
$pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping
if ($pingOutput = 'pong') {
echo "EMQX started OK"
} else {
echo "Failed to ping EMQX $pingOutput"
Exit 1
}
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
echo "EMQX stopped"
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
echo "EMQX installed"
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
echo "EMQX uninstalled"
- uses: actions/upload-artifact@v3
if: success()
with:
name: ${{ matrix.profile }}
path: _packages/${{ matrix.profile }}/
retention-days: 7
mac:
strategy:
fail-fast: false
@ -126,9 +74,9 @@ jobs:
otp:
- ${{ inputs.otp_vsn }}
os:
- macos-11
- macos-12
- macos-12-arm64
- macos-13
runs-on: ${{ matrix.os }}
steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3

56
.github/workflows/build_packages_cron.yaml vendored
View File

@ -130,59 +130,3 @@ jobs:
with:
payload: |
{"text": "Scheduled build of ${{ matrix.profile }} package for ${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
windows:
if: github.repository_owner == 'emqx'
runs-on: windows-2019
strategy:
fail-fast: false
matrix:
profile:
- emqx
otp:
- 25.3.2
steps:
- uses: actions/checkout@v3
- uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1.16.0
with:
otp-version: ${{ matrix.otp }}
- name: build
env:
PYTHON: python
DIAGNOSTIC: 1
run: |
# ensure crypto app (openssl)
erl -eval "erlang:display(crypto:info_lib())" -s init stop
make ${{ matrix.profile }}-tgz
- name: run emqx
timeout-minutes: 5
run: |
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
Start-Sleep -s 10
$pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping
if ($pingOutput = 'pong') {
echo "EMQX started OK"
} else {
echo "Failed to ping EMQX $pingOutput"
Exit 1
}
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
echo "EMQX stopped"
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
echo "EMQX installed"
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
echo "EMQX uninstalled"
- uses: actions/upload-artifact@v3
with:
name: windows
path: _packages/${{ matrix.profile }}/*
retention-days: 7
- name: Send notification to Slack
uses: slackapi/slack-github-action@v1.23.0
if: failure()
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
with:
payload: |
{"text": "Scheduled build of ${{ matrix.profile }} package for Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}

4
.tool-versions
View File

@ -1,2 +1,2 @@
erlang 25.3.2-2
elixir 1.14.5-otp-25
erlang 26.1.2-1
elixir 1.15.7-otp-26

2
README-CN.md
View File

@ -77,7 +77,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl
优雅的跨平台 MQTT 5.0 客户端工具，提供了桌面端、命令行、Web 三种版本，帮助您更快的开发和调试 MQTT 服务和应用。
- [车联网平台搭建从入门到精通 ](https://www.emqx.com/zh/blog/category/internet-of-vehicles)
- [车联网平台搭建从入门到精通](https://www.emqx.com/zh/blog/category/internet-of-vehicles)
结合 EMQ 在车联网领域的实践经验，从协议选择等理论知识，到平台架构设计等实战操作，分享如何搭建一个可靠、高效、符合行业场景需求的车联网平台。

3
apps/emqx/include/emqx.hrl
View File

@ -39,9 +39,6 @@
%% System topic
-define(SYSTOP, <<"$SYS/">>).
%% Queue topic
-define(QUEUE, <<"$queue/">>).
%%--------------------------------------------------------------------
%% alarms
%%--------------------------------------------------------------------

20
apps/emqx/include/emqx_mqtt.hrl
View File

@ -55,6 +55,17 @@
%% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
-define(MAX_TOPIC_LEN, 65535).
%%--------------------------------------------------------------------
%% MQTT Share-Sub Internal
%%--------------------------------------------------------------------
-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}).
%% guards
-define(IS_TOPIC(T),
(is_binary(T) orelse is_record(T, share))
).
%%--------------------------------------------------------------------
%% MQTT QoS Levels
%%--------------------------------------------------------------------
@ -661,13 +672,10 @@ end).
-define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}).
-define(SHARE, "$share").
-define(QUEUE, "$queue").
-define(SHARE(Group, Topic), emqx_topic:join([<<?SHARE>>, Group, Topic])).
-define(IS_SHARE(Topic),
case Topic of
<<?SHARE, _/binary>> -> true;
_ -> false
end
).
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
-define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).

1
apps/emqx/include/emqx_trace.hrl
View File

@ -32,6 +32,5 @@
-define(SHARD, ?COMMON_SHARD).
-define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-endif.

1
apps/emqx/include/http_api.hrl
View File

@ -17,6 +17,7 @@
%% HTTP API Auth
-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
%% Bad Request
-define(BAD_REQUEST, 'BAD_REQUEST').

34
apps/emqx/include/logger.hrl
View File

@ -40,7 +40,9 @@
end
).
-define(AUDIT_HANDLER, emqx_audit).
-define(TRACE_FILTER, emqx_trace_filter).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)).
@ -61,25 +63,35 @@
)
end).
-define(AUDIT(_Level_, _From_, _Meta_), begin
case emqx_config:get([log, audit], #{enable => false}) of
#{enable := false} ->
-ifdef(EMQX_RELEASE_EDITION).
-if(?EMQX_RELEASE_EDITION == ee).
-define(AUDIT(_LevelFun_, _MetaFun_), begin
case logger_config:get(logger, ?AUDIT_HANDLER) of
{error, {not_found, _}} ->
ok;
#{enable := true, level := _AllowLevel_} ->
{ok, Handler = #{level := _AllowLevel_}} ->
_Level_ = _LevelFun_,
case logger:compare_levels(_AllowLevel_, _Level_) of
_R_ when _R_ == lt; _R_ == eq ->
emqx_trace:log(
_Level_,
[{emqx_audit, fun(L, _) -> L end, undefined, undefined}],
_Msg = undefined,
_Meta_#{from => _From_}
);
gt ->
emqx_audit:log(_Level_, _MetaFun_, Handler);
_ ->
ok
end
end
end).
-else.
%% Only for compile pass, ce edition will not call it
-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
-endif.
-else.
%% Only for compile pass, ce edition will not call it
-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
-endif.
%% print to 'user' group leader
-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).

324
apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl
View File

@ -9,14 +9,9 @@
-include_lib("stdlib/include/assert.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/src/emqx_persistent_session_ds.hrl").
-define(DEFAULT_KEYSPACE, default).
-define(DS_SHARD_ID, <<"local">>).
-define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}).
-import(emqx_common_test_helpers, [on_exit/1]).
%%------------------------------------------------------------------------------
@ -45,7 +40,7 @@ init_per_testcase(TestCase, Config) when
Cluster = cluster(#{n => 1}),
ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(NodeSpecs),
[
{cluster, Cluster},
{node_specs, NodeSpecs},
@ -53,12 +48,36 @@ init_per_testcase(TestCase, Config) when
{nodes, Nodes}
| Config
];
init_per_testcase(t_session_gc = TestCase, Config) ->
Opts = #{
n => 3,
roles => [core, core, replicant],
extra_emqx_conf =>
"\n session_persistence {"
"\n last_alive_update_interval = 500ms "
"\n session_gc_interval = 2s "
"\n session_gc_batch_size = 1 "
"\n }"
},
Cluster = cluster(Opts),
ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
[
{cluster, Cluster},
{node_specs, NodeSpecs},
{cluster_opts, ClusterOpts},
{nodes, Nodes},
{gc_interval, timer:seconds(2)}
| Config
];
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(TestCase, Config) when
TestCase =:= t_session_subscription_idempotency;
TestCase =:= t_session_unsubscription_idempotency
TestCase =:= t_session_unsubscription_idempotency;
TestCase =:= t_session_gc
->
Nodes = ?config(nodes, Config),
emqx_common_test_helpers:call_janitor(60_000),
@ -72,32 +91,38 @@ end_per_testcase(_TestCase, _Config) ->
%% Helper fns
%%------------------------------------------------------------------------------
cluster(#{n := N}) ->
Spec = #{role => core, apps => app_specs()},
cluster(#{n := N} = Opts) ->
MkRole = fun(M) ->
case maps:get(roles, Opts, undefined) of
undefined ->
core;
Roles ->
lists:nth(M, Roles)
end
end,
MkSpec = fun(M) -> #{role => MkRole(M), apps => app_specs(Opts)} end,
lists:map(
fun(M) ->
Name = list_to_atom("ds_SUITE" ++ integer_to_list(M)),
{Name, Spec}
{Name, MkSpec(M)}
end,
lists:seq(1, N)
).
app_specs() ->
app_specs(_Opts = #{}).
app_specs(Opts) ->
ExtraEMQXConf = maps:get(extra_emqx_conf, Opts, ""),
[
emqx_durable_storage,
{emqx, "persistent_session_store = {ds = true}"}
{emqx, "session_persistence = {enable = true}" ++ ExtraEMQXConf}
].
get_mqtt_port(Node, Type) ->
{_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]),
Port.
get_all_iterator_ids(Node) ->
Fn = fun(K, _V, Acc) -> [K | Acc] end,
erpc:call(Node, fun() ->
emqx_ds_storage_layer:foldl_iterator_prefix(?DS_SHARD, <<>>, Fn, [])
end).
wait_nodeup(Node) ->
?retry(
_Sleep0 = 500,
@ -127,33 +152,37 @@ start_client(Opts0 = #{}) ->
restart_node(Node, NodeSpec) ->
?tp(will_restart_node, #{}),
?tp(notice, "restarting node", #{node => Node}),
true = monitor_node(Node, true),
ok = erpc:call(Node, init, restart, []),
receive
{nodedown, Node} ->
ok
after 10_000 ->
ct:fail("node ~p didn't stop", [Node])
end,
?tp(notice, "waiting for nodeup", #{node => Node}),
emqx_cth_cluster:restart(Node, NodeSpec),
wait_nodeup(Node),
wait_gen_rpc_down(NodeSpec),
?tp(notice, "restarting apps", #{node => Node}),
Apps = maps:get(apps, NodeSpec),
ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]),
_ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]),
%% have to re-inject this so that we may stop the node succesfully at the
%% end....
ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec),
ok = snabbkaffe:forward_trace(Node),
?tp(notice, "node restarted", #{node => Node}),
?tp(restarted_node, #{}),
ok.
is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}}) ->
EI > 0.
list_all_sessions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []).
list_all_subscriptions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []).
list_all_pubranges(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []).
prop_only_cores_run_gc(CoreNodes) ->
{"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}.
prop_only_cores_run_gc(Trace, CoreNodes) ->
GCNodes = lists:usort([
N
|| #{
?snk_kind := K,
?snk_meta := #{node := N}
} <- Trace,
lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]),
N =/= node()
]),
?assertEqual(lists:usort(CoreNodes), GCNodes).
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
@ -233,10 +262,10 @@ t_session_subscription_idempotency(Config) ->
end,
fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]),
SubTopicFilterWords = emqx_topic:words(SubTopicFilter),
ConnInfo = #{},
?assertMatch(
{ok, #{}, #{SubTopicFilterWords := #{}}},
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
#{subscriptions := #{SubTopicFilter := #{}}},
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
)
end
),
@ -307,9 +336,10 @@ t_session_unsubscription_idempotency(Config) ->
end,
fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]),
ConnInfo = #{},
?assertMatch(
{ok, #{}, Subs = #{}} when map_size(Subs) =:= 0,
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
#{subscriptions := Subs = #{}} when map_size(Subs) =:= 0,
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
),
ok
end
@ -370,18 +400,12 @@ do_t_session_discard(Params) ->
_Attempts0 = 50,
true = map_size(emqx_persistent_session_ds:list_all_streams()) > 0
),
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
true = map_size(emqx_persistent_session_ds:list_all_iterators()) > 0
),
ok = emqtt:stop(Client0),
?tp(notice, "disconnected", #{}),
?tp(notice, "reconnecting", #{}),
%% we still have iterators and streams
%% we still have streams
?assert(map_size(emqx_persistent_session_ds:list_all_streams()) > 0),
?assert(map_size(emqx_persistent_session_ds:list_all_iterators()) > 0),
Client1 = start_client(ReconnectOpts),
{ok, _} = emqtt:connect(Client1),
?assertEqual([], emqtt:subscriptions(Client1)),
@ -394,7 +418,7 @@ do_t_session_discard(Params) ->
?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
?assertEqual([], emqx_persistent_session_ds_router:topics()),
?assertEqual(#{}, emqx_persistent_session_ds:list_all_streams()),
?assertEqual(#{}, emqx_persistent_session_ds:list_all_iterators()),
?assertEqual(#{}, emqx_persistent_session_ds:list_all_pubranges()),
ok = emqtt:stop(Client1),
?tp(notice, "disconnected", #{}),
@ -406,3 +430,201 @@ do_t_session_discard(Params) ->
end
),
ok.
t_session_expiration1(Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
sequence => [
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 1}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
]
},
do_t_session_expiration(Config, Opts).
t_session_expiration2(Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
sequence => [
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{
'Session-Expiry-Interval' => 1
}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
]
},
do_t_session_expiration(Config, Opts).
do_t_session_expiration(_Config, Opts) ->
#{
clientid := ClientId,
sequence := [
{FirstConn, FirstDisconn},
{SecondConn, SecondDisconn},
{ThirdConn, ThirdDisconn}
]
} = Opts,
CommonParams = #{proto_ver => v5, clientid => ClientId},
?check_trace(
begin
Topic = <<"some/topic">>,
Params0 = maps:merge(CommonParams, FirstConn),
Client0 = start_client(Params0),
{ok, _} = emqtt:connect(Client0),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2),
Subs0 = emqx_persistent_session_ds:list_all_subscriptions(),
?assertEqual(1, map_size(Subs0), #{subs => Subs0}),
Info0 = maps:from_list(emqtt:info(Client0)),
?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}),
emqtt:disconnect(Client0, ?RC_NORMAL_DISCONNECTION, FirstDisconn),
Params1 = maps:merge(CommonParams, SecondConn),
Client1 = start_client(Params1),
{ok, _} = emqtt:connect(Client1),
Info1 = maps:from_list(emqtt:info(Client1)),
?assertEqual(1, maps:get(session_present, Info1), #{info => Info1}),
Subs1 = emqtt:subscriptions(Client1),
?assertEqual([], Subs1),
emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn),
ct:sleep(1_500),
Params2 = maps:merge(CommonParams, ThirdConn),
Client2 = start_client(Params2),
{ok, _} = emqtt:connect(Client2),
Info2 = maps:from_list(emqtt:info(Client2)),
?assertEqual(0, maps:get(session_present, Info2), #{info => Info2}),
Subs2 = emqtt:subscriptions(Client2),
?assertEqual([], Subs2),
emqtt:publish(Client2, Topic, <<"payload">>),
?assertNotReceive({publish, #{topic := Topic}}),
%% ensure subscriptions are absent from table.
?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn),
ok
end,
[]
),
ok.
t_session_gc(Config) ->
GCInterval = ?config(gc_interval, Config),
[Node1, Node2, Node3] = Nodes = ?config(nodes, Config),
CoreNodes = [Node1, Node2],
[
Port1,
Port2,
Port3
] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes),
CommonParams = #{
clean_start => false,
proto_ver => v5
},
StartClient = fun(ClientId, Port, ExpiryInterval) ->
Params = maps:merge(CommonParams, #{
clientid => ClientId,
port => Port,
properties => #{'Session-Expiry-Interval' => ExpiryInterval}
}),
Client = start_client(Params),
{ok, _} = emqtt:connect(Client),
Client
end,
?check_trace(
begin
ClientId0 = <<"session_gc0">>,
Client0 = StartClient(ClientId0, Port1, 30),
ClientId1 = <<"session_gc1">>,
Client1 = StartClient(ClientId1, Port2, 1),
ClientId2 = <<"session_gc2">>,
Client2 = StartClient(ClientId2, Port3, 1),
lists:foreach(
fun(Client) ->
Topic = <<"some/topic">>,
Payload = <<"hi">>,
{ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client, Topic, ?QOS_1),
{ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1),
ok
end,
[Client0, Client1, Client2]
),
%% Clients are still alive; no session is garbage collected.
Res0 = ?block_until(
#{
?snk_kind := ds_session_gc,
?snk_span := {complete, _},
?snk_meta := #{node := N}
} when
N =/= node(),
3 * GCInterval + 1_000
),
?assertMatch({ok, _}, Res0),
{ok, #{?snk_meta := #{time := T0}}} = Res0,
Sessions0 = list_all_sessions(Node1),
Subs0 = list_all_subscriptions(Node1),
?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}),
?assertEqual(3, map_size(Subs0), #{subs => Subs0}),
%% Now we disconnect 2 of them; only those should be GC'ed.
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client1),
#{?snk_kind := terminate},
1_000
)
),
ct:pal("disconnected client1"),
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client2),
#{?snk_kind := terminate},
1_000
)
),
ct:pal("disconnected client2"),
?assertMatch(
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId1]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
)
),
?assertMatch(
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId2]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
)
),
Sessions1 = list_all_sessions(Node1),
Subs1 = list_all_subscriptions(Node1),
?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}),
?assertEqual(1, map_size(Subs1), #{subs => Subs1}),
ok
end,
[
prop_only_cores_run_gc(CoreNodes)
]
),
ok.

2
apps/emqx/priv/bpapi.versions
View File

@ -39,10 +39,12 @@
{emqx_mgmt_api_plugins,2}.
{emqx_mgmt_cluster,1}.
{emqx_mgmt_cluster,2}.
{emqx_mgmt_data_backup,1}.
{emqx_mgmt_trace,1}.
{emqx_mgmt_trace,2}.
{emqx_node_rebalance,1}.
{emqx_node_rebalance,2}.
{emqx_node_rebalance,3}.
{emqx_node_rebalance_api,1}.
{emqx_node_rebalance_api,2}.
{emqx_node_rebalance_evacuation,1}.

8
apps/emqx/rebar.config
View File

@ -27,9 +27,9 @@
{lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}},
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
@ -45,7 +45,7 @@
{meck, "0.9.2"},
{proper, "1.4.0"},
{bbmustache, "1.10.0"},
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.1"}}}
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}}
]},
{extra_src_dirs, [{"test", [recursive]},
{"integration_test", [recursive]}]}
@ -55,7 +55,7 @@
{meck, "0.9.2"},
{proper, "1.4.0"},
{bbmustache, "1.10.0"},
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.1"}}}
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}}
]},
{extra_src_dirs, [{"test", [recursive]}]}
]}

2
apps/emqx/rebar.config.script
View File

@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
end,
Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.202"}}}.
Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.303"}}}.
Dialyzer = fun(Config) ->
{dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),

17
apps/emqx/src/bhvrs/emqx_db_backup.erl
View File

@ -16,4 +16,21 @@
-module(emqx_db_backup).
-type traverse_break_reason() :: over | migrate.
-callback backup_tables() -> [mria:table()].
%% validate the backup
%% return `ok` to traverse the next item
%% return `{ok, over}` to finish the traverse
%% return `{ok, migrate}` to call the migration callback
-callback validate_mnesia_backup(tuple()) ->
ok
| {ok, traverse_break_reason()}
| {error, term()}.
-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
-export_type([traverse_break_reason/0]).

7
apps/emqx/src/config/emqx_config_logger.erl
View File

@ -23,8 +23,9 @@
-export([post_config_update/5]).
-export([filter_audit/2]).
-include("logger.hrl").
-define(LOG, [log]).
-define(AUDIT_HANDLER, emqx_audit).
add_handler() ->
ok = emqx_config_handler:add_handler(?LOG, ?MODULE),
@ -95,6 +96,10 @@ update_log_handlers(NewHandlers) ->
ok = application:set_env(kernel, logger, NewHandlers),
ok.
%% Don't remove audit log handler here, we need record this removed action into audit log file.
%% we will remove audit log handler after audit log is record in emqx_audit:log/3.
update_log_handler({removed, ?AUDIT_HANDLER}) ->
ok;
update_log_handler({removed, Id}) ->
log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]),
logger:remove_handler(Id);

2
apps/emqx/src/emqx.app.src
View File

@ -2,7 +2,7 @@
{application, emqx, [
{id, "emqx"},
{description, "EMQX Core"},
{vsn, "5.1.14"},
{vsn, "5.1.15"},
{modules, []},
{registered, []},
{applications, [

88
apps/emqx/src/emqx_broker.erl
View File

@ -118,18 +118,20 @@ create_tabs() ->
%% Subscribe API
%%------------------------------------------------------------------------------
-spec subscribe(emqx_types:topic()) -> ok.
subscribe(Topic) when is_binary(Topic) ->
-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok.
subscribe(Topic) when ?IS_TOPIC(Topic) ->
subscribe(Topic, undefined).
-spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok.
subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) ->
-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) ->
ok.
subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) ->
subscribe(Topic, SubId, ?DEFAULT_SUBOPTS);
subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) ->
subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) ->
subscribe(Topic, undefined, SubOpts).
-spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok.
subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) ->
ok.
subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0),
_ = emqx_trace:subscribe(Topic, SubId, SubOpts),
SubPid = self(),
@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) ->
with_subid(SubId, SubOpts) ->
maps:put(subid, SubId, SubOpts).
%% @private
do_subscribe(Topic, SubPid, SubOpts) ->
true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}),
Group = maps:get(share, SubOpts, undefined),
do_subscribe(Group, Topic, SubPid, SubOpts).
do_subscribe2(Topic, SubPid, SubOpts).
do_subscribe(undefined, Topic, SubPid, SubOpts) ->
do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) ->
%% FIXME: subscribe shard bug
%% https://emqx.atlassian.net/browse/EMQX-10214
case emqx_broker_helper:get_sub_shard(SubPid, Topic) of
0 ->
true = ets:insert(?SUBSCRIBER, {Topic, SubPid}),
@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) ->
true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}),
call(pick({Topic, I}), {subscribe, Topic, I})
end;
%% Shared subscription
do_subscribe(Group, Topic, SubPid, SubOpts) ->
do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when
is_binary(RealTopic)
->
true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}),
emqx_shared_sub:subscribe(Group, Topic, SubPid).
emqx_shared_sub:subscribe(Group, RealTopic, SubPid).
%%--------------------------------------------------------------------
%% Unsubscribe API
%%--------------------------------------------------------------------
-spec unsubscribe(emqx_types:topic()) -> ok.
unsubscribe(Topic) when is_binary(Topic) ->
-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok.
unsubscribe(Topic) when ?IS_TOPIC(Topic) ->
SubPid = self(),
case ets:lookup(?SUBOPTION, {Topic, SubPid}) of
[{_, SubOpts}] ->
_ = emqx_broker_helper:reclaim_seq(Topic),
_ = emqx_trace:unsubscribe(Topic, SubOpts),
do_unsubscribe(Topic, SubPid, SubOpts);
[] ->
ok
end.
-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
ok.
do_unsubscribe(Topic, SubPid, SubOpts) ->
true = ets:delete(?SUBOPTION, {Topic, SubPid}),
true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}),
Group = maps:get(share, SubOpts, undefined),
do_unsubscribe(Group, Topic, SubPid, SubOpts).
do_unsubscribe2(Topic, SubPid, SubOpts).
do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
ok.
do_unsubscribe2(Topic, SubPid, SubOpts) when
is_binary(Topic), is_pid(SubPid), is_map(SubOpts)
->
_ = emqx_broker_helper:reclaim_seq(Topic),
case maps:get(shard, SubOpts, 0) of
0 ->
true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
cast(pick({Topic, I}), {unsubscribed, Topic, I})
end;
do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when
is_binary(Group), is_binary(Topic), is_pid(SubPid)
->
emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
%%--------------------------------------------------------------------
@ -306,7 +316,9 @@ aggre([], true, Acc) ->
lists:usort(Acc).
%% @doc Forward message to another node.
-spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) ->
-spec forward(
node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async
) ->
emqx_types:deliver_result().
forward(Node, To, Delivery, async) ->
true = emqx_broker_proto_v1:forward_async(Node, To, Delivery),
@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) ->
Result
end.
-spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
emqx_types:deliver_result().
dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) ->
case emqx:is_running() of
true ->
@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) ->
end.
-compile({inline, [subscribers/1]}).
-spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) ->
-spec subscribers(
emqx_types:topic()
| emqx_types:share()
| {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()}
) ->
[pid()].
subscribers(Topic) when is_binary(Topic) ->
lookup_value(?SUBSCRIBER, Topic, []);
@ -372,7 +389,7 @@ subscriber_down(SubPid) ->
SubOpts when is_map(SubOpts) ->
_ = emqx_broker_helper:reclaim_seq(Topic),
true = ets:delete(?SUBOPTION, {Topic, SubPid}),
do_unsubscribe(undefined, Topic, SubPid, SubOpts);
do_unsubscribe2(Topic, SubPid, SubOpts);
undefined ->
ok
end
@ -386,7 +403,7 @@ subscriber_down(SubPid) ->
%%--------------------------------------------------------------------
-spec subscriptions(pid() | emqx_types:subid()) ->
[{emqx_types:topic(), emqx_types:subopts()}].
[{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}].
subscriptions(SubPid) when is_pid(SubPid) ->
[
{Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})}
@ -400,20 +417,22 @@ subscriptions(SubId) ->
[]
end.
-spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()].
-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()].
subscriptions_via_topic(Topic) ->
MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}],
ets:select(?SUBOPTION, MatchSpec).
-spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean().
-spec subscribed(
pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share()
) -> boolean().
subscribed(SubPid, Topic) when is_pid(SubPid) ->
ets:member(?SUBOPTION, {Topic, SubPid});
subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
SubPid = emqx_broker_helper:lookup_subpid(SubId),
ets:member(?SUBOPTION, {Topic, SubPid}).
-spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()).
get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) ->
-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
lookup_value(?SUBOPTION, {Topic, SubPid});
get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
case emqx_broker_helper:lookup_subpid(SubId) of
@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
undefined
end.
-spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean().
-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean().
set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) ->
set_subopts(self(), Topic, NewOpts).
@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) ->
false
end.
-spec topics() -> [emqx_types:topic()].
-spec topics() -> [emqx_types:topic() | emqx_types:share()].
topics() ->
emqx_router:topics().
@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%--------------------------------------------------------------------
-spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
emqx_types:deliver_result().
do_dispatch(Topic, #delivery{message = Msg}) ->
DispN = lists:foldl(
fun(Sub, N) ->
@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) ->
{ok, DispN}
end.
%% Donot dispatch to share subscriber here.
%% we do it in `emqx_shared_sub.erl` with configured strategy
do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
case erlang:is_process_alive(SubPid) of
true ->

308
apps/emqx/src/emqx_channel.erl
View File

@ -423,6 +423,7 @@ handle_in(
{ok, Channel}
end;
handle_in(
%% TODO: Why discard the Reason Code?
?PUBREC_PACKET(PacketId, _ReasonCode, Properties),
Channel =
#channel{clientinfo = ClientInfo, session = Session}
@ -476,60 +477,27 @@ handle_in(
ok = emqx_metrics:inc('packets.pubcomp.missed'),
{ok, Channel}
end;
handle_in(
SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
Channel = #channel{clientinfo = ClientInfo}
) ->
case emqx_packet:check(SubPkt) of
ok ->
TopicFilters0 = parse_topic_filters(TopicFilters),
TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
HasAuthzDeny = lists:any(
fun({_TopicFilter, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end,
TupleTopicFilters0
),
DenyAction = emqx:get_config([authorization, deny_action], ignore),
case DenyAction =:= disconnect andalso HasAuthzDeny of
true ->
handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
false ->
TopicFilters2 = [
TopicFilter
|| {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) ->
Pipe = pipeline(
[
fun check_subscribe/2,
fun enrich_subscribe/2,
%% TODO && FIXME (EMQX-10786): mount topic before authz check.
fun check_sub_authzs/2,
fun check_sub_caps/2
],
TopicFilters3 = run_hooks(
'client.subscribe',
[ClientInfo, Properties],
TopicFilters2
SubPkt,
Channel0
),
{TupleTopicFilters1, NChannel} = process_subscribe(
TopicFilters3,
Properties,
Channel
),
TupleTopicFilters2 =
lists:foldl(
fun
({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) ->
Key = {Topic, maps:without([deny_subscription], Opts)},
lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR});
(Tuple = {Key, _Value}, Acc) ->
lists:keyreplace(Key, 1, Acc, Tuple)
end,
TupleTopicFilters0,
TupleTopicFilters1
),
ReasonCodes2 = [
ReasonCode
|| {_TopicFilter, ReasonCode} <- TupleTopicFilters2
],
handle_out(suback, {PacketId, ReasonCodes2}, NChannel)
end;
{error, ReasonCode} ->
handle_out(disconnect, ReasonCode, Channel)
case Pipe of
{ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} ->
{TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel),
ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC),
handle_out(suback, {PacketId, ReasonCodes}, NChannel);
{error, {disconnect, RC}, Channel} ->
%% funcs in pipeline always cause action: `disconnect`
%% And Only one ReasonCode in DISCONNECT packet
handle_out(disconnect, RC, Channel)
end;
handle_in(
Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
@ -540,7 +508,7 @@ handle_in(
TopicFilters1 = run_hooks(
'client.unsubscribe',
[ClientInfo, Properties],
parse_topic_filters(TopicFilters)
parse_raw_topic_filters(TopicFilters)
),
{ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel),
handle_out(unsuback, {PacketId, ReasonCodes}, NChannel);
@ -782,32 +750,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) ->
%% Process Subscribe
%%--------------------------------------------------------------------
-compile({inline, [process_subscribe/3]}).
process_subscribe(TopicFilters, SubProps, Channel) ->
process_subscribe(TopicFilters, SubProps, Channel, []).
process_subscribe(TopicFilters, Channel) ->
process_subscribe(TopicFilters, Channel, []).
process_subscribe([], _SubProps, Channel, Acc) ->
process_subscribe([], Channel, Acc) ->
{lists:reverse(Acc), Channel};
process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) ->
case check_sub_caps(TopicFilter, SubOpts, Channel) of
ok ->
{ReasonCode, NChannel} = do_subscribe(
TopicFilter,
SubOpts#{sub_props => SubProps},
Channel
),
process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]);
{error, ReasonCode} ->
?SLOG(
warning,
#{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(ReasonCode)
},
#{topic => TopicFilter}
),
process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
end.
process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) ->
{NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel),
process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]).
do_subscribe(
TopicFilter,
@ -818,11 +768,13 @@ do_subscribe(
session = Session
}
) ->
%% TODO && FIXME (EMQX-10786): mount topic before authz check.
NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter),
NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel),
case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of
case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of
{ok, NSession} ->
{QoS, Channel#channel{session = NSession}};
%% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now
RC = QoS,
{RC, Channel#channel{session = NSession}};
{error, RC} ->
?SLOG(
warning,
@ -835,6 +787,30 @@ do_subscribe(
{RC, Channel}
end.
gen_reason_codes(TFChecked, TFSubedWitNhRC) ->
do_gen_reason_codes([], TFChecked, TFSubedWitNhRC).
%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2
%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and
%% NRC should override the initial RC.
do_gen_reason_codes(Acc, [], []) ->
lists:reverse(Acc);
do_gen_reason_codes(
Acc,
[{_, ?RC_SUCCESS} | RestTF],
[{_, NRC} | RestWithNRC]
) ->
%% will passing through `process_subscribe/2`
%% use NRC to override IintialRC
do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC);
do_gen_reason_codes(
Acc,
[{_, InitialRC} | Rest],
RestWithNRC
) ->
%% InitialRC is not `RC_SUCCESS`, use it.
do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC).
%%--------------------------------------------------------------------
%% Process Unsubscribe
%%--------------------------------------------------------------------
@ -1213,13 +1189,8 @@ handle_call(Req, Channel) ->
ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}.
handle_info({subscribe, TopicFilters}, Channel) ->
{_, NChannel} = lists:foldl(
fun({TopicFilter, SubOpts}, {_, ChannelAcc}) ->
do_subscribe(TopicFilter, SubOpts, ChannelAcc)
end,
{[], Channel},
parse_topic_filters(TopicFilters)
),
NTopicFilters = enrich_subscribe(TopicFilters, Channel),
{_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel),
{ok, NChannel};
handle_info({unsubscribe, TopicFilters}, Channel) ->
{_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel),
@ -1234,12 +1205,13 @@ handle_info(
#channel{
conn_state = ConnState,
clientinfo = ClientInfo,
conninfo = ConnInfo,
session = Session
}
) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
{Intent, Session1} = emqx_session:disconnect(ClientInfo, Session),
{Intent, Session1} = emqx_session:disconnect(ClientInfo, ConnInfo, Session),
Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
Channel2 = Channel1#channel{session = Session1},
case maybe_shutdown(Reason, Intent, Channel2) of
@ -1351,7 +1323,8 @@ handle_timeout(
{ok, Replies, NSession} ->
handle_out(publish, Replies, Channel#channel{session = NSession})
end;
handle_timeout(_TRef, expire_session, Channel) ->
handle_timeout(_TRef, expire_session, Channel = #channel{session = Session}) ->
ok = emqx_session:destroy(Session),
shutdown(expired, Channel);
handle_timeout(
_TRef,
@ -1859,49 +1832,156 @@ check_pub_caps(
) ->
emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}).
%%--------------------------------------------------------------------
%% Check Subscribe Packet
check_subscribe(SubPkt, _Channel) ->
case emqx_packet:check(SubPkt) of
ok -> ok;
{error, RC} -> {error, {disconnect, RC}}
end.
%%--------------------------------------------------------------------
%% Check Sub Authorization
check_sub_authzs(TopicFilters, Channel) ->
check_sub_authzs(TopicFilters, Channel, []).
check_sub_authzs(
[TopicFilter = {Topic, _} | More],
Channel = #channel{clientinfo = ClientInfo},
Acc
?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0),
Channel = #channel{clientinfo = ClientInfo}
) ->
CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo),
HasAuthzDeny = lists:any(
fun({{_TopicFilter, _SubOpts}, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end,
CheckResult
),
DenyAction = emqx:get_config([authorization, deny_action], ignore),
case DenyAction =:= disconnect andalso HasAuthzDeny of
true ->
{error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel};
false ->
{ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}
end.
do_check_sub_authzs(TopicFilters, ClientInfo) ->
do_check_sub_authzs(ClientInfo, TopicFilters, []).
do_check_sub_authzs(_ClientInfo, [], Acc) ->
lists:reverse(Acc);
do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) ->
%% subsclibe authz check only cares the real topic filter when shared-sub
%% e.g. only check <<"t/#">> for <<"$share/g/t/#">>
Action = authz_action(TopicFilter),
case emqx_access_control:authorize(ClientInfo, Action, Topic) of
case
emqx_access_control:authorize(
ClientInfo,
Action,
emqx_topic:get_shared_real_topic(Topic)
)
of
%% TODO: support maximum QoS granted
%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
%% Not implemented yet:
%% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]);
allow ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]);
deny ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end;
check_sub_authzs([], _Channel, Acc) ->
lists:reverse(Acc).
do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end.
%%--------------------------------------------------------------------
%% Check Sub Caps
check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts).
check_sub_caps(
?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters),
Channel = #channel{clientinfo = ClientInfo}
) ->
CheckResult = do_check_sub_caps(ClientInfo, TopicFilters),
{ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}.
do_check_sub_caps(ClientInfo, TopicFilters) ->
do_check_sub_caps(ClientInfo, TopicFilters, []).
do_check_sub_caps(_ClientInfo, [], Acc) ->
lists:reverse(Acc);
do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) ->
case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of
ok ->
do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]);
{error, NRC} ->
?SLOG(
warning,
#{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(NRC)
},
#{topic => Topic}
),
do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc])
end;
do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) ->
do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]).
%%--------------------------------------------------------------------
%% Enrich SubId
%% Run Subscribe Hooks
enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
[{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
enrich_subopts_subid(_Properties, TopicFilters) ->
TopicFilters.
run_sub_hooks(
?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0),
_Channel = #channel{clientinfo = ClientInfo}
) ->
TopicFilters = [
TopicFilter
|| {TopicFilter, ?RC_SUCCESS} <- TopicFilters0
],
_NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters).
%%--------------------------------------------------------------------
%% Enrich SubOpts
enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) ->
SubOpts;
enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
%% for api subscribe without sub-authz check and sub-caps check.
enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) ->
do_enrich_subscribe(#{}, TopicFilters, Channel);
%% for mqtt clients sent subscribe packet.
enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) ->
NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel),
{ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}.
do_enrich_subscribe(Properties, TopicFilters, Channel) ->
_NTopicFilters = run_fold(
[
%% TODO: do try catch with reason code here
fun(TFs, _) -> parse_raw_topic_filters(TFs) end,
fun enrich_subopts_subid/2,
fun enrich_subopts_porps/2,
fun enrich_subopts_flags/2
],
TopicFilters,
#{sub_props => Properties, channel => Channel}
).
enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) ->
[{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
enrich_subopts_subid(TopicFilters, _State) ->
TopicFilters.
enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) ->
[{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters].
enrich_subopts_flags(TopicFilters, #{channel := Channel}) ->
do_enrich_subopts_flags(TopicFilters, Channel).
do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) ->
[{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters];
do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
Rap = flag(IsBridge),
NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)),
SubOpts#{rap => flag(IsBridge), nl => NL}.
[
{Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}}
|| {Topic, SubOpts} <- TopicFilters
].
merge_default_subopts(SubOpts) ->
maps:merge(?DEFAULT_SUBOPTS, SubOpts).
%%--------------------------------------------------------------------
%% Enrich ConnAck Caps
@ -2091,8 +2171,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) ->
%%--------------------------------------------------------------------
%% Parse Topic Filters
-compile({inline, [parse_topic_filters/1]}).
parse_topic_filters(TopicFilters) ->
%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _]
parse_raw_topic_filters(TopicFilters) ->
lists:map(fun emqx_topic:parse/1, TopicFilters).
%%--------------------------------------------------------------------

1
apps/emqx/src/emqx_cm.erl
View File

@ -91,6 +91,7 @@
clean_down/1,
mark_channel_connected/1,
mark_channel_disconnected/1,
is_channel_connected/1,
get_connected_client_count/0
]).

12
apps/emqx/src/emqx_cm_sup.erl
View File

@ -47,7 +47,17 @@ init([]) ->
Locker = child_spec(emqx_cm_locker, 5000, worker),
Registry = child_spec(emqx_cm_registry, 5000, worker),
Manager = child_spec(emqx_cm, 5000, worker),
{ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}.
DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
Children =
[
Banned,
Flapping,
Locker,
Registry,
Manager,
DSSessionGCSup
],
{ok, {SupFlags, Children}}.
%%--------------------------------------------------------------------
%% Internal functions

4
apps/emqx/src/emqx_message.erl
View File

@ -301,7 +301,9 @@ update_expiry(Msg) ->
Msg.
%% @doc Message to PUBLISH Packet.
-spec to_packet(emqx_types:packet_id(), emqx_types:message()) ->
%%
%% When QoS=0 then packet id must be `undefined'
-spec to_packet(emqx_types:packet_id() | undefined, emqx_types:message()) ->
emqx_types:packet().
to_packet(
PacketId,

47
apps/emqx/src/emqx_mountpoint.erl
View File

@ -17,6 +17,7 @@
-module(emqx_mountpoint).
-include("emqx.hrl").
-include("emqx_mqtt.hrl").
-include("emqx_placeholder.hrl").
-include("types.hrl").
@ -34,38 +35,54 @@
-spec mount(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
| emqx_types:message()
| emqx_types:topic_filters().
mount(undefined, Any) ->
Any;
mount(MountPoint, Topic) when is_binary(Topic) ->
prefix(MountPoint, Topic);
mount(MountPoint, Msg = #message{topic = Topic}) ->
Msg#message{topic = prefix(MountPoint, Topic)};
mount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
prefix_maybe_share(MountPoint, Topic);
mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
Msg#message{topic = prefix_maybe_share(MountPoint, Topic)};
mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
[{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
[{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
%% @private
-compile({inline, [prefix/2]}).
prefix(MountPoint, Topic) ->
<<MountPoint/binary, Topic/binary>>.
-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share().
prefix_maybe_share(MountPoint, Topic) when
is_binary(MountPoint) andalso is_binary(Topic)
->
<<MountPoint/binary, Topic/binary>>;
prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
is_binary(MountPoint) andalso is_binary(Topic)
->
#share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
-spec unmount(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
| emqx_types:message().
unmount(undefined, Any) ->
Any;
unmount(MountPoint, Topic) when is_binary(Topic) ->
unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
unmount_maybe_share(MountPoint, Topic);
unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}.
unmount_maybe_share(MountPoint, Topic) when
is_binary(MountPoint) andalso is_binary(Topic)
->
case string:prefix(Topic, MountPoint) of
nomatch -> Topic;
Topic1 -> Topic1
end;
unmount(MountPoint, Msg = #message{topic = Topic}) ->
case string:prefix(Topic, MountPoint) of
nomatch -> Msg;
Topic1 -> Msg#message{topic = Topic1}
end.
unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
is_binary(MountPoint) andalso is_binary(Topic)
->
TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
-spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
replvar(undefined, _Vars) ->

15
apps/emqx/src/emqx_mqtt_caps.erl
View File

@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) ->
-spec check_sub(
emqx_types:clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts()
) ->
ok_or_error(emqx_types:reason_code()).
check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) ->
Caps = emqx_config:get_zone_conf(Zone, [mqtt]),
Flags = #{
%% TODO: qos check
%% (max_qos_allowed, Map) ->
%% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2),
topic_levels => emqx_topic:levels(Topic),
is_wildcard => emqx_topic:wildcard(Topic),
is_shared => maps:is_key(share, SubOpts),
is_shared => erlang:is_record(Topic, share),
is_exclusive => maps:get(is_exclusive, SubOpts, false)
},
do_check_sub(Flags, Caps, ClientInfo, Topic).
@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) ->
{error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) ->
{error, ?RC_TOPIC_FILTER_INVALID};
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) ->
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when
is_binary(Topic)
->
case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of
deny ->
{error, ?RC_QUOTA_EXCEEDED};
_ ->
ok
end;
%% for max_qos_allowed
%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2
%% do_check_sub(_, _) ->
%% {ok, RC};
do_check_sub(_Flags, _Caps, _, _) ->
ok.

41
apps/emqx/src/emqx_persistent_message.erl
View File

@ -19,7 +19,7 @@
-include("emqx.hrl").
-export([init/0]).
-export([is_store_enabled/0]).
-export([is_persistence_enabled/0, force_ds/0]).
%% Message persistence
-export([
@ -28,9 +28,8 @@
-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message).
%% FIXME
-define(WHEN_ENABLED(DO),
case is_store_enabled() of
case is_persistence_enabled() of
true -> DO;
false -> {skipped, disabled}
end
@ -40,18 +39,40 @@
init() ->
?WHEN_ENABLED(begin
ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{
backend => builtin,
storage => {emqx_ds_storage_bitfield_lts, #{}}
}),
Backend = storage_backend(),
ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, Backend),
ok = emqx_persistent_session_ds_router:init_tables(),
ok = emqx_persistent_session_ds:create_tables(),
ok
end).
-spec is_store_enabled() -> boolean().
is_store_enabled() ->
emqx_config:get([persistent_session_store, ds]).
-spec is_persistence_enabled() -> boolean().
is_persistence_enabled() ->
emqx_config:get([session_persistence, enable]).
-spec storage_backend() -> emqx_ds:create_db_opts().
storage_backend() ->
storage_backend(emqx_config:get([session_persistence, storage])).
%% Dev-only option: force all messages to go through
%% `emqx_persistent_session_ds':
-spec force_ds() -> boolean().
force_ds() ->
emqx_config:get([session_persistence, force_persistence]).
storage_backend(#{
builtin := #{enable := true, n_shards := NShards, replication_factor := ReplicationFactor}
}) ->
#{
backend => builtin,
storage => {emqx_ds_storage_bitfield_lts, #{}},
n_shards => NShards,
replication_factor => ReplicationFactor
};
storage_backend(#{
fdb := #{enable := true} = FDBConfig
}) ->
FDBConfig#{backend => fdb}.
%%--------------------------------------------------------------------

676
apps/emqx/src/emqx_persistent_message_ds_replayer.erl
View File

@ -19,12 +19,18 @@
-module(emqx_persistent_message_ds_replayer).
%% API:
-export([new/0, next_packet_id/1, replay/2, commit_offset/3, poll/3, n_inflight/1]).
-export([new/0, open/1, next_packet_id/1, n_inflight/1]).
-export([poll/4, replay/2, commit_offset/4]).
-export([seqno_to_packet_id/1, packet_id_to_seqno/2]).
-export([committed_until/2]).
%% internal exports:
-export([]).
-export_type([inflight/0]).
-export_type([inflight/0, seqno/0]).
-include_lib("emqx/include/logger.hrl").
-include("emqx_persistent_session_ds.hrl").
@ -34,6 +40,13 @@
-include_lib("eunit/include/eunit.hrl").
-endif.
-define(EPOCH_SIZE, 16#10000).
-define(ACK, 0).
-define(COMP, 1).
-define(TRACK_FLAG(WHICH), (1 bsl WHICH)).
%%================================================================================
%% Type declarations
%%================================================================================
@ -41,23 +54,23 @@
%% Note: sequence numbers are monotonic; they don't wrap around:
-type seqno() :: non_neg_integer().
-record(range, {
stream :: emqx_ds:stream(),
first :: seqno(),
last :: seqno(),
iterator_next :: emqx_ds:iterator() | undefined
}).
-type range() :: #range{}.
-type track() :: ack | comp.
-type commit_type() :: rec.
-record(inflight, {
next_seqno = 0 :: seqno(),
acked_seqno = 0 :: seqno(),
offset_ranges = [] :: [range()]
next_seqno = 1 :: seqno(),
commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()},
%% Ranges are sorted in ascending order of their sequence numbers.
offset_ranges = [] :: [ds_pubrange()]
}).
-opaque inflight() :: #inflight{}.
-type reply_fun() :: fun(
(seqno(), emqx_types:message()) ->
emqx_session:replies() | {_AdvanceSeqno :: false, emqx_session:replies()}
).
%%================================================================================
%% API funcions
%%================================================================================
@ -66,85 +79,98 @@
new() ->
#inflight{}.
-spec open(emqx_persistent_session_ds:id()) -> inflight().
open(SessionId) ->
{Ranges, RecUntil} = ro_transaction(
fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end
),
{Commits, NextSeqno} = compute_inflight_range(Ranges),
#inflight{
commits = Commits#{rec => RecUntil},
next_seqno = NextSeqno,
offset_ranges = Ranges
}.
-spec next_packet_id(inflight()) -> {emqx_types:packet_id(), inflight()}.
next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqNo}) ->
Inflight = Inflight0#inflight{next_seqno = LastSeqNo + 1},
case LastSeqNo rem 16#10000 of
0 ->
%% We skip sequence numbers that lead to PacketId = 0 to
%% simplify math. Note: it leads to occasional gaps in the
%% sequence numbers.
next_packet_id(Inflight);
PacketId ->
{PacketId, Inflight}
end.
next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) ->
Inflight = Inflight0#inflight{next_seqno = next_seqno(LastSeqno)},
{seqno_to_packet_id(LastSeqno), Inflight}.
-spec n_inflight(inflight()) -> non_neg_integer().
n_inflight(#inflight{next_seqno = NextSeqNo, acked_seqno = AckedSeqno}) ->
%% NOTE: this function assumes that gaps in the sequence ID occur
%% _only_ when the packet ID wraps:
case AckedSeqno >= ((NextSeqNo bsr 16) bsl 16) of
true ->
NextSeqNo - AckedSeqno;
false ->
NextSeqNo - AckedSeqno - 1
end.
-spec replay(emqx_persistent_session_ds:id(), inflight()) ->
emqx_session:replies().
replay(_SessionId, _Inflight = #inflight{offset_ranges = _Ranges}) ->
[].
-spec commit_offset(emqx_persistent_session_ds:id(), emqx_types:packet_id(), inflight()) ->
{_IsValidOffset :: boolean(), inflight()}.
commit_offset(
SessionId,
PacketId,
Inflight0 = #inflight{
acked_seqno = AckedSeqno0, next_seqno = NextSeqNo, offset_ranges = Ranges0
}
) ->
AckedSeqno =
case packet_id_to_seqno(NextSeqNo, PacketId) of
N when N > AckedSeqno0; AckedSeqno0 =:= 0 ->
n_inflight(#inflight{offset_ranges = Ranges}) ->
%% TODO
%% This is not very efficient. Instead, we can take the maximum of
%% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`.
%% This won't be exact number but a pessimistic estimate, but this way we
%% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2
%% messages for some reason. For that to work, we need to additionally track
%% actual `AckedUntil` / `CompUntil` during `commit_offset/4`.
lists:foldl(
fun
(#ds_pubrange{type = ?T_CHECKPOINT}, N) ->
N;
OutOfRange ->
?SLOG(warning, #{
msg => "out-of-order_ack",
prev_seqno => AckedSeqno0,
acked_seqno => OutOfRange,
next_seqno => NextSeqNo,
packet_id => PacketId
}),
AckedSeqno0
(#ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until}, N) ->
N + range_size(First, Until)
end,
Ranges = lists:filter(
fun(#range{stream = Stream, last = LastSeqno, iterator_next = ItNext}) ->
case LastSeqno =< AckedSeqno of
true ->
%% This range has been fully
%% acked. Remove it and replace saved
%% iterator with the trailing iterator.
update_iterator(SessionId, Stream, ItNext),
false;
false ->
%% This range still has unacked
%% messages:
true
end
0,
Ranges
).
-spec replay(reply_fun(), inflight()) -> {emqx_session:replies(), inflight()}.
replay(ReplyFun, Inflight0 = #inflight{offset_ranges = Ranges0}) ->
{Ranges, Replies} = lists:mapfoldr(
fun(Range, Acc) ->
replay_range(ReplyFun, Range, Acc)
end,
[],
Ranges0
),
Inflight = Inflight0#inflight{acked_seqno = AckedSeqno, offset_ranges = Ranges},
{true, Inflight}.
Inflight = Inflight0#inflight{offset_ranges = Ranges},
{Replies, Inflight}.
-spec poll(emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
-spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) ->
{_IsValidOffset :: boolean(), inflight()}
when
Offset :: track() | commit_type().
commit_offset(
SessionId,
Track,
PacketId,
Inflight0 = #inflight{commits = Commits}
) when Track == ack orelse Track == comp ->
case validate_commit(Track, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
%% TODO
%% We do not preserve `CommitUntil` in the database. Instead, we discard
%% fully acked ranges from the database. In effect, this means that the
%% most recent `CommitUntil` the client has sent may be lost in case of a
%% crash or client loss.
Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}},
Inflight = discard_committed(SessionId, Inflight1),
{true, Inflight};
false ->
{false, Inflight0}
end;
commit_offset(
SessionId,
CommitType = rec,
PacketId,
Inflight0 = #inflight{commits = Commits}
) ->
case validate_commit(CommitType, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
update_committed_offset(SessionId, CommitType, CommitUntil),
Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}},
{true, Inflight};
false ->
{false, Inflight0}
end.
-spec poll(reply_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
{emqx_session:replies(), inflight()}.
poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff ->
#inflight{next_seqno = NextSeqNo0, acked_seqno = AckedSeqno} =
Inflight0,
poll(ReplyFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE ->
FetchThreshold = max(1, WindowSize div 2),
FreeSpace = AckedSeqno + WindowSize - NextSeqNo0,
FreeSpace = WindowSize - n_inflight(Inflight0),
case FreeSpace >= FetchThreshold of
false ->
%% TODO: this branch is meant to avoid fetching data from
@ -153,10 +179,27 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
%% client get stuck even?
{[], Inflight0};
true ->
%% TODO: Wrap this in `mria:async_dirty/2`?
Streams = shuffle(get_streams(SessionId)),
fetch(SessionId, Inflight0, Streams, FreeSpace, [])
fetch(ReplyFun, SessionId, Inflight0, Streams, FreeSpace, [])
end.
%% Which seqno this track is committed until.
%% "Until" means this is first seqno that is _not yet committed_ for this track.
-spec committed_until(track() | commit_type(), inflight()) -> seqno().
committed_until(Track, #inflight{commits = Commits}) ->
maps:get(Track, Commits).
-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
seqno_to_packet_id(Seqno) ->
Seqno rem ?EPOCH_SIZE.
%% Reconstruct session counter by adding most significant bits from
%% the current counter to the packet id.
-spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno().
packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) ->
packet_id_to_seqno_(NextSeqno, PacketId).
%%================================================================================
%% Internal exports
%%================================================================================
@ -165,87 +208,329 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
%% Internal functions
%%================================================================================
fetch(_SessionId, Inflight, _Streams = [], _N, Acc) ->
{lists:reverse(Acc), Inflight};
fetch(_SessionId, Inflight, _Streams, 0, Acc) ->
{lists:reverse(Acc), Inflight};
fetch(SessionId, Inflight0, [Stream | Streams], N, Publishes0) ->
#inflight{next_seqno = FirstSeqNo, offset_ranges = Ranges0} = Inflight0,
ItBegin = get_last_iterator(SessionId, Stream, Ranges0),
{ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N),
{NMessages, Publishes, Inflight1} =
lists:foldl(
fun(Msg, {N0, PubAcc0, InflightAcc0}) ->
{PacketId, InflightAcc} = next_packet_id(InflightAcc0),
PubAcc = [{PacketId, Msg} | PubAcc0],
{N0 + 1, PubAcc, InflightAcc}
end,
{0, Publishes0, Inflight0},
Messages
),
#inflight{next_seqno = LastSeqNo} = Inflight1,
case NMessages > 0 of
true ->
Range = #range{
first = FirstSeqNo,
last = LastSeqNo - 1,
stream = Stream,
iterator_next = ItEnd
compute_inflight_range([]) ->
{#{ack => 1, comp => 1}, 1};
compute_inflight_range(Ranges) ->
_RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges),
AckedUntil = find_committed_until(ack, Ranges),
CompUntil = find_committed_until(comp, Ranges),
Commits = #{
ack => emqx_maybe:define(AckedUntil, LastSeqno),
comp => emqx_maybe:define(CompUntil, LastSeqno)
},
Inflight = Inflight1#inflight{offset_ranges = Ranges0 ++ [Range]},
fetch(SessionId, Inflight, Streams, N - NMessages, Publishes);
false ->
fetch(SessionId, Inflight1, Streams, N, Publishes)
end.
{Commits, LastSeqno}.
-spec update_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream(), emqx_ds:iterator()) -> ok.
update_iterator(DSSessionId, Stream, Iterator) ->
%% Workaround: we convert `Stream' to a binary before attempting to store it in
%% mnesia(rocksdb) because of a bug in `mnesia_rocksdb' when trying to do
%% `mnesia:dirty_all_keys' later.
StreamBin = term_to_binary(Stream),
mria:dirty_write(?SESSION_ITER_TAB, #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator}).
get_last_iterator(SessionId, Stream, Ranges) ->
case lists:keyfind(Stream, #range.stream, lists:reverse(Ranges)) of
false ->
get_iterator(SessionId, Stream);
#range{iterator_next = Next} ->
Next
end.
-spec get_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream()) -> emqx_ds:iterator().
get_iterator(DSSessionId, Stream) ->
%% See comment in `update_iterator'.
StreamBin = term_to_binary(Stream),
Id = {DSSessionId, StreamBin},
[#ds_iter{iter = It}] = mnesia:dirty_read(?SESSION_ITER_TAB, Id),
It.
-spec get_streams(emqx_persistent_session_ds:id()) -> [emqx_ds:stream()].
get_streams(SessionId) ->
lists:map(
fun(#ds_stream{stream = Stream}) ->
Stream
find_committed_until(Track, Ranges) ->
RangesUncommitted = lists:dropwhile(
fun(Range) ->
case Range of
#ds_pubrange{type = ?T_CHECKPOINT} ->
true;
#ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} ->
not has_track(Track, Tracks)
end
end,
mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId)
Ranges
),
case RangesUncommitted of
[#ds_pubrange{id = {_, CommittedUntil}} | _] ->
CommittedUntil;
[] ->
undefined
end.
-spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()].
get_ranges(SessionId) ->
Pat = erlang:make_tuple(
record_info(size, ds_pubrange),
'_',
[{1, ds_pubrange}, {#ds_pubrange.id, {SessionId, '_'}}]
),
mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read).
fetch(ReplyFun, SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 ->
#inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0,
ItBegin = get_last_iterator(DSStream, Ranges),
{ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N),
case Messages of
[] ->
fetch(ReplyFun, SessionId, Inflight0, Streams, N, Acc);
_ ->
%% We need to preserve the iterator pointing to the beginning of the
%% range, so that we can replay it if needed.
{Publishes, {UntilSeqno, Tracks}} = publish(ReplyFun, FirstSeqno, Messages),
Size = range_size(FirstSeqno, UntilSeqno),
Range0 = #ds_pubrange{
id = {SessionId, FirstSeqno},
type = ?T_INFLIGHT,
tracks = Tracks,
until = UntilSeqno,
stream = DSStream#ds_stream.ref,
iterator = ItBegin
},
ok = preserve_range(Range0),
%% ...Yet we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off: it will become
%% `ItBegin` of the next range for this stream.
Range = keep_next_iterator(ItEnd, Range0),
Inflight = Inflight0#inflight{
next_seqno = UntilSeqno,
offset_ranges = Ranges ++ [Range]
},
fetch(ReplyFun, SessionId, Inflight, Streams, N - Size, [Publishes | Acc])
end;
fetch(_ReplyFun, _SessionId, Inflight, _Streams, _N, Acc) ->
Publishes = lists:append(lists:reverse(Acc)),
{Publishes, Inflight}.
discard_committed(
SessionId,
Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0}
) ->
%% TODO: This could be kept and incrementally updated in the inflight state.
Checkpoints = find_checkpoints(Ranges0),
%% TODO: Wrap this in `mria:async_dirty/2`?
Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0),
Inflight0#inflight{offset_ranges = Ranges}.
find_checkpoints(Ranges) ->
lists:foldl(
fun(#ds_pubrange{stream = StreamRef, until = Until}, Acc) ->
%% For each stream, remember the last range over this stream.
Acc#{StreamRef => Until}
end,
#{},
Ranges
).
%% Reconstruct session counter by adding most significant bits from
%% the current counter to the packet id.
-spec packet_id_to_seqno(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer().
packet_id_to_seqno(NextSeqNo, PacketId) ->
Epoch = NextSeqNo bsr 16,
case packet_id_to_seqno_(Epoch, PacketId) of
N when N =< NextSeqNo ->
N;
_ ->
packet_id_to_seqno_(Epoch - 1, PacketId)
discard_committed_ranges(
SessionId,
Commits,
Checkpoints,
Ranges = [Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest]
) ->
case discard_committed_range(Commits, Range) of
discard ->
%% This range has been fully committed.
%% Either discard it completely, or preserve the iterator for the next range
%% over this stream (i.e. a checkpoint).
RangeKept =
case maps:get(StreamRef, Checkpoints) of
CP when CP > Until ->
discard_range(Range),
[];
Until ->
[checkpoint_range(Range)]
end,
%% Since we're (intentionally) not using transactions here, it's important to
%% issue database writes in the same order in which ranges are stored: from
%% the oldest to the newest. This is also why we need to compute which ranges
%% should become checkpoints before we start writing anything.
RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest);
keep ->
%% This range has not been fully committed.
[Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)];
keep_all ->
%% The rest of ranges (if any) still have uncommitted messages.
Ranges;
TracksLeft ->
%% Only some track has been committed.
%% Preserve the uncommitted tracks in the database.
RangeKept = Range#ds_pubrange{tracks = TracksLeft},
preserve_range(restore_first_iterator(RangeKept)),
[RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]
end;
discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) ->
[].
discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) ->
discard;
discard_committed_range(
#{ack := AckedUntil, comp := CompUntil},
#ds_pubrange{until = Until}
) when Until > AckedUntil andalso Until > CompUntil ->
keep_all;
discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) ->
case discard_tracks(Commits, Until, Tracks) of
0 ->
discard;
Tracks ->
keep;
TracksLeft ->
TracksLeft
end.
-spec packet_id_to_seqno_(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer().
packet_id_to_seqno_(Epoch, PacketId) ->
(Epoch bsl 16) + PacketId.
discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) ->
TAck =
case Until > AckedUntil of
true -> ?TRACK_FLAG(?ACK) band Tracks;
false -> 0
end,
TComp =
case Until > CompUntil of
true -> ?TRACK_FLAG(?COMP) band Tracks;
false -> 0
end,
TAck bor TComp.
replay_range(
ReplyFun,
Range0 = #ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until, iterator = It},
Acc
) ->
Size = range_size(First, Until),
{ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
%% Asserting that range is consistent with the message storage state.
{Replies, {Until, _TracksInitial}} = publish(ReplyFun, First, MessagesUnacked),
%% Again, we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off.
Range = keep_next_iterator(ItNext, Range0),
{Range, Replies ++ Acc};
replay_range(_ReplyFun, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) ->
{Range0, Acc}.
validate_commit(
Track,
PacketId,
Inflight = #inflight{commits = Commits, next_seqno = NextSeqno}
) ->
Seqno = packet_id_to_seqno_(NextSeqno, PacketId),
CommittedUntil = maps:get(Track, Commits),
CommitNext = get_commit_next(Track, Inflight),
case Seqno >= CommittedUntil andalso Seqno < CommitNext of
true ->
next_seqno(Seqno);
false ->
?SLOG(warning, #{
msg => "out-of-order_commit",
track => Track,
packet_id => PacketId,
commit_seqno => Seqno,
committed_until => CommittedUntil,
commit_next => CommitNext
}),
false
end.
get_commit_next(ack, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(rec, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(comp, #inflight{commits = Commits}) ->
maps:get(rec, Commits).
publish(ReplyFun, FirstSeqno, Messages) ->
lists:mapfoldl(
fun(Message, {Seqno, TAcc}) ->
case ReplyFun(Seqno, Message) of
{_Advance = false, Reply} ->
{Reply, {Seqno, TAcc}};
Reply ->
NextSeqno = next_seqno(Seqno),
NextTAcc = add_msg_track(Message, TAcc),
{Reply, {NextSeqno, NextTAcc}}
end
end,
{FirstSeqno, 0},
Messages
).
add_msg_track(Message, Tracks) ->
case emqx_message:qos(Message) of
1 -> ?TRACK_FLAG(?ACK) bor Tracks;
2 -> ?TRACK_FLAG(?COMP) bor Tracks;
_ -> Tracks
end.
keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) ->
Range#ds_pubrange{
iterator = ItNext,
%% We need to keep the first iterator around, in case we need to preserve
%% this range again, updating still uncommitted tracks it's part of.
misc = Misc#{iterator_first => ItFirst}
}.
restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) ->
Range#ds_pubrange{
iterator = ItFirst,
misc = maps:remove(iterator_first, Misc)
}.
-spec preserve_range(ds_pubrange()) -> ok.
preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) ->
mria:dirty_write(?SESSION_PUBRANGE_TAB, Range).
has_track(ack, Tracks) ->
(?TRACK_FLAG(?ACK) band Tracks) > 0;
has_track(comp, Tracks) ->
(?TRACK_FLAG(?COMP) band Tracks) > 0.
-spec discard_range(ds_pubrange()) -> ok.
discard_range(#ds_pubrange{id = RangeId}) ->
mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId).
-spec checkpoint_range(ds_pubrange()) -> ds_pubrange().
checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) ->
Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}},
ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range),
Range;
checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) ->
%% This range should have been checkpointed already.
Range.
get_last_iterator(DSStream = #ds_stream{ref = StreamRef}, Ranges) ->
case lists:keyfind(StreamRef, #ds_pubrange.stream, lists:reverse(Ranges)) of
false ->
DSStream#ds_stream.beginning;
#ds_pubrange{iterator = ItNext} ->
ItNext
end.
-spec get_streams(emqx_persistent_session_ds:id()) -> [ds_stream()].
get_streams(SessionId) ->
mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId).
-spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno().
get_committed_offset(SessionId, Name) ->
case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of
[] ->
1;
[#ds_committed_offset{until = Seqno}] ->
Seqno
end.
-spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok.
update_committed_offset(SessionId, Name, Until) ->
mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{
id = {SessionId, Name}, until = Until
}).
next_seqno(Seqno) ->
NextSeqno = Seqno + 1,
case seqno_to_packet_id(NextSeqno) of
0 ->
%% We skip sequence numbers that lead to PacketId = 0 to
%% simplify math. Note: it leads to occasional gaps in the
%% sequence numbers.
NextSeqno + 1;
_ ->
NextSeqno
end.
packet_id_to_seqno_(NextSeqno, PacketId) ->
Epoch = NextSeqno bsr 16,
case (Epoch bsl 16) + PacketId of
N when N =< NextSeqno ->
N;
N ->
N - ?EPOCH_SIZE
end.
range_size(FirstSeqno, UntilSeqno) ->
%% This function assumes that gaps in the sequence ID occur _only_ when the
%% packet ID wraps.
Size = UntilSeqno - FirstSeqno,
Size + (FirstSeqno bsr 16) - (UntilSeqno bsr 16).
-spec shuffle([A]) -> [A].
shuffle(L0) ->
@ -259,24 +544,28 @@ shuffle(L0) ->
{_, L} = lists:unzip(L2),
L.
ro_transaction(Fun) ->
{atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
Res.
-ifdef(TEST).
%% This test only tests boundary conditions (to make sure property-based test didn't skip them):
packet_id_to_seqno_test() ->
%% Packet ID = 1; first epoch:
?assertEqual(1, packet_id_to_seqno(1, 1)),
?assertEqual(1, packet_id_to_seqno(10, 1)),
?assertEqual(1, packet_id_to_seqno(1 bsl 16 - 1, 1)),
?assertEqual(1, packet_id_to_seqno(1 bsl 16, 1)),
?assertEqual(1, packet_id_to_seqno_(1, 1)),
?assertEqual(1, packet_id_to_seqno_(10, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)),
%% Packet ID = 1; second and 3rd epochs:
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(1 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(2 bsl 16, 1)),
?assertEqual(2 bsl 16 + 1, packet_id_to_seqno(2 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)),
?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)),
%% Packet ID = 16#ffff:
PID = 1 bsl 16 - 1,
?assertEqual(PID, packet_id_to_seqno(PID, PID)),
?assertEqual(PID, packet_id_to_seqno(1 bsl 16, PID)),
?assertEqual(1 bsl 16 + PID, packet_id_to_seqno(2 bsl 16, PID)),
?assertEqual(PID, packet_id_to_seqno_(PID, PID)),
?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)),
?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)),
ok.
packet_id_to_seqno_test_() ->
@ -291,8 +580,8 @@ packet_id_to_seqno_prop() ->
SeqNo,
seqno_gen(NextSeqNo),
begin
PacketId = SeqNo rem 16#10000,
?assertEqual(SeqNo, packet_id_to_seqno(NextSeqNo, PacketId)),
PacketId = seqno_to_packet_id(SeqNo),
?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)),
true
end
)
@ -311,4 +600,55 @@ seqno_gen(NextSeqNo) ->
Max = max(0, NextSeqNo - 1),
range(Min, Max).
range_size_test_() ->
[
?_assertEqual(0, range_size(42, 42)),
?_assertEqual(1, range_size(42, 43)),
?_assertEqual(1, range_size(16#ffff, 16#10001)),
?_assertEqual(16#ffff - 456 + 123, range_size(16#1f0000 + 456, 16#200000 + 123))
].
compute_inflight_range_test_() ->
[
?_assertEqual(
{#{ack => 1, comp => 1}, 1},
compute_inflight_range([])
),
?_assertEqual(
{#{ack => 12, comp => 13}, 42},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{
id = {<<>>, 12},
until = 13,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK)
},
#ds_pubrange{
id = {<<>>, 13},
until = 20,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?COMP)
},
#ds_pubrange{
id = {<<>>, 20},
until = 42,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)
}
])
),
?_assertEqual(
{#{ack => 13, comp => 13}, 13},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 12}, until = 13, type = ?T_CHECKPOINT}
])
)
].
-endif.

567
apps/emqx/src/emqx_persistent_session_ds.erl
View File

@ -56,13 +56,16 @@
deliver/3,
replay/3,
handle_timeout/3,
disconnect/1,
disconnect/2,
terminate/2
]).
%% session table operations
-export([create_tables/0]).
%% internal export used by session GC process
-export([destroy_session/1]).
%% Remove me later (satisfy checks for an unused BPAPI)
-export([
do_open_iterator/3,
@ -70,24 +73,27 @@
do_ensure_all_iterators_closed/1
]).
-export([print_session/1]).
-ifdef(TEST).
-export([
session_open/1,
session_open/2,
list_all_sessions/0,
list_all_subscriptions/0,
list_all_streams/0,
list_all_iterators/0
list_all_pubranges/0
]).
-endif.
%% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be
%% an atom, in theory (?).
-type id() :: binary().
-type topic_filter() :: emqx_ds:topic_filter().
-type topic_filter() :: emqx_types:topic().
-type topic_filter_words() :: emqx_ds:topic_filter().
-type subscription_id() :: {id(), topic_filter()}.
-type subscription() :: #{
start_time := emqx_ds:time(),
propts := map(),
props := map(),
extra := map()
}.
-type session() :: #{
@ -95,23 +101,26 @@
id := id(),
%% When the session was created
created_at := timestamp(),
%% When the session should expire
expires_at := timestamp() | never,
%% When the client was last considered alive
last_alive_at := timestamp(),
%% Client’s Subscriptions.
iterators := #{topic() => subscription()},
subscriptions := #{topic_filter() => subscription()},
%% Inflight messages
inflight := emqx_persistent_message_ds_replayer:inflight(),
%% Receive maximum
receive_maximum := pos_integer(),
%% Connection Info
conninfo := emqx_types:conninfo(),
%%
props := map()
}.
-type timestamp() :: emqx_utils_calendar:epoch_millisecond().
-type topic() :: emqx_types:topic().
-type millisecond() :: non_neg_integer().
-type clientinfo() :: emqx_types:clientinfo().
-type conninfo() :: emqx_session:conninfo().
-type replies() :: emqx_session:replies().
-type timer() :: pull | get_streams | bump_last_alive_at.
-define(STATS_KEYS, [
subscriptions_cnt,
@ -121,6 +130,12 @@
next_pkt_id
]).
-define(IS_EXPIRED(NOW_MS, LAST_ALIVE_AT, EI),
(is_number(LAST_ALIVE_AT) andalso
is_number(EI) andalso
(NOW_MS >= LAST_ALIVE_AT + EI))
).
-export_type([id/0]).
%%
@ -142,7 +157,7 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) ->
%% somehow isolate those idling not-yet-expired sessions into a separate process
%% space, and move this call back into `emqx_cm` where it belongs.
ok = emqx_cm:discard_session(ClientID),
case open_session(ClientID) of
case session_open(ClientID, ConnInfo) of
Session0 = #{} ->
ensure_timers(),
ReceiveMaximum = receive_maximum(ConnInfo),
@ -153,24 +168,13 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) ->
end.
ensure_session(ClientID, ConnInfo, Conf) ->
{ok, Session, #{}} = session_ensure_new(ClientID, Conf),
Session = session_ensure_new(ClientID, ConnInfo, Conf),
ReceiveMaximum = receive_maximum(ConnInfo),
Session#{iterators => #{}, receive_maximum => ReceiveMaximum}.
open_session(ClientID) ->
case session_open(ClientID) of
{ok, Session, Subscriptions} ->
Session#{iterators => prep_subscriptions(Subscriptions)};
false ->
false
end.
prep_subscriptions(Subscriptions) ->
maps:fold(
fun(Topic, Subscription, Acc) -> Acc#{emqx_topic:join(Topic) => Subscription} end,
#{},
Subscriptions
).
Session#{
conninfo => ConnInfo,
receive_maximum => ReceiveMaximum,
subscriptions => #{}
}.
-spec destroy(session() | clientinfo()) -> ok.
destroy(#{id := ClientID}) ->
@ -195,9 +199,9 @@ info(created_at, #{created_at := CreatedAt}) ->
CreatedAt;
info(is_persistent, #{}) ->
true;
info(subscriptions, #{iterators := Iters}) ->
info(subscriptions, #{subscriptions := Iters}) ->
maps:map(fun(_, #{props := SubOpts}) -> SubOpts end, Iters);
info(subscriptions_cnt, #{iterators := Iters}) ->
info(subscriptions_cnt, #{subscriptions := Iters}) ->
maps:size(Iters);
info(subscriptions_max, #{props := Conf}) ->
maps:get(max_subscriptions, Conf);
@ -235,51 +239,71 @@ info(await_rel_timeout, #{props := Conf}) ->
stats(Session) ->
info(?STATS_KEYS, Session).
%% Debug/troubleshooting
-spec print_session(emqx_types:client_id()) -> map() | undefined.
print_session(ClientId) ->
catch ro_transaction(
fun() ->
case mnesia:read(?SESSION_TAB, ClientId) of
[Session] ->
#{
session => Session,
streams => mnesia:read(?SESSION_STREAM_TAB, ClientId),
pubranges => session_read_pubranges(ClientId),
offsets => session_read_offsets(ClientId),
subscriptions => session_read_subscriptions(ClientId)
};
[] ->
undefined
end
end
).
%%--------------------------------------------------------------------
%% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE
%%--------------------------------------------------------------------
-spec subscribe(topic(), emqx_types:subopts(), session()) ->
-spec subscribe(topic_filter(), emqx_types:subopts(), session()) ->
{ok, session()} | {error, emqx_types:reason_code()}.
subscribe(
TopicFilter,
SubOpts,
Session = #{id := ID, iterators := Iters}
) when is_map_key(TopicFilter, Iters) ->
Iterator = maps:get(TopicFilter, Iters),
NIterator = update_subscription(TopicFilter, Iterator, SubOpts, ID),
{ok, Session#{iterators := Iters#{TopicFilter => NIterator}}};
Session = #{id := ID, subscriptions := Subs}
) when is_map_key(TopicFilter, Subs) ->
Subscription = maps:get(TopicFilter, Subs),
NSubscription = update_subscription(TopicFilter, Subscription, SubOpts, ID),
{ok, Session#{subscriptions := Subs#{TopicFilter => NSubscription}}};
subscribe(
TopicFilter,
SubOpts,
Session = #{id := ID, iterators := Iters}
Session = #{id := ID, subscriptions := Subs}
) ->
% TODO: max_subscriptions
Iterator = add_subscription(TopicFilter, SubOpts, ID),
{ok, Session#{iterators := Iters#{TopicFilter => Iterator}}}.
Subscription = add_subscription(TopicFilter, SubOpts, ID),
{ok, Session#{subscriptions := Subs#{TopicFilter => Subscription}}}.
-spec unsubscribe(topic(), session()) ->
-spec unsubscribe(topic_filter(), session()) ->
{ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}.
unsubscribe(
TopicFilter,
Session = #{id := ID, iterators := Iters}
) when is_map_key(TopicFilter, Iters) ->
Iterator = maps:get(TopicFilter, Iters),
SubOpts = maps:get(props, Iterator),
Session = #{id := ID, subscriptions := Subs}
) when is_map_key(TopicFilter, Subs) ->
Subscription = maps:get(TopicFilter, Subs),
SubOpts = maps:get(props, Subscription),
ok = del_subscription(TopicFilter, ID),
{ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts};
{ok, Session#{subscriptions := maps:remove(TopicFilter, Subs)}, SubOpts};
unsubscribe(
_TopicFilter,
_Session = #{}
) ->
{error, ?RC_NO_SUBSCRIPTION_EXISTED}.
-spec get_subscription(topic(), session()) ->
-spec get_subscription(topic_filter(), session()) ->
emqx_types:subopts() | undefined.
get_subscription(TopicFilter, #{iterators := Iters}) ->
case maps:get(TopicFilter, Iters, undefined) of
Iterator = #{} ->
maps:get(props, Iterator);
get_subscription(TopicFilter, #{subscriptions := Subs}) ->
case maps:get(TopicFilter, Subs, undefined) of
Subscription = #{} ->
maps:get(props, Subscription);
undefined ->
undefined
end.
@ -289,12 +313,12 @@ get_subscription(TopicFilter, #{iterators := Iters}) ->
%%--------------------------------------------------------------------
-spec publish(emqx_types:packet_id(), emqx_types:message(), session()) ->
{ok, emqx_types:publish_result(), replies(), session()}
{ok, emqx_types:publish_result(), session()}
| {error, emqx_types:reason_code()}.
publish(_PacketId, Msg, Session) ->
%% TODO:
%% TODO: QoS2
Result = emqx_broker:publish(Msg),
{ok, Result, [], Session}.
{ok, Result, Session}.
%%--------------------------------------------------------------------
%% Client -> Broker: PUBACK
@ -307,12 +331,13 @@ publish(_PacketId, Msg, Session) ->
{ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}.
puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, PacketId, Inflight0) of
case emqx_persistent_message_ds_replayer:commit_offset(Id, ack, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = #message{},
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, [], Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
@ -323,9 +348,16 @@ puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
-spec pubrec(emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), session()}
| {error, emqx_types:reason_code()}.
pubrec(_PacketId, _Session = #{}) ->
% TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
pubrec(PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, rec, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
%%--------------------------------------------------------------------
%% Client -> Broker: PUBREL
@ -344,16 +376,23 @@ pubrel(_PacketId, Session = #{}) ->
-spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}.
pubcomp(_ClientInfo, _PacketId, _Session = #{}) ->
% TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
pubcomp(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, comp, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, [], Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
%%--------------------------------------------------------------------
-spec deliver(clientinfo(), [emqx_types:deliver()], session()) ->
{ok, replies(), session()}.
deliver(_ClientInfo, _Delivers, Session) ->
%% TODO: QoS0 and system messages end up here.
%% TODO: system messages end up here.
{ok, [], Session}.
-spec handle_timeout(clientinfo(), _Timeout, session()) ->
@ -363,43 +402,82 @@ handle_timeout(
pull,
Session = #{id := Id, inflight := Inflight0, receive_maximum := ReceiveMaximum}
) ->
{Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(Id, Inflight0, ReceiveMaximum),
%% TODO: make these values configurable:
{Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(
fun
(_Seqno, Message = #message{qos = ?QOS_0}) ->
{false, {undefined, Message}};
(Seqno, Message) ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{PacketId, Message}
end,
Id,
Inflight0,
ReceiveMaximum
),
IdlePollInterval = emqx_config:get([session_persistence, idle_poll_interval]),
Timeout =
case Publishes of
[] ->
100;
IdlePollInterval;
[_ | _] ->
0
end,
ensure_timer(pull, Timeout),
{ok, Publishes, Session#{inflight => Inflight}};
handle_timeout(_ClientInfo, get_streams, Session = #{id := Id}) ->
renew_streams(Id),
{ok, Publishes, Session#{inflight := Inflight}};
handle_timeout(_ClientInfo, get_streams, Session) ->
renew_streams(Session),
ensure_timer(get_streams),
{ok, [], Session};
handle_timeout(_ClientInfo, bump_last_alive_at, Session0) ->
%% Note: we take a pessimistic approach here and assume that the client will be alive
%% until the next bump timeout. With this, we avoid garbage collecting this session
%% too early in case the session/connection/node crashes earlier without having time
%% to commit the time.
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
EstimatedLastAliveAt = now_ms() + BumpInterval,
Session = session_set_last_alive_at_trans(Session0, EstimatedLastAliveAt),
ensure_timer(bump_last_alive_at),
{ok, [], Session}.
-spec replay(clientinfo(), [], session()) ->
{ok, replies(), session()}.
replay(_ClientInfo, [], Session = #{}) ->
{ok, [], Session}.
replay(_ClientInfo, [], Session = #{inflight := Inflight0}) ->
AckedUntil = emqx_persistent_message_ds_replayer:committed_until(ack, Inflight0),
RecUntil = emqx_persistent_message_ds_replayer:committed_until(rec, Inflight0),
CompUntil = emqx_persistent_message_ds_replayer:committed_until(comp, Inflight0),
ReplyFun = fun
(_Seqno, #message{qos = ?QOS_0}) ->
{false, []};
(Seqno, #message{qos = ?QOS_1}) when Seqno < AckedUntil ->
[];
(Seqno, #message{qos = ?QOS_2}) when Seqno < CompUntil ->
[];
(Seqno, #message{qos = ?QOS_2}) when Seqno < RecUntil ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{pubrel, PacketId};
(Seqno, Message) ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{PacketId, emqx_message:set_flag(dup, true, Message)}
end,
{Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(ReplyFun, Inflight0),
{ok, Replies, Session#{inflight := Inflight}}.
%%--------------------------------------------------------------------
-spec disconnect(session()) -> {shutdown, session()}.
disconnect(Session = #{}) ->
-spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}.
disconnect(Session0, ConnInfo) ->
Session = session_set_last_alive_at_trans(Session0, ConnInfo, now_ms()),
{shutdown, Session}.
-spec terminate(Reason :: term(), session()) -> ok.
terminate(_Reason, _Session = #{}) ->
% TODO: close iterators
ok.
%%--------------------------------------------------------------------
-spec add_subscription(topic(), emqx_types:subopts(), id()) ->
-spec add_subscription(topic_filter(), emqx_types:subopts(), id()) ->
subscription().
add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
add_subscription(TopicFilter, SubOpts, DSSessionID) ->
%% N.B.: we chose to update the router before adding the subscription to the
%% session/iterator table. The reasoning for this is as follows:
%%
@ -418,8 +496,7 @@ add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
%% since it is guarded by a transaction context: we consider a subscription
%% operation to be successful if it ended up changing this table. Both router
%% and iterator information can be reconstructed from this table, if needed.
ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID),
TopicFilter = emqx_topic:words(TopicFilterBin),
ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, DSSessionID),
{ok, DSSubExt, IsNew} = session_add_subscription(
DSSessionID, TopicFilter, SubOpts
),
@ -427,20 +504,19 @@ add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
%% we'll list streams and open iterators when implementing message replay.
DSSubExt.
-spec update_subscription(topic(), subscription(), emqx_types:subopts(), id()) ->
-spec update_subscription(topic_filter(), subscription(), emqx_types:subopts(), id()) ->
subscription().
update_subscription(TopicFilterBin, DSSubExt, SubOpts, DSSessionID) ->
TopicFilter = emqx_topic:words(TopicFilterBin),
update_subscription(TopicFilter, DSSubExt, SubOpts, DSSessionID) ->
{ok, NDSSubExt, false} = session_add_subscription(
DSSessionID, TopicFilter, SubOpts
),
ok = ?tp(persistent_session_ds_iterator_updated, #{sub => DSSubExt}),
NDSSubExt.
-spec del_subscription(topic(), id()) ->
-spec del_subscription(topic_filter(), id()) ->
ok.
del_subscription(TopicFilterBin, DSSessionId) ->
TopicFilter = emqx_topic:words(TopicFilterBin),
del_subscription(TopicFilter, DSSessionId) ->
%% TODO: transaction?
?tp_span(
persistent_session_ds_subscription_delete,
#{session_id => DSSessionId},
@ -449,7 +525,7 @@ del_subscription(TopicFilterBin, DSSessionId) ->
?tp_span(
persistent_session_ds_subscription_route_delete,
#{session_id => DSSessionId},
ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId)
ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, DSSessionId)
).
%%--------------------------------------------------------------------
@ -457,10 +533,6 @@ del_subscription(TopicFilterBin, DSSessionId) ->
%%--------------------------------------------------------------------
create_tables() ->
ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{
backend => builtin,
storage => {emqx_ds_storage_bitfield_lts, #{}}
}),
ok = mria:create_table(
?SESSION_TAB,
[
@ -492,17 +564,31 @@ create_tables() ->
]
),
ok = mria:create_table(
?SESSION_ITER_TAB,
?SESSION_PUBRANGE_TAB,
[
{rlog_shard, ?DS_MRIA_SHARD},
{type, ordered_set},
{storage, storage()},
{record_name, ds_pubrange},
{attributes, record_info(fields, ds_pubrange)}
]
),
ok = mria:create_table(
?SESSION_COMMITTED_OFFSET_TAB,
[
{rlog_shard, ?DS_MRIA_SHARD},
{type, set},
{storage, storage()},
{record_name, ds_iter},
{attributes, record_info(fields, ds_iter)}
{record_name, ds_committed_offset},
{attributes, record_info(fields, ds_committed_offset)}
]
),
ok = mria:wait_for_tables([
?SESSION_TAB, ?SESSION_SUBSCRIPTIONS_TAB, ?SESSION_STREAM_TAB, ?SESSION_ITER_TAB
?SESSION_TAB,
?SESSION_SUBSCRIPTIONS_TAB,
?SESSION_STREAM_TAB,
?SESSION_PUBRANGE_TAB,
?SESSION_COMMITTED_OFFSET_TAB
]),
ok.
@ -521,60 +607,103 @@ storage() ->
%%
%% Note: session API doesn't handle session takeovers, it's the job of
%% the broker.
-spec session_open(id()) ->
{ok, session(), #{topic() => subscription()}} | false.
session_open(SessionId) ->
-spec session_open(id(), emqx_types:conninfo()) ->
session() | false.
session_open(SessionId, NewConnInfo) ->
NowMS = now_ms(),
transaction(fun() ->
case mnesia:read(?SESSION_TAB, SessionId, write) of
[Record = #session{}] ->
[Record0 = #session{last_alive_at = LastAliveAt, conninfo = ConnInfo}] ->
EI = expiry_interval(ConnInfo),
case ?IS_EXPIRED(NowMS, LastAliveAt, EI) of
true ->
session_drop(SessionId),
false;
false ->
%% new connection being established
Record1 = Record0#session{conninfo = NewConnInfo},
Record = session_set_last_alive_at(Record1, NowMS),
Session = export_session(Record),
DSSubs = session_read_subscriptions(SessionId),
Subscriptions = export_subscriptions(DSSubs),
{ok, Session, Subscriptions};
[] ->
Inflight = emqx_persistent_message_ds_replayer:open(SessionId),
Session#{
conninfo => NewConnInfo,
inflight => Inflight,
subscriptions => Subscriptions
}
end;
_ ->
false
end
end).
-spec session_ensure_new(id(), _Props :: map()) ->
{ok, session(), #{topic() => subscription()}}.
session_ensure_new(SessionId, Props) ->
-spec session_ensure_new(id(), emqx_types:conninfo(), _Props :: map()) ->
session().
session_ensure_new(SessionId, ConnInfo, Props) ->
transaction(fun() ->
ok = session_drop_subscriptions(SessionId),
Session = export_session(session_create(SessionId, Props)),
{ok, Session, #{}}
Session = export_session(session_create(SessionId, ConnInfo, Props)),
Session#{
subscriptions => #{},
inflight => emqx_persistent_message_ds_replayer:new()
}
end).
session_create(SessionId, Props) ->
session_create(SessionId, ConnInfo, Props) ->
Session = #session{
id = SessionId,
created_at = erlang:system_time(millisecond),
expires_at = never,
props = Props,
inflight = emqx_persistent_message_ds_replayer:new()
created_at = now_ms(),
last_alive_at = now_ms(),
conninfo = ConnInfo,
props = Props
},
ok = mnesia:write(?SESSION_TAB, Session, write),
Session.
session_set_last_alive_at_trans(Session, LastAliveAt) ->
#{conninfo := ConnInfo} = Session,
session_set_last_alive_at_trans(Session, ConnInfo, LastAliveAt).
session_set_last_alive_at_trans(Session, NewConnInfo, LastAliveAt) ->
#{id := SessionId} = Session,
transaction(fun() ->
case mnesia:read(?SESSION_TAB, SessionId, write) of
[#session{} = SessionRecord0] ->
SessionRecord = SessionRecord0#session{conninfo = NewConnInfo},
_ = session_set_last_alive_at(SessionRecord, LastAliveAt),
ok;
_ ->
%% log and crash?
ok
end
end),
Session#{conninfo := NewConnInfo, last_alive_at := LastAliveAt}.
session_set_last_alive_at(SessionRecord0, LastAliveAt) ->
SessionRecord = SessionRecord0#session{last_alive_at = LastAliveAt},
ok = mnesia:write(?SESSION_TAB, SessionRecord, write),
SessionRecord.
%% @doc Called when a client reconnects with `clean session=true' or
%% during session GC
-spec session_drop(id()) -> ok.
session_drop(DSSessionId) ->
transaction(fun() ->
ok = session_drop_subscriptions(DSSessionId),
ok = session_drop_iterators(DSSessionId),
ok = session_drop_pubranges(DSSessionId),
ok = session_drop_offsets(DSSessionId),
ok = session_drop_streams(DSSessionId),
ok = mnesia:delete(?SESSION_TAB, DSSessionId, write)
end).
-spec session_drop_subscriptions(id()) -> ok.
session_drop_subscriptions(DSSessionId) ->
Subscriptions = session_read_subscriptions(DSSessionId),
Subscriptions = session_read_subscriptions(DSSessionId, write),
lists:foreach(
fun(#ds_sub{id = DSSubId} = DSSub) ->
TopicFilter = subscription_id_to_topic_filter(DSSubId),
TopicFilterBin = emqx_topic:join(TopicFilter),
ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId),
ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, DSSessionId),
ok = session_del_subscription(DSSub)
end,
Subscriptions
@ -633,19 +762,44 @@ session_del_subscription(DSSessionId, TopicFilter) ->
session_del_subscription(#ds_sub{id = DSSubId}) ->
mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write).
session_read_subscriptions(DSSessionId) ->
session_read_subscriptions(DSSessionID) ->
session_read_subscriptions(DSSessionID, read).
session_read_subscriptions(DSSessionId, LockKind) ->
MS = ets:fun2ms(
fun(Sub = #ds_sub{id = {Sess, _}}) when Sess =:= DSSessionId ->
Sub
end
),
mnesia:select(?SESSION_SUBSCRIPTIONS_TAB, MS, read).
mnesia:select(?SESSION_SUBSCRIPTIONS_TAB, MS, LockKind).
session_read_pubranges(DSSessionID) ->
session_read_pubranges(DSSessionID, read).
session_read_pubranges(DSSessionId, LockKind) ->
MS = ets:fun2ms(
fun(#ds_pubrange{id = {Sess, First}}) when Sess =:= DSSessionId ->
{DSSessionId, First}
end
),
mnesia:select(?SESSION_PUBRANGE_TAB, MS, LockKind).
session_read_offsets(DSSessionID) ->
session_read_offsets(DSSessionID, read).
session_read_offsets(DSSessionId, LockKind) ->
MS = ets:fun2ms(
fun(#ds_committed_offset{id = {Sess, Type}}) when Sess =:= DSSessionId ->
{DSSessionId, Type}
end
),
mnesia:select(?SESSION_COMMITTED_OFFSET_TAB, MS, LockKind).
-spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), integer()}.
new_subscription_id(DSSessionId, TopicFilter) ->
%% Note: here we use _milliseconds_ to match with the timestamp
%% field of `#message' record.
NowMS = erlang:system_time(millisecond),
NowMS = now_ms(),
DSSubId = {DSSessionId, TopicFilter},
{DSSubId, NowMS}.
@ -653,6 +807,9 @@ new_subscription_id(DSSessionId, TopicFilter) ->
subscription_id_to_topic_filter({_DSSessionId, TopicFilter}) ->
TopicFilter.
now_ms() ->
erlang:system_time(millisecond).
%%--------------------------------------------------------------------
%% RPC targets (v1)
%%--------------------------------------------------------------------
@ -677,84 +834,100 @@ do_ensure_all_iterators_closed(_DSSessionID) ->
%% Reading batches
%%--------------------------------------------------------------------
-spec renew_streams(id()) -> ok.
renew_streams(DSSessionId) ->
Subscriptions = ro_transaction(fun() -> session_read_subscriptions(DSSessionId) end),
ExistingStreams = ro_transaction(fun() -> mnesia:read(?SESSION_STREAM_TAB, DSSessionId) end),
lists:foreach(
fun(#ds_sub{id = {_, TopicFilter}, start_time = StartTime}) ->
renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime)
-spec renew_streams(session()) -> ok.
renew_streams(#{id := SessionId, subscriptions := Subscriptions}) ->
transaction(fun() ->
ExistingStreams = mnesia:read(?SESSION_STREAM_TAB, SessionId, write),
maps:fold(
fun(TopicFilter, #{start_time := StartTime}, Streams) ->
TopicFilterWords = emqx_topic:words(TopicFilter),
renew_topic_streams(SessionId, TopicFilterWords, StartTime, Streams)
end,
ExistingStreams,
Subscriptions
)
end),
ok.
-spec renew_topic_streams(id(), topic_filter_words(), emqx_ds:time(), _Acc :: [ds_stream()]) -> ok.
renew_topic_streams(DSSessionId, TopicFilter, StartTime, ExistingStreams) ->
TopicStreams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
lists:foldl(
fun({Rank, Stream}, Streams) ->
case lists:keymember(Stream, #ds_stream.stream, Streams) of
true ->
Streams;
false ->
StreamRef = length(Streams) + 1,
DSStream = session_store_stream(
DSSessionId,
StreamRef,
Stream,
Rank,
TopicFilter,
StartTime
),
[DSStream | Streams]
end
end,
ExistingStreams,
TopicStreams
).
-spec renew_streams(id(), [ds_stream()], emqx_ds:topic_filter(), emqx_ds:time()) -> ok.
renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime) ->
AllStreams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
transaction(
fun() ->
lists:foreach(
fun({Rank, Stream}) ->
Rec = #ds_stream{
session = DSSessionId,
topic_filter = TopicFilter,
stream = Stream,
rank = Rank
},
case lists:member(Rec, ExistingStreams) of
true ->
ok;
false ->
mnesia:write(?SESSION_STREAM_TAB, Rec, write),
{ok, Iterator} = emqx_ds:make_iterator(
?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime
session_store_stream(DSSessionId, StreamRef, Stream, Rank, TopicFilter, StartTime) ->
{ok, ItBegin} = emqx_ds:make_iterator(
?PERSISTENT_MESSAGE_DB,
Stream,
TopicFilter,
StartTime
),
%% Workaround: we convert `Stream' to a binary before
%% attempting to store it in mnesia(rocksdb) because of a bug
%% in `mnesia_rocksdb' when trying to do
%% `mnesia:dirty_all_keys' later.
StreamBin = term_to_binary(Stream),
IterRec = #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator},
mnesia:write(?SESSION_ITER_TAB, IterRec, write)
end
end,
AllStreams
)
end
).
DSStream = #ds_stream{
session = DSSessionId,
ref = StreamRef,
stream = Stream,
rank = Rank,
beginning = ItBegin
},
mnesia:write(?SESSION_STREAM_TAB, DSStream, write),
DSStream.
%% must be called inside a transaction
-spec session_drop_streams(id()) -> ok.
session_drop_streams(DSSessionId) ->
MS = ets:fun2ms(
fun(#ds_stream{session = DSSessionId0}) when DSSessionId0 =:= DSSessionId ->
DSSessionId0
end
),
StreamIDs = mnesia:select(?SESSION_STREAM_TAB, MS, write),
lists:foreach(fun(Key) -> mnesia:delete(?SESSION_STREAM_TAB, Key, write) end, StreamIDs).
mnesia:delete(?SESSION_STREAM_TAB, DSSessionId, write).
%% must be called inside a transaction
-spec session_drop_iterators(id()) -> ok.
session_drop_iterators(DSSessionId) ->
MS = ets:fun2ms(
fun(#ds_iter{id = {DSSessionId0, StreamBin}}) when DSSessionId0 =:= DSSessionId ->
StreamBin
end
),
StreamBins = mnesia:select(?SESSION_ITER_TAB, MS, write),
-spec session_drop_pubranges(id()) -> ok.
session_drop_pubranges(DSSessionId) ->
RangeIds = session_read_pubranges(DSSessionId, write),
lists:foreach(
fun(StreamBin) ->
mnesia:delete(?SESSION_ITER_TAB, {DSSessionId, StreamBin}, write)
fun(RangeId) ->
mnesia:delete(?SESSION_PUBRANGE_TAB, RangeId, write)
end,
StreamBins
RangeIds
).
%% must be called inside a transaction
-spec session_drop_offsets(id()) -> ok.
session_drop_offsets(DSSessionId) ->
OffsetIds = session_read_offsets(DSSessionId, write),
lists:foreach(
fun(OffsetId) ->
mnesia:delete(?SESSION_COMMITTED_OFFSET_TAB, OffsetId, write)
end,
OffsetIds
).
%%--------------------------------------------------------------------------------
transaction(Fun) ->
case mnesia:is_transaction() of
true ->
Fun();
false ->
{atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
Res.
Res
end.
ro_transaction(Fun) ->
{atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
@ -772,7 +945,7 @@ export_subscriptions(DSSubs) ->
).
export_session(#session{} = Record) ->
export_record(Record, #session.id, [id, created_at, expires_at, inflight, props], #{}).
export_record(Record, #session.id, [id, created_at, last_alive_at, conninfo, props], #{}).
export_subscription(#ds_sub{} = Record) ->
export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}).
@ -786,13 +959,17 @@ export_record(_, _, [], Acc) ->
%% effects. Add `CBM:init' callback to the session behavior?
ensure_timers() ->
ensure_timer(pull),
ensure_timer(get_streams).
ensure_timer(get_streams),
ensure_timer(bump_last_alive_at).
-spec ensure_timer(pull | get_streams) -> ok.
-spec ensure_timer(timer()) -> ok.
ensure_timer(bump_last_alive_at = Type) ->
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
ensure_timer(Type, BumpInterval);
ensure_timer(Type) ->
ensure_timer(Type, 100).
-spec ensure_timer(pull | get_streams, non_neg_integer()) -> ok.
-spec ensure_timer(timer(), non_neg_integer()) -> ok.
ensure_timer(Type, Timeout) ->
_ = emqx_utils:start_timer(Timeout, {emqx_session, Type}),
ok.
@ -804,13 +981,23 @@ receive_maximum(ConnInfo) ->
%% indicates that it's optional.
maps:get(receive_maximum, ConnInfo, 65_535).
-spec expiry_interval(conninfo()) -> millisecond().
expiry_interval(ConnInfo) ->
maps:get(expiry_interval, ConnInfo, 0).
-ifdef(TEST).
list_all_sessions() ->
DSSessionIds = mnesia:dirty_all_keys(?SESSION_TAB),
Sessions = lists:map(
ConnInfo = #{},
Sessions = lists:filtermap(
fun(SessionID) ->
{ok, Session, Subscriptions} = session_open(SessionID),
{SessionID, #{session => Session, subscriptions => Subscriptions}}
Sess = session_open(SessionID, ConnInfo),
case Sess of
false ->
false;
_ ->
{true, {SessionID, Sess}}
end
end,
DSSessionIds
),
@ -850,16 +1037,18 @@ list_all_streams() ->
),
maps:from_list(DSStreams).
list_all_iterators() ->
DSIterIds = mnesia:dirty_all_keys(?SESSION_ITER_TAB),
DSIters = lists:map(
fun(DSIterId) ->
[Record] = mnesia:dirty_read(?SESSION_ITER_TAB, DSIterId),
{DSIterId, export_record(Record, #ds_iter.id, [id, iter], #{})}
end,
DSIterIds
list_all_pubranges() ->
DSPubranges = mnesia:dirty_match_object(?SESSION_PUBRANGE_TAB, #ds_pubrange{_ = '_'}),
lists:foldl(
fun(Record = #ds_pubrange{id = {SessionId, First}}, Acc) ->
Range = export_record(
Record, #ds_pubrange.until, [until, stream, type, iterator], #{first => First}
),
maps:from_list(DSIters).
maps:put(SessionId, maps:get(SessionId, Acc, []) ++ [Range], Acc)
end,
#{},
DSPubranges
).
%% ifdef(TEST)
-endif.

59
apps/emqx/src/emqx_persistent_session_ds.hrl
View File

@ -21,9 +21,13 @@
-define(SESSION_TAB, emqx_ds_session).
-define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
-define(SESSION_STREAM_TAB, emqx_ds_stream_tab).
-define(SESSION_ITER_TAB, emqx_ds_iter_tab).
-define(SESSION_PUBRANGE_TAB, emqx_ds_pubrange_tab).
-define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab).
-define(DS_MRIA_SHARD, emqx_ds_session_shard).
-define(T_INFLIGHT, 1).
-define(T_CHECKPOINT, 2).
-record(ds_sub, {
id :: emqx_persistent_session_ds:subscription_id(),
start_time :: emqx_ds:time(),
@ -34,16 +38,53 @@
-record(ds_stream, {
session :: emqx_persistent_session_ds:id(),
topic_filter :: emqx_ds:topic_filter(),
ref :: _StreamRef,
stream :: emqx_ds:stream(),
rank :: emqx_ds:stream_rank()
rank :: emqx_ds:stream_rank(),
beginning :: emqx_ds:iterator()
}).
-type ds_stream() :: #ds_stream{}.
-type ds_stream_bin() :: binary().
-record(ds_iter, {
id :: {emqx_persistent_session_ds:id(), ds_stream_bin()},
iter :: emqx_ds:iterator()
-record(ds_pubrange, {
id :: {
%% What session this range belongs to.
_Session :: emqx_persistent_session_ds:id(),
%% Where this range starts.
_First :: emqx_persistent_message_ds_replayer:seqno()
},
%% Where this range ends: the first seqno that is not included in the range.
until :: emqx_persistent_message_ds_replayer:seqno(),
%% Which stream this range is over.
stream :: _StreamRef,
%% Type of a range:
%% * Inflight range is a range of yet unacked messages from this stream.
%% * Checkpoint range was already acked, its purpose is to keep track of the
%% very last iterator for this stream.
type :: ?T_INFLIGHT | ?T_CHECKPOINT,
%% What commit tracks this range is part of.
%% This is rarely stored: we only need to persist it when the range contains
%% QoS 2 messages.
tracks = 0 :: non_neg_integer(),
%% Meaning of this depends on the type of the range:
%% * For inflight range, this is the iterator pointing to the first message in
%% the range.
%% * For checkpoint range, this is the iterator pointing right past the last
%% message in the range.
iterator :: emqx_ds:iterator(),
%% Reserved for future use.
misc = #{} :: map()
}).
-type ds_pubrange() :: #ds_pubrange{}.
-record(ds_committed_offset, {
id :: {
%% What session this marker belongs to.
_Session :: emqx_persistent_session_ds:id(),
%% Marker name.
_CommitType
},
%% Where this marker is pointing to: the first seqno that is not marked.
until :: emqx_persistent_message_ds_replayer:seqno()
}).
-record(session, {
@ -51,8 +92,8 @@
id :: emqx_persistent_session_ds:id(),
%% creation time
created_at :: _Millisecond :: non_neg_integer(),
expires_at = never :: _Millisecond :: non_neg_integer() | never,
inflight :: emqx_persistent_message_ds_replayer:inflight(),
last_alive_at :: _Millisecond :: non_neg_integer(),
conninfo :: emqx_types:conninfo(),
%% for future usage
props = #{} :: map()
}).

161
apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl Normal file
View File

@ -0,0 +1,161 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_gc_worker).
-behaviour(gen_server).
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("stdlib/include/qlc.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include("emqx_persistent_session_ds.hrl").
%% API
-export([
start_link/0
]).
%% `gen_server' API
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2
]).
%% call/cast/info records
-record(gc, {}).
%%--------------------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------------------
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
%%--------------------------------------------------------------------------------
%% `gen_server' API
%%--------------------------------------------------------------------------------
init(_Opts) ->
ensure_gc_timer(),
State = #{},
{ok, State}.
handle_call(_Call, _From, State) ->
{reply, error, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
handle_info(#gc{}, State) ->
try_gc(),
ensure_gc_timer(),
{noreply, State};
handle_info(_Info, State) ->
{noreply, State}.
%%--------------------------------------------------------------------------------
%% Internal fns
%%--------------------------------------------------------------------------------
ensure_gc_timer() ->
Timeout = emqx_config:get([session_persistence, session_gc_interval]),
_ = erlang:send_after(Timeout, self(), #gc{}),
ok.
try_gc() ->
%% Only cores should run GC.
CoreNodes = mria_membership:running_core_nodelist(),
Res = global:trans(
{?MODULE, self()},
fun() -> ?tp_span(ds_session_gc, #{}, start_gc()) end,
CoreNodes,
%% Note: we set retries to 1 here because, in rare occasions, GC might start at the
%% same time in more than one node, and each one will abort the other. By allowing
%% one retry, at least one node will (hopefully) get to enter the transaction and
%% the other will abort. If GC runs too fast, both nodes might run in sequence.
%% But, in that case, GC is clearly not too costly, and that shouldn't be a problem,
%% resource-wise.
_Retries = 1
),
case Res of
aborted ->
?tp(ds_session_gc_lock_taken, #{}),
ok;
ok ->
ok
end.
now_ms() ->
erlang:system_time(millisecond).
start_gc() ->
do_gc(more).
zombie_session_ms() ->
NowMS = now_ms(),
GCInterval = emqx_config:get([session_persistence, session_gc_interval]),
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
TimeThreshold = max(GCInterval, BumpInterval) * 3,
ets:fun2ms(
fun(
#session{
id = DSSessionId,
last_alive_at = LastAliveAt,
conninfo = #{expiry_interval := EI}
}
) when
LastAliveAt + EI + TimeThreshold =< NowMS
->
DSSessionId
end
).
do_gc(more) ->
GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]),
MS = zombie_session_ms(),
{atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() ->
Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write),
case Res of
'$end_of_table' ->
done;
{[], Cont} ->
%% since `GCBatchsize' is just a "recommendation" for `select', we try only
%% _once_ the continuation and then stop if it yields nothing, to avoid a
%% dead loop.
case mnesia:select(Cont) of
'$end_of_table' ->
done;
{[], _Cont} ->
done;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end
end),
do_gc(Next);
do_gc(done) ->
ok.
do_gc_(DSSessionIds) ->
lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds),
?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}),
ok.

78
apps/emqx/src/emqx_persistent_session_ds_sup.erl Normal file
View File

@ -0,0 +1,78 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_sup).
-behaviour(supervisor).
%% API
-export([
start_link/0
]).
%% `supervisor' API
-export([
init/1
]).
%%--------------------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------------------
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%%--------------------------------------------------------------------------------
%% `supervisor' API
%%--------------------------------------------------------------------------------
init(Opts) ->
case emqx_persistent_message:is_persistence_enabled() of
true ->
do_init(Opts);
false ->
ignore
end.
do_init(_Opts) ->
SupFlags = #{
strategy => rest_for_one,
intensity => 10,
period => 2,
auto_shutdown => never
},
CoreChildren = [
worker(gc_worker, emqx_persistent_session_ds_gc_worker, [])
],
Children =
case mria_rlog:role() of
core -> CoreChildren;
replicant -> []
end,
{ok, {SupFlags, Children}}.
%%--------------------------------------------------------------------------------
%% Internal fns
%%--------------------------------------------------------------------------------
worker(Id, Mod, Args) ->
#{
id => Id,
start => {Mod, start_link, Args},
type => worker,
restart => permanent,
shutdown => 10_000,
significant => false
}.

2
apps/emqx/src/emqx_quic_stream.erl
View File

@ -184,7 +184,7 @@ peer_send_aborted(Stream, ErrorCode, S) ->
-spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret().
peer_send_shutdown(Stream, undefined, S) ->
ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
_ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
{ok, S}.
-spec send_complete(stream_handle(), boolean(), cb_data()) -> cb_ret().

1
apps/emqx/src/emqx_reason_codes.erl
View File

@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER;
compat(connack, 16#9F) -> ?CONNACK_SERVER;
compat(suback, Code) when Code =< ?QOS_2 -> Code;
compat(suback, Code) when Code >= 16#80 -> 16#80;
%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1
compat(unsuback, _Code) -> undefined;
compat(_Other, _Code) -> undefined.

82
apps/emqx/src/emqx_router.erl
View File

@ -52,6 +52,9 @@
lookup_routes/1
]).
%% Topics API
-export([select/3]).
-export([print_routes/1]).
-export([
@ -59,7 +62,10 @@
foldr_routes/2
]).
-export([topics/0]).
-export([
topics/0,
stats/1
]).
%% Exported for tests
-export([has_route/2]).
@ -219,6 +225,19 @@ mria_delete_route(v2, Topic, Dest) ->
mria_delete_route(v1, Topic, Dest) ->
mria_delete_route_v1(Topic, Dest).
-spec select(Spec, _Limit :: pos_integer(), Continuation) ->
{[emqx_types:route()], Continuation} | '$end_of_table'
when
Spec :: {_TopicPat, _DestPat},
Continuation :: term() | '$end_of_table'.
select(MatchSpec, Limit, Cont) ->
select(get_schema_vsn(), MatchSpec, Limit, Cont).
select(v2, MatchSpec, Limit, Cont) ->
select_v2(MatchSpec, Limit, Cont);
select(v1, MatchSpec, Limit, Cont) ->
select_v1(MatchSpec, Limit, Cont).
-spec topics() -> list(emqx_types:topic()).
topics() ->
topics(get_schema_vsn()).
@ -228,6 +247,15 @@ topics(v2) ->
topics(v1) ->
list_topics_v1().
-spec stats(n_routes) -> non_neg_integer().
stats(Item) ->
stats(get_schema_vsn(), Item).
stats(v2, Item) ->
get_stats_v2(Item);
stats(v1, Item) ->
get_stats_v1(Item).
%% @doc Print routes to a topic
-spec print_routes(emqx_types:topic()) -> ok.
print_routes(Topic) ->
@ -345,9 +373,17 @@ cleanup_routes_v1(Node) ->
]
end).
select_v1({MTopic, MDest}, Limit, undefined) ->
ets:match_object(?ROUTE_TAB, #route{topic = MTopic, dest = MDest}, Limit);
select_v1(_Spec, _Limit, Cont) ->
ets:select(Cont).
list_topics_v1() ->
list_route_tab_topics().
get_stats_v1(n_routes) ->
emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0).
list_route_tab_topics() ->
mnesia:dirty_all_keys(?ROUTE_TAB).
@ -436,11 +472,52 @@ get_dest_node({_, Node}) ->
get_dest_node(Node) ->
Node.
select_v2(Spec, Limit, undefined) ->
Stream = mk_route_stream(Spec),
select_next(Limit, Stream);
select_v2(_Spec, Limit, Stream) ->
select_next(Limit, Stream).
select_next(N, Stream) ->
case emqx_utils_stream:consume(N, Stream) of
{Routes, SRest} ->
{Routes, SRest};
Routes ->
{Routes, '$end_of_table'}
end.
mk_route_stream(Spec) ->
emqx_utils_stream:chain(
mk_route_stream(route, Spec),
mk_route_stream(filter, Spec)
).
mk_route_stream(route, Spec) ->
emqx_utils_stream:ets(fun(Cont) -> select_v1(Spec, 1, Cont) end);
mk_route_stream(filter, {MTopic, MDest}) ->
emqx_utils_stream:map(
fun routeidx_to_route/1,
emqx_utils_stream:ets(
fun
(undefined) ->
MatchSpec = #routeidx{entry = emqx_trie_search:make_pat(MTopic, MDest)},
ets:match_object(?ROUTE_TAB_FILTERS, MatchSpec, 1);
(Cont) ->
ets:match_object(Cont)
end
)
).
list_topics_v2() ->
Pat = #routeidx{entry = '$1'},
Filters = [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_FILTERS, Pat)],
list_route_tab_topics() ++ Filters.
get_stats_v2(n_routes) ->
NTopics = emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0),
NWildcards = emqx_maybe:define(ets:info(?ROUTE_TAB_FILTERS, size), 0),
NTopics + NWildcards.
fold_routes_v2(FunName, FoldFun, AccIn) ->
FilterFoldFun = mk_filtertab_fold_fun(FoldFun),
Acc = ets:FunName(FoldFun, AccIn, ?ROUTE_TAB),
@ -449,6 +526,9 @@ fold_routes_v2(FunName, FoldFun, AccIn) ->
mk_filtertab_fold_fun(FoldFun) ->
fun(#routeidx{entry = K}, Acc) -> FoldFun(match_to_route(K), Acc) end.
routeidx_to_route(#routeidx{entry = M}) ->
match_to_route(M).
match_to_route(M) ->
#route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}.

7
apps/emqx/src/emqx_router_helper.erl
View File

@ -190,12 +190,7 @@ code_change(_OldVsn, State, _Extra) ->
%%--------------------------------------------------------------------
stats_fun() ->
case ets:info(?ROUTE_TAB, size) of
undefined ->
ok;
Size ->
emqx_stats:setstat('topics.count', 'topics.max', Size)
end.
emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)).
cleanup_routes(Node) ->
emqx_router:cleanup_routes(Node).

143
apps/emqx/src/emqx_schema.erl
View File

@ -294,7 +294,19 @@ roots(low) ->
{"persistent_session_store",
sc(
ref("persistent_session_store"),
#{importance => ?IMPORTANCE_HIDDEN}
#{
%% NOTE
%% Due to some quirks in interaction between `emqx_config` and
%% `hocon_tconf`, schema roots cannot currently be deprecated.
importance => ?IMPORTANCE_HIDDEN
}
)},
{"session_persistence",
sc(
ref("session_persistence"),
#{
importance => ?IMPORTANCE_HIDDEN
}
)},
{"trace",
sc(
@ -309,11 +321,12 @@ roots(low) ->
].
fields("persistent_session_store") ->
Deprecated = #{deprecated => {since, "5.4.0"}},
[
{"enabled",
sc(
boolean(),
#{
Deprecated#{
default => false,
%% TODO(5.2): change field name to 'enable' and keep 'enabled' as an alias
aliases => [enable],
@ -323,7 +336,7 @@ fields("persistent_session_store") ->
{"ds",
sc(
boolean(),
#{
Deprecated#{
default => false,
importance => ?IMPORTANCE_HIDDEN
}
@ -331,7 +344,7 @@ fields("persistent_session_store") ->
{"on_disc",
sc(
boolean(),
#{
Deprecated#{
default => true,
desc => ?DESC(persistent_store_on_disc)
}
@ -339,7 +352,7 @@ fields("persistent_session_store") ->
{"ram_cache",
sc(
boolean(),
#{
Deprecated#{
default => false,
desc => ?DESC(persistent_store_ram_cache)
}
@ -347,7 +360,7 @@ fields("persistent_session_store") ->
{"backend",
sc(
hoconsc:union([ref("persistent_session_builtin")]),
#{
Deprecated#{
default => #{
<<"type">> => <<"builtin">>,
<<"session">> =>
@ -363,7 +376,7 @@ fields("persistent_session_store") ->
{"max_retain_undelivered",
sc(
duration(),
#{
Deprecated#{
default => <<"1h">>,
desc => ?DESC(persistent_session_store_max_retain_undelivered)
}
@ -371,7 +384,7 @@ fields("persistent_session_store") ->
{"message_gc_interval",
sc(
duration(),
#{
Deprecated#{
default => <<"1h">>,
desc => ?DESC(persistent_session_store_message_gc_interval)
}
@ -379,7 +392,7 @@ fields("persistent_session_store") ->
{"session_message_gc_interval",
sc(
duration(),
#{
Deprecated#{
default => <<"1m">>,
desc => ?DESC(persistent_session_store_session_message_gc_interval)
}
@ -1382,7 +1395,7 @@ fields("broker_routing") ->
sc(
hoconsc:enum([v1, v2]),
#{
default => v1,
default => v2,
'readOnly' => true,
desc => ?DESC(broker_routing_storage_schema)
}
@ -1740,6 +1753,103 @@ fields("trace") ->
importance => ?IMPORTANCE_HIDDEN,
desc => ?DESC(fields_trace_payload_encode)
})}
];
fields("session_persistence") ->
[
{"enable",
sc(
boolean(), #{
desc => ?DESC(session_persistence_enable),
default => false
}
)},
{"storage",
sc(
ref("session_storage_backend"), #{
desc => ?DESC(session_persistence_storage),
validator => fun validate_backend_enabled/1,
default => #{
<<"builtin">> => #{}
}
}
)},
{"idle_poll_interval",
sc(
timeout_duration(),
#{
default => <<"100ms">>,
desc => ?DESC(session_ds_idle_poll_interval)
}
)},
{"last_alive_update_interval",
sc(
timeout_duration(),
#{
default => <<"5000ms">>,
desc => ?DESC(session_ds_last_alive_update_interval)
}
)},
{"session_gc_interval",
sc(
timeout_duration(),
#{
default => <<"10m">>,
desc => ?DESC(session_ds_session_gc_interval)
}
)},
{"session_gc_batch_size",
sc(
pos_integer(),
#{
default => 100,
importance => ?IMPORTANCE_LOW,
desc => ?DESC(session_ds_session_gc_batch_size)
}
)},
{"force_persistence",
sc(
boolean(),
#{
default => false,
%% Only for testing, shall remain hidden
importance => ?IMPORTANCE_HIDDEN
}
)}
];
fields("session_storage_backend") ->
[
{"builtin",
sc(ref("session_storage_backend_builtin"), #{
desc => ?DESC(session_storage_backend_builtin),
required => {false, recursively}
})}
] ++ emqx_schema_hooks:injection_point('session_persistence.storage_backends', []);
fields("session_storage_backend_builtin") ->
[
{"enable",
sc(
boolean(),
#{
desc => ?DESC(session_storage_backend_enable),
default => true
}
)},
{"n_shards",
sc(
pos_integer(),
#{
desc => ?DESC(session_builtin_n_shards),
default => 16
}
)},
{"replication_factor",
sc(
pos_integer(),
#{
default => 3,
importance => ?IMPORTANCE_HIDDEN
}
)}
].
mqtt_listener(Bind) ->
@ -1992,6 +2102,8 @@ desc("ocsp") ->
"Per listener OCSP Stapling configuration.";
desc("crl_cache") ->
"Global CRL cache options.";
desc("session_persistence") ->
"Settings governing durable sessions persistence.";
desc(_) ->
undefined.
@ -2014,6 +2126,17 @@ ensure_list(V) ->
filter(Opts) ->
[{K, V} || {K, V} <- Opts, V =/= undefined].
validate_backend_enabled(Config) ->
Enabled = maps:filter(fun(_, #{<<"enable">> := E}) -> E end, Config),
case maps:to_list(Enabled) of
[{_Type, _BackendConfig}] ->
ok;
_Conflicts = [_ | _] ->
{error, multiple_enabled_backends};
_None = [] ->
{error, no_enabled_backend}
end.
%% @private This function defines the SSL opts which are commonly used by
%% SSL listener and client.
-spec common_ssl_opts_schema(map(), server | client) -> hocon_schema:field_schema().

85
apps/emqx/src/emqx_schema_secret.erl Normal file
View File

@ -0,0 +1,85 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc HOCON schema that defines _secret_ concept.
-module(emqx_schema_secret).
-include_lib("emqx/include/logger.hrl").
-include_lib("typerefl/include/types.hrl").
-export([mk/1]).
%% HOCON Schema API
-export([convert_secret/2]).
%% @doc Secret value.
-type t() :: binary().
%% @doc Source of the secret value.
%% * "file://...": file path to a file containing secret value.
%% * other binaries: secret value itself.
-type source() :: iodata().
-type secret() :: binary() | function().
-reflect_type([secret/0]).
-define(SCHEMA, #{
required => false,
format => <<"password">>,
sensitive => true,
converter => fun ?MODULE:convert_secret/2
}).
-dialyzer({nowarn_function, source/1}).
%%
-spec mk(#{atom() => _}) -> hocon_schema:field_schema().
mk(Overrides = #{}) ->
hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)).
convert_secret(undefined, #{}) ->
undefined;
convert_secret(Secret, #{make_serializable := true}) ->
unicode:characters_to_binary(source(Secret));
convert_secret(Secret, #{}) when is_function(Secret, 0) ->
Secret;
convert_secret(Secret, #{}) when is_integer(Secret) ->
wrap(integer_to_binary(Secret));
convert_secret(Secret, #{}) ->
try unicode:characters_to_binary(Secret) of
String when is_binary(String) ->
wrap(String);
{error, _, _} ->
throw(invalid_string)
catch
error:_ ->
throw(invalid_type)
end.
-spec wrap(source()) -> emqx_secret:t(t()).
wrap(<<"file://", Filename/binary>>) ->
emqx_secret:wrap_load({file, Filename});
wrap(Secret) ->
emqx_secret:wrap(Secret).
-spec source(emqx_secret:t(t())) -> source().
source(Secret) when is_function(Secret) ->
source(emqx_secret:term(Secret));
source({file, Filename}) ->
<<"file://", Filename/binary>>;
source(Secret) ->
Secret.

31
apps/emqx/src/emqx_secret.erl
View File

@ -19,23 +19,52 @@
-module(emqx_secret).
%% API:
-export([wrap/1, unwrap/1]).
-export([wrap/1, wrap_load/1, unwrap/1, term/1]).
-export_type([t/1]).
-opaque t(T) :: T | fun(() -> t(T)).
%% Secret loader module.
%% Any changes related to processing of secrets should be made there.
-define(LOADER, emqx_secret_loader).
%%================================================================================
%% API funcions
%%================================================================================
%% @doc Wrap a term in a secret closure.
%% This effectively hides the term from any term formatting / printing code.
-spec wrap(T) -> t(T).
wrap(Term) ->
fun() ->
Term
end.
%% @doc Wrap a loader function call over a term in a secret closure.
%% This is slightly more flexible form of `wrap/1` with the same basic purpose.
-spec wrap_load(emqx_secret_loader:source()) -> t(_).
wrap_load(Source) ->
fun() ->
apply(?LOADER, load, [Source])
end.
%% @doc Unwrap a secret closure, revealing the secret.
%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped.
-spec unwrap(t(T)) -> T.
unwrap(Term) when is_function(Term, 0) ->
%% Handle potentially nested funs
unwrap(Term());
unwrap(Term) ->
Term.
%% @doc Inspect the term wrapped in a secret closure.
-spec term(t(_)) -> _Term.
term(Wrap) when is_function(Wrap, 0) ->
case erlang:fun_info(Wrap, module) of
{module, ?MODULE} ->
{env, Env} = erlang:fun_info(Wrap, env),
lists:last(Env);
_ ->
error(badarg, [Wrap])
end.

42
apps/emqx/src/emqx_secret_loader.erl Normal file
View File

@ -0,0 +1,42 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_secret_loader).
%% API
-export([load/1]).
-export([file/1]).
-export_type([source/0]).
-type source() :: {file, file:filename_all()}.
-spec load(source()) -> binary() | no_return().
load({file, Filename}) ->
file(Filename).
-spec file(file:filename_all()) -> binary() | no_return().
file(Filename) ->
case file:read_file(Filename) of
{ok, Secret} ->
string:trim(Secret, trailing);
{error, Reason} ->
throw(#{
msg => failed_to_read_secret_file,
path => Filename,
reason => emqx_utils:explain_posix(Reason)
})
end.

38
apps/emqx/src/emqx_session.erl
View File

@ -84,7 +84,7 @@
-export([
deliver/3,
handle_timeout/3,
disconnect/2,
disconnect/3,
terminate/3
]).
@ -267,7 +267,7 @@ destroy(Session) ->
-spec subscribe(
clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts(),
t()
) ->
@ -287,7 +287,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) ->
-spec unsubscribe(
clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts(),
t()
) ->
@ -418,7 +418,13 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
end.
enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
SubOpts = ?IMPL(Session):get_subscription(Topic, Session),
SubOpts =
case Msg of
#message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} ->
?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session);
_ ->
?IMPL(Session):get_subscription(Topic, Session)
end,
enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS).
enrich_message(
@ -497,10 +503,10 @@ cancel_timer(Name, Timers) ->
%%--------------------------------------------------------------------
-spec disconnect(clientinfo(), t()) ->
-spec disconnect(clientinfo(), eqmx_types:conninfo(), t()) ->
{idle | shutdown, t()}.
disconnect(_ClientInfo, Session) ->
?IMPL(Session):disconnect(Session).
disconnect(_ClientInfo, ConnInfo, Session) ->
?IMPL(Session):disconnect(Session, ConnInfo).
-spec terminate(clientinfo(), Reason :: term(), t()) ->
ok.
@ -611,21 +617,27 @@ maybe_mock_impl_mod(_) ->
-spec choose_impl_mod(conninfo()) -> module().
choose_impl_mod(#{expiry_interval := EI}) ->
hd(choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled())).
hd(choose_impl_candidates(EI, emqx_persistent_message:is_persistence_enabled())).
-spec choose_impl_candidates(conninfo()) -> [module()].
choose_impl_candidates(#{expiry_interval := EI}) ->
choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled()).
choose_impl_candidates(EI, emqx_persistent_message:is_persistence_enabled()).
choose_impl_candidates(_, _IsPSStoreEnabled = false) ->
[emqx_session_mem];
choose_impl_candidates(0, _IsPSStoreEnabled = true) ->
case emqx_persistent_message:force_ds() of
false ->
%% NOTE
%% If ExpiryInterval is 0, the natural choice is `emqx_session_mem`. Yet we still
%% need to look the existing session up in the `emqx_persistent_session_ds` store
%% first, because previous connection may have set ExpiryInterval to a non-zero
%% value.
%% If ExpiryInterval is 0, the natural choice is
%% `emqx_session_mem'. Yet we still need to look the
%% existing session up in the `emqx_persistent_session_ds'
%% store first, because previous connection may have set
%% ExpiryInterval to a non-zero value.
[emqx_session_mem, emqx_persistent_session_ds];
true ->
[emqx_persistent_session_ds]
end;
choose_impl_candidates(EI, _IsPSStoreEnabled = true) when EI > 0 ->
[emqx_persistent_session_ds].

8
apps/emqx/src/emqx_session_mem.erl
View File

@ -87,7 +87,7 @@
deliver/3,
replay/3,
handle_timeout/3,
disconnect/1,
disconnect/2,
terminate/2
]).
@ -316,7 +316,7 @@ unsubscribe(
{error, ?RC_NO_SUBSCRIPTION_EXISTED}
end.
-spec get_subscription(emqx_types:topic(), session()) ->
-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) ->
emqx_types:subopts() | undefined.
get_subscription(Topic, #session{subscriptions = Subs}) ->
maps:get(Topic, Subs, undefined).
@ -725,8 +725,8 @@ append(L1, L2) -> L1 ++ L2.
%%--------------------------------------------------------------------
-spec disconnect(session()) -> {idle, session()}.
disconnect(Session = #session{}) ->
-spec disconnect(session(), emqx_types:conninfo()) -> {idle, session()}.
disconnect(Session = #session{}, _ConnInfo) ->
% TODO: isolate expiry timer / timeout handling here?
{idle, Session}.

18
apps/emqx/src/emqx_shared_sub.erl
View File

@ -95,7 +95,6 @@
-define(ACK, shared_sub_ack).
-define(NACK(Reason), {shared_sub_nack, Reason}).
-define(NO_ACK, no_ack).
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-define(SUBSCRIBER_DOWN, noproc).
-type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
@ -234,19 +233,18 @@ without_group_ack(Msg) ->
get_group_ack(Msg) ->
emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
Msg;
%% always add `redispatch_to` header to the message
%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1
with_redispatch_to(Msg, Group, Topic) ->
emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
%% @hidden Redispatch is needed only for the messages with redispatch_to header added.
is_redispatch_needed(#message{} = Msg) ->
case get_redispatch_to(Msg) of
?REDISPATCH_TO(_, _) ->
%% @hidden Redispatch is needed only for the messages which not QOS_0
is_redispatch_needed(#message{qos = ?QOS_0}) ->
false;
is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) ->
true;
_ ->
false
end.
is_redispatch_needed(#message{}) ->
false.
%% @doc Redispatch shared deliveries to other members in the group.
redispatch(Messages0) ->

94
apps/emqx/src/emqx_topic.erl
View File

@ -36,9 +36,16 @@
parse/2
]).
-export([
maybe_format_share/1,
get_shared_real_topic/1,
make_shared_record/2
]).
-type topic() :: emqx_types:topic().
-type word() :: emqx_types:word().
-type words() :: emqx_types:words().
-type share() :: emqx_types:share().
%% Guards
-define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
@ -50,7 +57,9 @@
%%--------------------------------------------------------------------
%% @doc Is wildcard topic?
-spec wildcard(topic() | words()) -> true | false.
-spec wildcard(topic() | share() | words()) -> true | false.
wildcard(#share{topic = Topic}) when is_binary(Topic) ->
wildcard(Topic);
wildcard(Topic) when is_binary(Topic) ->
wildcard(words(Topic));
wildcard([]) ->
@ -64,7 +73,7 @@ wildcard([_H | T]) ->
%% @doc Match Topic name with filter.
-spec match(Name, Filter) -> boolean() when
Name :: topic() | words(),
Name :: topic() | share() | words(),
Filter :: topic() | words().
match(<<$$, _/binary>>, <<$+, _/binary>>) ->
false;
@ -72,6 +81,10 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) ->
false;
match(Name, Filter) when is_binary(Name), is_binary(Filter) ->
match(words(Name), words(Filter));
match(#share{} = Name, Filter) ->
match_share(Name, Filter);
match(Name, #share{} = Filter) ->
match_share(Name, Filter);
match([], []) ->
true;
match([H | T1], [H | T2]) ->
@ -87,12 +100,29 @@ match([_H1 | _], []) ->
match([], [_H | _T2]) ->
false.
-spec match_share(Name, Filter) -> boolean() when
Name :: share(),
Filter :: topic() | share().
match_share(#share{topic = Name}, Filter) when is_binary(Filter) ->
%% only match real topic filter for normal topic filter.
match(words(Name), words(Filter));
match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) ->
%% Matching real topic filter When subed same share group.
match(words(Name), words(Filter));
match_share(#share{}, _) ->
%% Otherwise, non-matched.
false;
match_share(Name, #share{topic = Filter}) when is_binary(Name) ->
%% Only match real topic filter for normal topic_filter/topic_name.
match(Name, Filter).
-spec match_any(Name, [Filter]) -> boolean() when
Name :: topic() | words(),
Filter :: topic() | words().
match_any(Topic, Filters) ->
lists:any(fun(Filter) -> match(Topic, Filter) end, Filters).
%% TODO: validate share topic #share{} for emqx_trace.erl
%% @doc Validate topic name or filter
-spec validate(topic() | {name | filter, topic()}) -> true.
validate(Topic) when is_binary(Topic) ->
@ -107,7 +137,7 @@ validate(_, <<>>) ->
validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
%% MQTT-5.0 [MQTT-4.7.3-3]
error(topic_too_long);
validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
validate(filter, SharedFilter = <<?SHARE, "/", _Rest/binary>>) ->
validate_share(SharedFilter);
validate(filter, Filter) when is_binary(Filter) ->
validate2(words(Filter));
@ -139,12 +169,12 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
validate3(<<_/utf8, Rest/binary>>) ->
validate3(Rest).
validate_share(<<"$share/", Rest/binary>>) when
validate_share(<<?SHARE, "/", Rest/binary>>) when
Rest =:= <<>> orelse Rest =:= <<"/">>
->
%% MQTT-5.0 [MQTT-4.8.2-1]
error(?SHARE_EMPTY_FILTER);
validate_share(<<"$share/", Rest/binary>>) ->
validate_share(<<?SHARE, "/", Rest/binary>>) ->
case binary:split(Rest, <<"/">>) of
%% MQTT-5.0 [MQTT-4.8.2-1]
[<<>>, _] ->
@ -156,7 +186,7 @@ validate_share(<<"$share/", Rest/binary>>) ->
validate_share(ShareName, Filter)
end.
validate_share(_, <<"$share/", _Rest/binary>>) ->
validate_share(_, <<?SHARE, "/", _Rest/binary>>) ->
error(?SHARE_RECURSIVELY);
validate_share(ShareName, Filter) ->
case binary:match(ShareName, [<<"+">>, <<"#">>]) of
@ -185,7 +215,9 @@ bin('#') -> <<"#">>;
bin(B) when is_binary(B) -> B;
bin(L) when is_list(L) -> list_to_binary(L).
-spec levels(topic()) -> pos_integer().
-spec levels(topic() | share()) -> pos_integer().
levels(#share{topic = Topic}) when is_binary(Topic) ->
levels(Topic);
levels(Topic) when is_binary(Topic) ->
length(tokens(Topic)).
@ -197,6 +229,8 @@ tokens(Topic) ->
%% @doc Split Topic Path to Words
-spec words(topic()) -> words().
words(#share{topic = Topic}) when is_binary(Topic) ->
words(Topic);
words(Topic) when is_binary(Topic) ->
[word(W) || W <- tokens(Topic)].
@ -237,26 +271,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
do_join(TopicAcc, [Word | Words]) ->
do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
-spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}.
parse(TopicFilter) when is_binary(TopicFilter) ->
parse(TopicFilter, #{});
parse({TopicFilter, Options}) when is_binary(TopicFilter) ->
parse(TopicFilter, Options).
-spec parse(topic(), map()) -> {topic(), map()}.
parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(<<"$queue/", TopicFilter/binary>>, Options) ->
parse(TopicFilter, Options#{share => <<"$queue">>});
parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
-spec parse(topic() | share(), map()) -> {topic() | share(), map()}.
%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">>
%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>`
parse(#share{topic = Topic = <<?QUEUE, "/", _/binary>>}, _Options) ->
error({invalid_topic_filter, Topic});
parse(#share{topic = Topic = <<?SHARE, "/", _/binary>>}, _Options) ->
error({invalid_topic_filter, Topic});
parse(<<?QUEUE, "/", Topic/binary>>, Options) ->
parse(#share{group = <<?QUEUE>>, topic = Topic}, Options);
parse(TopicFilter = <<?SHARE, "/", Rest/binary>>, Options) ->
case binary:split(Rest, <<"/">>) of
[_Any] ->
error({invalid_topic_filter, TopicFilter});
[ShareName, Filter] ->
case binary:match(ShareName, [<<"+">>, <<"#">>]) of
nomatch -> parse(Filter, Options#{share => ShareName});
%% `Group` could be `$share` or `$queue`
[Group, Topic] ->
case binary:match(Group, [<<"+">>, <<"#">>]) of
nomatch -> parse(#share{group = Group, topic = Topic}, Options);
_ -> error({invalid_topic_filter, TopicFilter})
end
end;
@ -267,5 +304,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) ->
_ ->
{Topic, Options#{is_exclusive => true}}
end;
parse(TopicFilter, Options) ->
parse(TopicFilter, Options) when
?IS_TOPIC(TopicFilter)
->
{TopicFilter, Options}.
get_shared_real_topic(#share{topic = TopicFilter}) ->
TopicFilter;
get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) ->
TopicFilter.
make_shared_record(Group, Topic) ->
#share{group = Group, topic = Topic}.
maybe_format_share(#share{group = <<?QUEUE>>, topic = Topic}) ->
join([<<?QUEUE>>, Topic]);
maybe_format_share(#share{group = Group, topic = Topic}) ->
join([<<?SHARE>>, Group, Topic]);
maybe_format_share(Topic) ->
join([Topic]).

2
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
ignore ->
ignore;
Log ->
case logger_config:get(ets:whereis(logger), Id) of
case logger_config:get(logger, Id) of
{ok, #{module := Module} = HandlerConfig0} ->
HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
try

8
apps/emqx/src/emqx_trie_search.erl
View File

@ -98,7 +98,7 @@
-module(emqx_trie_search).
-export([make_key/2, filter/1]).
-export([make_key/2, make_pat/2, filter/1]).
-export([match/2, matches/3, get_id/1, get_topic/1]).
-export_type([key/1, word/0, words/0, nextf/0, opts/0]).
@ -127,6 +127,12 @@ make_key(Topic, ID) when is_binary(Topic) ->
make_key(Words, ID) when is_list(Words) ->
{Words, {ID}}.
-spec make_pat(emqx_types:topic() | words() | '_', _ID | '_') -> _Pat.
make_pat(Pattern = '_', ID) ->
{Pattern, {ID}};
make_pat(Topic, ID) ->
make_key(Topic, ID).
%% @doc Parse a topic filter into a list of words. Returns `false` if it's not a filter.
-spec filter(emqx_types:topic()) -> words() | false.
filter(Topic) ->

10
apps/emqx/src/emqx_types.erl
View File

@ -40,6 +40,10 @@
words/0
]).
-export_type([
share/0
]).
-export_type([
socktype/0,
sockstate/0,
@ -136,11 +140,14 @@
-type subid() :: binary() | atom().
-type group() :: binary() | undefined.
%% '_' for match spec
-type group() :: binary() | '_'.
-type topic() :: binary().
-type word() :: '' | '+' | '#' | binary().
-type words() :: list(word()).
-type share() :: #share{}.
-type socktype() :: tcp | udp | ssl | proxy | atom().
-type sockstate() :: idle | running | blocked | closed.
-type conninfo() :: #{
@ -207,7 +214,6 @@
rap := 0 | 1,
nl := 0 | 1,
qos := qos(),
share => binary(),
atom() => term()
}.
-type reason_code() :: 0..16#FF.

3
apps/emqx/src/emqx_vm.erl
View File

@ -418,6 +418,9 @@ get_otp_version() ->
end.
read_otp_version() ->
string:trim(do_read_otp_version()).
do_read_otp_version() ->
ReleasesDir = filename:join([code:root_dir(), "releases"]),
Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILD_INFO"]),
case file:read_file(Filename) of

33
apps/emqx/test/emqx_broker_SUITE.erl
View File

@ -299,14 +299,19 @@ t_nosub_pub(Config) when is_list(Config) ->
?assertEqual(1, emqx_metrics:val('messages.dropped')).
t_shared_subscribe({init, Config}) ->
emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}),
emqx_broker:subscribe(
emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{}
),
ct:sleep(100),
Config;
t_shared_subscribe(Config) when is_list(Config) ->
emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)),
?assert(
receive
{deliver, <<"topic">>, #message{payload = <<"hello">>}} ->
{deliver, <<"topic">>, #message{
headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)},
payload = <<"hello">>
}} ->
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
@ -316,7 +321,7 @@ t_shared_subscribe(Config) when is_list(Config) ->
end
);
t_shared_subscribe({'end', _Config}) ->
emqx_broker:unsubscribe(<<"$share/group/topic">>).
emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)).
t_shared_subscribe_2({init, Config}) ->
Config;
@ -723,24 +728,6 @@ t_connack_auth_error(Config) when is_list(Config) ->
?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')),
ok.
t_handle_in_empty_client_subscribe_hook({init, Config}) ->
Hook = {?MODULE, client_subscribe_delete_all_hook, []},
ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100),
Config;
t_handle_in_empty_client_subscribe_hook({'end', _Config}) ->
emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}),
ok;
t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) ->
{ok, C} = emqtt:start_link(),
{ok, _} = emqtt:connect(C),
try
{ok, _, RCs} = emqtt:subscribe(C, <<"t">>),
?assertEqual([?RC_UNSPECIFIED_ERROR], RCs),
ok
after
emqtt:disconnect(C)
end.
authenticate_deny(_Credentials, _Default) ->
{stop, {error, bad_username_or_password}}.
@ -800,7 +787,3 @@ recv_msgs(Count, Msgs) ->
after 100 ->
Msgs
end.
client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) ->
EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter],
{stop, EmptyFilters}.

11
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -456,7 +456,7 @@ t_process_subscribe(_) ->
ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
{[{TopicFilter, ?RC_SUCCESS}], _Channel} =
emqx_channel:process_subscribe(TopicFilters, #{}, channel()).
emqx_channel:process_subscribe(TopicFilters, channel()).
t_process_unsubscribe(_) ->
ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end),
@ -914,7 +914,13 @@ t_check_pub_alias(_) ->
t_check_sub_authzs(_) ->
emqx_config:put_zone_conf(default, [authorization, enable], true),
TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
[{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()).
SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]),
CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]),
Channel = channel(),
?assertEqual(
{ok, CheckedSubPkt, Channel},
emqx_channel:check_sub_authzs(SubPkt, Channel)
).
t_enrich_connack_caps(_) ->
ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),
@ -1061,6 +1067,7 @@ clientinfo(InitProps) ->
clientid => <<"clientid">>,
username => <<"username">>,
is_superuser => false,
is_bridge => false,
mountpoint => undefined
},
InitProps

33
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -753,24 +753,15 @@ start_slave(Name, Opts) when is_map(Opts) ->
case SlaveMod of
ct_slave ->
ct:pal("~p: node data dir: ~s", [Node, NodeDataDir]),
ct_slave:start(
Node,
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 20_000},
{startup_timeout, 20_000},
{erl_flags, erl_flags()},
{env, [
Envs = [
{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"},
{"EMQX_NODE__COOKIE", Cookie},
{"EMQX_NODE__DATA_DIR", NodeDataDir}
]}
]
);
],
emqx_cth_peer:start(Node, erl_flags(), Envs);
slave ->
Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_",
slave:start_link(host(), Name, ebin_path() ++ Env)
Envs = [{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}],
emqx_cth_peer:start(Node, ebin_path(), Envs)
end
end,
case DoStart() of
@ -789,13 +780,7 @@ start_slave(Name, Opts) when is_map(Opts) ->
%% Node stopping
stop_slave(Node0) ->
Node = node_name(Node0),
SlaveMod = get_peer_mod(Node),
erase_peer_mod(Node),
case SlaveMod:stop(Node) of
ok -> ok;
{ok, _} -> ok;
{error, not_started, _} -> ok
end.
emqx_cth_peer:stop(Node).
%% EPMD starting
start_epmd() ->
@ -1022,11 +1007,11 @@ set_envs(Node, Env) ->
).
erl_flags() ->
%% One core and redirecting logs to master
"+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
%% One core
["+S", "1:1"] ++ ebin_path().
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
["-pa" | lists:filter(fun is_lib/1, code:get_path())].
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch andalso

6
apps/emqx/test/emqx_common_test_http.erl
View File

@ -34,6 +34,9 @@
-define(DEFAULT_APP_KEY, <<"default_app_key">>).
-define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
-define(ROLE_API_SUPERUSER, <<"administrator">>).
request_api(Method, Url, Auth) ->
request_api(Method, Url, [], Auth, []).
@ -96,7 +99,8 @@ create_default_app() ->
?DEFAULT_APP_SECRET,
true,
ExpiredAt,
<<"default app key for test">>
<<"default app key for test">>,
?ROLE_API_SUPERUSER
).
delete_default_app() ->

126
apps/emqx/test/emqx_cth_cluster.erl
View File

@ -38,19 +38,19 @@
%% in `end_per_suite/1` or `end_per_group/2`) with the result from step 2.
-module(emqx_cth_cluster).
-export([start/2]).
-export([start/1, start/2, restart/2]).
-export([stop/1, stop_node/1]).
-export([start_bare_node/2]).
-export([start_bare_nodes/1, start_bare_nodes/2]).
-export([share_load_module/2]).
-export([node_name/1, mk_nodespecs/2]).
-export([start_apps/2, set_node_opts/2]).
-export([start_apps/2]).
-define(APPS_CLUSTERING, [gen_rpc, mria, ekka]).
-define(TIMEOUT_NODE_START_MS, 15000).
-define(TIMEOUT_APPS_START_MS, 60000).
-define(TIMEOUT_APPS_START_MS, 30000).
-define(TIMEOUT_NODE_STOP_S, 15).
%%
@ -109,9 +109,12 @@ when
}.
start(Nodes, ClusterOpts) ->
NodeSpecs = mk_nodespecs(Nodes, ClusterOpts),
ct:pal("Starting cluster:\n ~p", [NodeSpecs]),
start(NodeSpecs).
start(NodeSpecs) ->
ct:pal("(Re)starting nodes:\n ~p", [NodeSpecs]),
% 1. Start bare nodes with only basic applications running
_ = emqx_utils:pmap(fun start_node_init/1, NodeSpecs, ?TIMEOUT_NODE_START_MS),
ok = start_nodes_init(NodeSpecs, ?TIMEOUT_NODE_START_MS),
% 2. Start applications needed to enable clustering
% Generally, this causes some applications to restart, but we deliberately don't
% start them yet.
@ -121,6 +124,11 @@ start(Nodes, ClusterOpts) ->
_ = emqx_utils:pmap(fun run_node_phase_apps/1, NodeSpecs, ?TIMEOUT_APPS_START_MS),
[Node || #{name := Node} <- NodeSpecs].
restart(Node, Spec) ->
ct:pal("Stopping peer node ~p", [Node]),
ok = emqx_cth_peer:stop(Node),
start([Spec#{boot_type => restart}]).
mk_nodespecs(Nodes, ClusterOpts) ->
NodeSpecs = lists:zipwith(
fun(N, {Name, Opts}) -> mk_init_nodespec(N, Name, Opts, ClusterOpts) end,
@ -282,8 +290,50 @@ allocate_listener_port(Type, #{base_port := BasePort}) ->
allocate_listener_ports(Types, Spec) ->
lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]).
start_node_init(Spec = #{name := Node}) ->
Node = start_bare_node(Node, Spec),
start_nodes_init(Specs, Timeout) ->
Names = lists:map(fun(#{name := Name}) -> Name end, Specs),
Nodes = start_bare_nodes(Names, Timeout),
lists:foreach(fun node_init/1, Nodes).
start_bare_nodes(Names) ->
start_bare_nodes(Names, ?TIMEOUT_NODE_START_MS).
start_bare_nodes(Names, Timeout) ->
Args = erl_flags(),
Envs = [],
Waits = lists:map(
fun(Name) ->
WaitTag = {boot_complete, Name},
WaitBoot = {self(), WaitTag},
{ok, _} = emqx_cth_peer:start(Name, Args, Envs, WaitBoot),
WaitTag
end,
Names
),
Deadline = erlang:monotonic_time() + erlang:convert_time_unit(Timeout, millisecond, nanosecond),
Nodes = wait_boot_complete(Waits, Deadline),
lists:foreach(fun(Node) -> pong = net_adm:ping(Node) end, Nodes),
Nodes.
wait_boot_complete([], _) ->
[];
wait_boot_complete(Waits, Deadline) ->
case erlang:monotonic_time() > Deadline of
true ->
error({timeout, Waits});
false ->
ok
end,
receive
{{boot_complete, _Name} = Wait, {started, Node, _Pid}} ->
ct:pal("~p", [Wait]),
[Node | wait_boot_complete(Waits -- [Wait], Deadline)];
{{boot_complete, _Name}, Otherwise} ->
error({unexpected, Otherwise})
after 100 ->
wait_boot_complete(Waits, Deadline)
end.
node_init(Node) ->
% Make it possible to call `ct:pal` and friends (if running under rebar3)
_ = share_load_module(Node, cthr),
% Enable snabbkaffe trace forwarding
@ -300,12 +350,6 @@ run_node_phase_apps(Spec = #{name := Node}) ->
ok = start_apps(Node, Spec),
ok.
set_node_opts(Node, Spec) ->
erpc:call(Node, persistent_term, put, [{?MODULE, opts}, Spec]).
get_node_opts(Node) ->
erpc:call(Node, persistent_term, get, [{?MODULE, opts}]).
load_apps(Node, #{apps := Apps}) ->
erpc:call(Node, emqx_cth_suite, load_apps, [Apps]).
@ -322,8 +366,12 @@ start_apps(Node, #{apps := Apps} = Spec) ->
ok.
suite_opts(Spec) ->
maps:with([work_dir], Spec).
maps:with([work_dir, boot_type], Spec).
maybe_join_cluster(_Node, #{boot_type := restart}) ->
%% when restart, the node should already be in the cluster
%% hence no need to (re)join
ok;
maybe_join_cluster(_Node, #{role := replicant}) ->
ok;
maybe_join_cluster(Node, Spec) ->
@ -352,23 +400,7 @@ stop(Nodes) ->
stop_node(Name) ->
Node = node_name(Name),
try get_node_opts(Node) of
Opts ->
stop_node(Name, Opts)
catch
error:{erpc, _} ->
ok
end.
stop_node(Node, #{driver := ct_slave}) ->
case ct_slave:stop(Node, [{stop_timeout, ?TIMEOUT_NODE_STOP_S}]) of
{ok, _} ->
ok;
{error, Reason, _} when Reason == not_connected; Reason == not_started ->
ok
end;
stop_node(Node, #{driver := slave}) ->
slave:stop(Node).
ok = emqx_cth_peer:stop(Node).
%% Ports
@ -391,36 +423,12 @@ listener_port(BasePort, wss) ->
%%
-spec start_bare_node(atom(), map()) -> node().
start_bare_node(Name, Spec = #{driver := ct_slave}) ->
{ok, Node} = ct_slave:start(
node_name(Name),
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 20_000},
{startup_timeout, 20_000},
{erl_flags, erl_flags()},
{env, []}
]
),
init_bare_node(Node, Spec);
start_bare_node(Name, Spec = #{driver := slave}) ->
{ok, Node} = slave:start_link(host(), Name, ebin_path()),
init_bare_node(Node, Spec).
init_bare_node(Node, Spec) ->
pong = net_adm:ping(Node),
% Preserve node spec right on the remote node
ok = set_node_opts(Node, Spec),
Node.
erl_flags() ->
%% One core and redirecting logs to master
"+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
%% One core
["+S", "1:1"] ++ ebin_path().
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
["-pa" | lists:filter(fun is_lib/1, code:get_path())].
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch andalso

79
apps/emqx/test/emqx_cth_peer.erl Normal file
View File

@ -0,0 +1,79 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc Common Test Helper proxy module for slave -> peer migration.
%% OTP 26 has slave module deprecated, use peer instead.
-module(emqx_cth_peer).
-export([start/2, start/3, start/4]).
-export([start_link/2, start_link/3, start_link/4]).
-export([stop/1]).
start(Name, Args) ->
start(Name, Args, []).
start(Name, Args, Envs) ->
start(Name, Args, Envs, timer:seconds(20)).
start(Name, Args, Envs, Timeout) when is_atom(Name) ->
do_start(Name, Args, Envs, Timeout, start).
start_link(Name, Args) ->
start_link(Name, Args, []).
start_link(Name, Args, Envs) ->
start_link(Name, Args, Envs, timer:seconds(20)).
start_link(Name, Args, Envs, Timeout) when is_atom(Name) ->
do_start(Name, Args, Envs, Timeout, start_link).
do_start(Name0, Args, Envs, Timeout, Func) when is_atom(Name0) ->
{Name, Host} = parse_node_name(Name0),
{ok, Pid, Node} = peer:Func(#{
name => Name,
host => Host,
args => Args,
env => Envs,
wait_boot => Timeout,
longnames => true,
shutdown => {halt, 1000}
}),
true = register(Node, Pid),
{ok, Node}.
stop(Node) when is_atom(Node) ->
Pid = whereis(Node),
case is_pid(Pid) of
true ->
unlink(Pid),
ok = peer:stop(Pid);
false ->
ct:pal("The control process for node ~p is unexpetedly down", [Node]),
ok
end.
parse_node_name(NodeName) ->
case string:tokens(atom_to_list(NodeName), "@") of
[Name, Host] ->
{list_to_atom(Name), Host};
[_] ->
{NodeName, host()}
end.
host() ->
[_Name, Host] = string:tokens(atom_to_list(node()), "@"),
Host.

3
apps/emqx/test/emqx_cth_suite.erl
View File

@ -453,6 +453,9 @@ stop_apps(Apps) ->
%%
verify_clean_suite_state(#{boot_type := restart}) ->
%% when testing node restart, we do not need to verify clean state
ok;
verify_clean_suite_state(#{work_dir := WorkDir}) ->
{ok, []} = file:list_dir(WorkDir),
false = emqx_schema_hooks:any_injections(),

20
apps/emqx/test/emqx_metrics_worker_SUITE.erl
View File

@ -53,9 +53,9 @@ t_get_metrics(_) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
counters := #{
@ -118,9 +118,9 @@ t_clear_metrics(_Config) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
slides := #{},
@ -145,7 +145,7 @@ t_clear_metrics(_Config) ->
#{
counters => #{},
gauges => #{},
rate => #{current => 0.0, last5m => 0.0, max => 0.0},
rate => #{current => +0.0, last5m => +0.0, max => +0.0},
slides => #{}
},
emqx_metrics_worker:get_metrics(?NAME, Id)
@ -160,9 +160,9 @@ t_reset_metrics(_) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
counters := #{

34
apps/emqx/test/emqx_mountpoint_SUITE.erl
View File

@ -29,6 +29,7 @@
).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
@ -52,6 +53,24 @@ t_mount(_) ->
mount(<<"device/1/">>, TopicFilters)
).
t_mount_share(_) ->
T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>),
TopicFilters = [T],
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
?assertEqual(
TopicFilter,
mount(undefined, TopicFilter)
),
?assertEqual(
#share{group = <<"group">>, topic = <<"device/1/topic">>},
mount(<<"device/1/">>, TopicFilter)
),
?assertEqual(
[{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}],
mount(<<"device/1/">>, TopicFilters)
).
t_unmount(_) ->
Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>),
?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)),
@ -61,6 +80,21 @@ t_unmount(_) ->
?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)),
?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)).
t_unmount_share(_) ->
{TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>),
MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>},
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
?assertEqual(
TopicFilter,
unmount(undefined, TopicFilter)
),
?assertEqual(
#share{group = <<"group">>, topic = <<"topic">>},
unmount(<<"device/1/">>, MountedTopicFilter)
).
t_replvar(_) ->
?assertEqual(undefined, replvar(undefined, #{})),
?assertEqual(

4
apps/emqx/test/emqx_mqtt_caps_SUITE.erl
View File

@ -76,6 +76,8 @@ t_check_sub(_) ->
),
?assertEqual(
{error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})
emqx_mqtt_caps:check_sub(
ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts
)
),
emqx_config:put([zones], OldConf).

55
apps/emqx/test/emqx_persistent_messages_SUITE.erl
View File

@ -233,6 +233,55 @@ t_session_subscription_iterators(Config) ->
),
ok.
t_qos0(_Config) ->
Sub = connect(<<?MODULE_STRING "1">>, true, 30),
Pub = connect(<<?MODULE_STRING "2">>, true, 0),
try
{ok, _, [1]} = emqtt:subscribe(Sub, <<"t/#">>, qos1),
Messages = [
{<<"t/1">>, <<"1">>, 0},
{<<"t/1">>, <<"2">>, 1},
{<<"t/1">>, <<"3">>, 0}
],
[emqtt:publish(Pub, Topic, Payload, Qos) || {Topic, Payload, Qos} <- Messages],
?assertMatch(
[
#{qos := 0, topic := <<"t/1">>, payload := <<"1">>},
#{qos := 1, topic := <<"t/1">>, payload := <<"2">>},
#{qos := 0, topic := <<"t/1">>, payload := <<"3">>}
],
receive_messages(3)
)
after
emqtt:stop(Sub),
emqtt:stop(Pub)
end.
t_publish_as_persistent(_Config) ->
Sub = connect(<<?MODULE_STRING "1">>, true, 30),
Pub = connect(<<?MODULE_STRING "2">>, true, 30),
try
{ok, _, [1]} = emqtt:subscribe(Sub, <<"t/#">>, qos1),
Messages = [
{<<"t/1">>, <<"1">>, 0},
{<<"t/1">>, <<"2">>, 1},
{<<"t/1">>, <<"3">>, 2}
],
[emqtt:publish(Pub, Topic, Payload, Qos) || {Topic, Payload, Qos} <- Messages],
?assertMatch(
[
#{qos := 0, topic := <<"t/1">>, payload := <<"1">>},
#{qos := 1, topic := <<"t/1">>, payload := <<"2">>},
#{qos := 2, topic := <<"t/1">>, payload := <<"3">>}
],
receive_messages(3)
)
after
emqtt:stop(Sub),
emqtt:stop(Pub)
end.
%%
connect(ClientId, CleanStart, EI) ->
@ -273,7 +322,7 @@ consume(It) ->
end.
receive_messages(Count) ->
receive_messages(Count, []).
lists:reverse(receive_messages(Count, [])).
receive_messages(0, Msgs) ->
Msgs;
@ -291,7 +340,7 @@ publish(Node, Message) ->
app_specs() ->
[
emqx_durable_storage,
{emqx, "persistent_session_store {ds = true}"}
{emqx, "session_persistence {enable = true}"}
].
cluster() ->
@ -307,4 +356,6 @@ get_mqtt_port(Node, Type) ->
clear_db() ->
ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB),
mria:stop(),
ok = mnesia:delete_schema([node()]),
ok.

546
apps/emqx/test/emqx_persistent_session_SUITE.erl
View File

@ -17,6 +17,7 @@
-module(emqx_persistent_session_SUITE).
-include_lib("stdlib/include/assert.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
@ -35,8 +36,8 @@ all() ->
% NOTE
% Tests are disabled while existing session persistence impl is being
% phased out.
{group, persistent_store_disabled},
{group, persistent_store_ds}
{group, persistence_disabled},
{group, persistence_enabled}
].
%% A persistent session can be resumed in two ways:
@ -53,28 +54,28 @@ all() ->
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
TCsNonGeneric = [t_choose_impl],
TCGroups = [{group, tcp}, {group, quic}, {group, ws}],
[
{persistent_store_disabled, [{group, no_kill_connection_process}]},
{persistent_store_ds, [{group, no_kill_connection_process}]},
{no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]},
{persistence_disabled, TCGroups},
{persistence_enabled, TCGroups},
{tcp, [], TCs},
{quic, [], TCs -- TCsNonGeneric},
{ws, [], TCs -- TCsNonGeneric}
].
init_per_group(persistent_store_disabled, Config) ->
init_per_group(persistence_disabled, Config) ->
[
{emqx_config, "persistent_session_store { enabled = false }"},
{persistent_store, false}
{emqx_config, "session_persistence { enable = false }"},
{persistence, false}
| Config
];
init_per_group(persistent_store_ds, Config) ->
init_per_group(persistence_enabled, Config) ->
[
{emqx_config, "persistent_session_store { ds = true }"},
{persistent_store, ds}
{emqx_config, "session_persistence { enable = true }"},
{persistence, ds}
| Config
];
init_per_group(Group, Config) when Group == tcp ->
init_per_group(tcp, Config) ->
Apps = emqx_cth_suite:start(
[{emqx, ?config(emqx_config, Config)}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
@ -85,7 +86,7 @@ init_per_group(Group, Config) when Group == tcp ->
{group_apps, Apps}
| Config
];
init_per_group(Group, Config) when Group == ws ->
init_per_group(ws, Config) ->
Apps = emqx_cth_suite:start(
[{emqx, ?config(emqx_config, Config)}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
@ -99,7 +100,7 @@ init_per_group(Group, Config) when Group == ws ->
{group_apps, Apps}
| Config
];
init_per_group(Group, Config) when Group == quic ->
init_per_group(quic, Config) ->
Apps = emqx_cth_suite:start(
[
{emqx,
@ -118,11 +119,7 @@ init_per_group(Group, Config) when Group == quic ->
{ssl, true},
{group_apps, Apps}
| Config
];
init_per_group(no_kill_connection_process, Config) ->
[{kill_connection_process, false} | Config];
init_per_group(kill_connection_process, Config) ->
[{kill_connection_process, true} | Config].
].
get_listener_port(Type, Name) ->
case emqx_config:get([listeners, Type, Name, bind]) of
@ -181,64 +178,106 @@ client_info(Key, Client) ->
maps:get(Key, maps:from_list(emqtt:info(Client)), undefined).
receive_messages(Count) ->
receive_messages(Count, []).
receive_messages(Count, 15000).
receive_messages(0, Msgs) ->
Msgs;
receive_messages(Count, Msgs) ->
receive_messages(Count, Timeout) ->
Deadline = erlang:monotonic_time(millisecond) + Timeout,
receive_message_loop(Count, Deadline).
receive_message_loop(0, _Deadline) ->
[];
receive_message_loop(Count, Deadline) ->
Timeout = max(0, Deadline - erlang:monotonic_time(millisecond)),
receive
{publish, Msg} ->
receive_messages(Count - 1, [Msg | Msgs]);
[Msg | receive_message_loop(Count - 1, Deadline)];
{pubrel, Msg} ->
[{pubrel, Msg} | receive_message_loop(Count - 1, Deadline)];
_Other ->
receive_messages(Count, Msgs)
after 15000 ->
Msgs
receive_message_loop(Count, Deadline)
after Timeout ->
[]
end.
maybe_kill_connection_process(ClientId, Config) ->
case ?config(kill_connection_process, Config) of
true ->
Persistence = ?config(persistence, Config),
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[ConnectionPid] when Persistence == ds ->
Ref = monitor(process, ConnectionPid),
ConnectionPid ! die_if_test,
?assertReceive(
{'DOWN', Ref, process, ConnectionPid, Reason} when
Reason == normal orelse Reason == noproc,
3000
),
wait_connection_process_unregistered(ClientId);
_ ->
ok
end.
wait_connection_process_dies(ClientId) ->
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[ConnectionPid] ->
?assert(is_pid(ConnectionPid)),
Ref = monitor(process, ConnectionPid),
ConnectionPid ! die_if_test,
receive
{'DOWN', Ref, process, ConnectionPid, normal} -> ok
after 3000 -> error(process_did_not_die)
end,
wait_for_cm_unregister(ClientId)
end;
false ->
ok
?assertReceive(
{'DOWN', Ref, process, ConnectionPid, Reason} when
Reason == normal orelse Reason == noproc,
3000
),
wait_connection_process_unregistered(ClientId)
end.
wait_for_cm_unregister(ClientId) ->
wait_for_cm_unregister(ClientId, 100).
wait_connection_process_unregistered(ClientId) ->
?retry(
_Timeout = 100,
_Retries = 20,
?assertEqual([], emqx_cm:lookup_channels(ClientId))
).
wait_for_cm_unregister(_ClientId, 0) ->
error(cm_did_not_unregister);
wait_for_cm_unregister(ClientId, N) ->
wait_channel_disconnected(ClientId) ->
?retry(
_Timeout = 100,
_Retries = 20,
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[_] ->
timer:sleep(100),
wait_for_cm_unregister(ClientId, N - 1)
end.
false;
[ChanPid] ->
false = emqx_cm:is_channel_connected(ChanPid)
end
).
publish(Topic, Payloads) ->
publish(Topic, Payloads, false, 2).
disconnect_client(ClientPid) ->
ClientId = proplists:get_value(clientid, emqtt:info(ClientPid)),
ok = emqtt:disconnect(ClientPid),
false = wait_channel_disconnected(ClientId),
ok.
publish(Topic, Payloads, WaitForUnregister, QoS) ->
Fun = fun(Client, Payload) ->
{ok, _} = emqtt:publish(Client, Topic, Payload, QoS)
messages(Topic, Payloads) ->
messages(Topic, Payloads, ?QOS_2).
messages(Topic, Payloads, QoS) ->
[#mqtt_msg{topic = Topic, payload = P, qos = QoS} || P <- Payloads].
publish(Topic, Payload) ->
publish(Topic, Payload, ?QOS_2).
publish(Topic, Payload, QoS) ->
publish_many(messages(Topic, [Payload], QoS)).
publish_many(Messages) ->
publish_many(Messages, false).
publish_many(Messages, WaitForUnregister) ->
Fun = fun(Client, Message) ->
{ok, _} = emqtt:publish(Client, Message)
end,
do_publish(Payloads, Fun, WaitForUnregister).
do_publish(Messages, Fun, WaitForUnregister).
do_publish(Payloads = [_ | _], PublishFun, WaitForUnregister) ->
do_publish(Messages = [_ | _], PublishFun, WaitForUnregister) ->
%% Publish from another process to avoid connection confusion.
{Pid, Ref} =
spawn_monitor(
@ -252,34 +291,16 @@ do_publish(Payloads = [_ | _], PublishFun, WaitForUnregister) ->
{port, 1883}
]),
{ok, _} = emqtt:connect(Client),
lists:foreach(fun(Payload) -> PublishFun(Client, Payload) end, Payloads),
lists:foreach(fun(Message) -> PublishFun(Client, Message) end, Messages),
ok = emqtt:disconnect(Client),
%% Snabbkaffe sometimes fails unless all processes are gone.
case WaitForUnregister of
false ->
ok;
true ->
case emqx_cm:lookup_channels(ClientID) of
[] ->
ok;
[ConnectionPid] ->
?assert(is_pid(ConnectionPid)),
Ref1 = monitor(process, ConnectionPid),
receive
{'DOWN', Ref1, process, ConnectionPid, _} -> ok
after 3000 -> error(process_did_not_die)
end,
wait_for_cm_unregister(ClientID)
end
end
WaitForUnregister andalso wait_connection_process_dies(ClientID)
end
),
receive
{'DOWN', Ref, process, Pid, normal} -> ok;
{'DOWN', Ref, process, Pid, What} -> error({failed_publish, What})
end;
do_publish(Payload, PublishFun, WaitForUnregister) ->
do_publish([Payload], PublishFun, WaitForUnregister).
end.
%%--------------------------------------------------------------------
%% Test Cases
@ -297,7 +318,7 @@ t_choose_impl(Config) ->
{ok, _} = emqtt:ConnFun(Client),
[ChanPid] = emqx_cm:lookup_channels(ClientId),
?assertEqual(
case ?config(persistent_store, Config) of
case ?config(persistence, Config) of
false -> emqx_session_mem;
ds -> emqx_persistent_session_ds
end,
@ -332,8 +353,6 @@ t_connect_discards_existing_client(Config) ->
end.
%% [MQTT-3.1.2-23]
t_connect_session_expiry_interval(init, Config) -> skip_ds_tc(Config);
t_connect_session_expiry_interval('end', _Config) -> ok.
t_connect_session_expiry_interval(Config) ->
ConnFun = ?config(conn_fun, Config),
Topic = ?config(topic, Config),
@ -341,6 +360,45 @@ t_connect_session_expiry_interval(Config) ->
Payload = <<"test message">>,
ClientId = ?config(client_id, Config),
{ok, Client1} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
{properties, #{'Session-Expiry-Interval' => 30}}
| Config
]),
{ok, _} = emqtt:ConnFun(Client1),
{ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client1, STopic, ?QOS_1),
ok = emqtt:disconnect(Client1),
maybe_kill_connection_process(ClientId, Config),
publish(Topic, Payload, ?QOS_1),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
{properties, #{'Session-Expiry-Interval' => 30}},
{clean_start, false}
| Config
]),
{ok, _} = emqtt:ConnFun(Client2),
[Msg | _] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Topic)}, maps:find(topic, Msg)),
?assertEqual({ok, iolist_to_binary(Payload)}, maps:find(payload, Msg)),
?assertEqual({ok, ?QOS_1}, maps:find(qos, Msg)),
ok = emqtt:disconnect(Client2).
%% [MQTT-3.1.2-23]
%% TODO: un-skip after QoS 2 support is implemented in DS.
t_connect_session_expiry_interval_qos2(init, Config) -> skip_ds_tc(Config);
t_connect_session_expiry_interval_qos2('end', _Config) -> ok.
t_connect_session_expiry_interval_qos2(Config) ->
ConnFun = ?config(conn_fun, Config),
Topic = ?config(topic, Config),
STopic = ?config(stopic, Config),
Payload = <<"test message">>,
ClientId = ?config(client_id, Config),
{ok, Client1} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
@ -423,7 +481,7 @@ t_cancel_on_disconnect(Config) ->
{ok, _} = emqtt:ConnFun(Client1),
ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 0}),
wait_for_cm_unregister(ClientId),
wait_connection_process_unregistered(ClientId),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
@ -455,7 +513,7 @@ t_persist_on_disconnect(Config) ->
%% Strangely enough, the disconnect is reported as successful by emqtt.
ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 30}),
wait_for_cm_unregister(ClientId),
wait_connection_process_unregistered(ClientId),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
@ -470,8 +528,6 @@ t_persist_on_disconnect(Config) ->
?assertEqual(0, client_info(session_present, Client2)),
ok = emqtt:disconnect(Client2).
t_process_dies_session_expires(init, Config) -> skip_ds_tc(Config);
t_process_dies_session_expires('end', _Config) -> ok.
t_process_dies_session_expires(Config) ->
%% Emulate an error in the connect process,
%% or that the node of the process goes down.
@ -494,7 +550,7 @@ t_process_dies_session_expires(Config) ->
maybe_kill_connection_process(ClientId, Config),
ok = publish(Topic, [Payload]),
ok = publish(Topic, Payload),
timer:sleep(1100),
@ -535,7 +591,7 @@ t_publish_while_client_is_gone_qos1(Config) ->
ok = emqtt:disconnect(Client1),
maybe_kill_connection_process(ClientId, Config),
ok = publish(Topic, [Payload1, Payload2], false, 1),
ok = publish_many(messages(Topic, [Payload1, Payload2], ?QOS_1)),
{ok, Client2} = emqtt:start_link([
{proto_ver, v5},
@ -547,7 +603,7 @@ t_publish_while_client_is_gone_qos1(Config) ->
{ok, _} = emqtt:ConnFun(Client2),
Msgs = receive_messages(2),
?assertMatch([_, _], Msgs),
[Msg2, Msg1] = Msgs,
[Msg1, Msg2] = Msgs,
?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)),
?assertEqual({ok, 1}, maps:find(qos, Msg1)),
?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg2)),
@ -555,8 +611,123 @@ t_publish_while_client_is_gone_qos1(Config) ->
ok = emqtt:disconnect(Client2).
t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config);
t_publish_while_client_is_gone('end', _Config) -> ok.
t_publish_many_while_client_is_gone_qos1(Config) ->
%% A persistent session should receive all of the still unacked messages
%% for its subscriptions after the client dies or reconnects, in addition
%% to new messages that were published while the client was gone. The order
%% of the messages should be consistent across reconnects.
ClientId = ?config(client_id, Config),
ConnFun = ?config(conn_fun, Config),
{ok, Client1} = emqtt:start_link([
{proto_ver, v5},
{clientid, ClientId},
{properties, #{'Session-Expiry-Interval' => 30}},
{clean_start, true},
{auto_ack, never}
| Config
]),
{ok, _} = emqtt:ConnFun(Client1),
STopics = [
<<"t/+/foo">>,
<<"msg/feed/#">>,
<<"loc/+/+/+">>
],
[{ok, _, [?QOS_1]} = emqtt:subscribe(Client1, ST, ?QOS_1) || ST <- STopics],
Pubs1 = [
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1},
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 1},
#mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 1},
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M5">>, qos = 1},
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M6">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M7">>, qos = 1}
],
ok = publish_many(Pubs1),
NPubs1 = length(Pubs1),
Msgs1 = receive_messages(NPubs1),
NMsgs1 = length(Msgs1),
?assertEqual(NPubs1, NMsgs1),
ct:pal("Msgs1 = ~p", [Msgs1]),
%% TODO
%% This assertion doesn't currently hold because `emqx_ds` doesn't enforce
%% strict ordering reflecting client publishing order. Instead, per-topic
%% ordering is guaranteed per each client. In fact, this violates the MQTT
%% specification, but we deemed it acceptable for now.
%% ?assertMatch([
%% #{payload := <<"M1">>},
%% #{payload := <<"M2">>},
%% #{payload := <<"M3">>},
%% #{payload := <<"M4">>},
%% #{payload := <<"M5">>},
%% #{payload := <<"M6">>},
%% #{payload := <<"M7">>}
%% ], Msgs1),
?assertEqual(
get_topicwise_order(Pubs1),
get_topicwise_order(Msgs1)
),
NAcked = 4,
[ok = emqtt:puback(Client1, PktId) || #{packet_id := PktId} <- lists:sublist(Msgs1, NAcked)],
%% Ensure that PUBACKs are propagated to the channel.
pong = emqtt:ping(Client1),
ok = disconnect_client(Client1),
maybe_kill_connection_process(ClientId, Config),
Pubs2 = [
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M8">>, qos = 1},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M9">>, qos = 1},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M10">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M11">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M12">>, qos = 1}
],
ok = publish_many(Pubs2),
NPubs2 = length(Pubs2),
{ok, Client2} = emqtt:start_link([
{proto_ver, v5},
{clientid, ClientId},
{properties, #{'Session-Expiry-Interval' => 30}},
{clean_start, false},
{auto_ack, false}
| Config
]),
{ok, _} = emqtt:ConnFun(Client2),
%% Try to receive _at most_ `NPubs` messages.
%% There shouldn't be that much unacked messages in the replay anyway,
%% but it's an easy number to pick.
NPubs = NPubs1 + NPubs2,
Msgs2 = receive_messages(NPubs, _Timeout = 2000),
NMsgs2 = length(Msgs2),
ct:pal("Msgs2 = ~p", [Msgs2]),
?assert(NMsgs2 < NPubs, Msgs2),
?assert(NMsgs2 > NPubs2, Msgs2),
?assert(NMsgs2 >= NPubs - NAcked, Msgs2),
NSame = NMsgs2 - NPubs2,
?assert(
lists:all(fun(#{dup := Dup}) -> Dup end, lists:sublist(Msgs2, NSame))
),
?assertNot(
lists:all(fun(#{dup := Dup}) -> Dup end, lists:nthtail(NSame, Msgs2))
),
?assertEqual(
[maps:with([packet_id, topic, payload], M) || M <- lists:nthtail(NMsgs1 - NSame, Msgs1)],
[maps:with([packet_id, topic, payload], M) || M <- lists:sublist(Msgs2, NSame)]
),
ok = disconnect_client(Client2).
t_publish_while_client_is_gone(Config) ->
%% A persistent session should receive messages in its
%% subscription even if the process owning the session dies.
@ -579,7 +750,7 @@ t_publish_while_client_is_gone(Config) ->
ok = emqtt:disconnect(Client1),
maybe_kill_connection_process(ClientId, Config),
ok = publish(Topic, [Payload1, Payload2]),
ok = publish_many(messages(Topic, [Payload1, Payload2])),
{ok, Client2} = emqtt:start_link([
{proto_ver, v5},
@ -591,7 +762,7 @@ t_publish_while_client_is_gone(Config) ->
{ok, _} = emqtt:ConnFun(Client2),
Msgs = receive_messages(2),
?assertMatch([_, _], Msgs),
[Msg2, Msg1] = Msgs,
[Msg1, Msg2] = Msgs,
?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)),
?assertEqual({ok, 2}, maps:find(qos, Msg1)),
?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg2)),
@ -599,9 +770,157 @@ t_publish_while_client_is_gone(Config) ->
ok = emqtt:disconnect(Client2).
%% TODO: don't skip after QoS2 support is added to DS.
t_clean_start_drops_subscriptions(init, Config) -> skip_ds_tc(Config);
t_clean_start_drops_subscriptions('end', _Config) -> ok.
t_publish_many_while_client_is_gone(Config) ->
%% A persistent session should receive all of the still unacked messages
%% for its subscriptions after the client dies or reconnects, in addition
%% to PUBRELs for the messages it has PUBRECed. While client must send
%% PUBACKs and PUBRECs in order, those orders are independent of each other.
ClientId = ?config(client_id, Config),
ConnFun = ?config(conn_fun, Config),
ClientOpts = [
{proto_ver, v5},
{clientid, ClientId},
{properties, #{'Session-Expiry-Interval' => 30}},
{auto_ack, never}
| Config
],
{ok, Client1} = emqtt:start_link([{clean_start, true} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client1),
{ok, _, [?QOS_1]} = emqtt:subscribe(Client1, <<"t/+/foo">>, ?QOS_1),
{ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"msg/feed/#">>, ?QOS_2),
{ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"loc/+/+/+">>, ?QOS_2),
Pubs1 = [
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1},
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 2},
#mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M5">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M6">>, qos = 1},
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M7">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M8">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M9">>, qos = 2}
],
ok = publish_many(Pubs1),
NPubs1 = length(Pubs1),
Msgs1 = receive_messages(NPubs1),
ct:pal("Msgs1 = ~p", [Msgs1]),
NMsgs1 = length(Msgs1),
?assertEqual(NPubs1, NMsgs1),
?assertEqual(
get_topicwise_order(Pubs1),
get_topicwise_order(Msgs1)
),
%% PUBACK every QoS 1 message.
lists:foreach(
fun(PktId) -> ok = emqtt:puback(Client1, PktId) end,
[PktId || #{qos := 1, packet_id := PktId} <- Msgs1]
),
%% PUBREC first `NRecs` QoS 2 messages.
NRecs = 3,
PubRecs1 = lists:sublist([PktId || #{qos := 2, packet_id := PktId} <- Msgs1], NRecs),
lists:foreach(
fun(PktId) -> ok = emqtt:pubrec(Client1, PktId) end,
PubRecs1
),
%% Ensure that PUBACKs / PUBRECs are propagated to the channel.
pong = emqtt:ping(Client1),
%% Receive PUBRELs for the sent PUBRECs.
PubRels1 = receive_messages(NRecs),
ct:pal("PubRels1 = ~p", [PubRels1]),
?assertEqual(
PubRecs1,
[PktId || {pubrel, #{packet_id := PktId}} <- PubRels1],
PubRels1
),
ok = disconnect_client(Client1),
maybe_kill_connection_process(ClientId, Config),
Pubs2 = [
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M10">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M11">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M12">>, qos = 2}
],
ok = publish_many(Pubs2),
NPubs2 = length(Pubs2),
{ok, Client2} = emqtt:start_link([{clean_start, false} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client2),
%% Try to receive _at most_ `NPubs` messages.
%% There shouldn't be that much unacked messages in the replay anyway,
%% but it's an easy number to pick.
NPubs = NPubs1 + NPubs2,
Msgs2 = receive_messages(NPubs, _Timeout = 2000),
ct:pal("Msgs2 = ~p", [Msgs2]),
%% We should again receive PUBRELs for the PUBRECs we sent earlier.
?assertEqual(
get_msgs_essentials(PubRels1),
[get_msg_essentials(PubRel) || PubRel = {pubrel, _} <- Msgs2]
),
%% We should receive duplicates only for QoS 2 messages where PUBRELs were
%% not sent, in the same order as the original messages.
Msgs2Dups = [get_msg_essentials(M) || M = #{dup := true} <- Msgs2],
?assertEqual(
Msgs2Dups,
[M || M = #{qos := 2} <- Msgs2Dups]
),
?assertEqual(
get_msgs_essentials(pick_respective_msgs(Msgs2Dups, Msgs1)),
Msgs2Dups
),
%% Now complete all yet incomplete QoS 2 message flows instead.
PubRecs2 = [PktId || #{qos := 2, packet_id := PktId} <- Msgs2],
lists:foreach(
fun(PktId) -> ok = emqtt:pubrec(Client2, PktId) end,
PubRecs2
),
PubRels2 = receive_messages(length(PubRecs2)),
ct:pal("PubRels2 = ~p", [PubRels2]),
?assertEqual(
PubRecs2,
[PktId || {pubrel, #{packet_id := PktId}} <- PubRels2],
PubRels2
),
%% PUBCOMP every PUBREL.
PubComps = [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1 ++ PubRels2],
lists:foreach(
fun(PktId) -> ok = emqtt:pubcomp(Client2, PktId) end,
PubComps
),
%% Ensure that PUBCOMPs are propagated to the channel.
pong = emqtt:ping(Client2),
ok = disconnect_client(Client2),
maybe_kill_connection_process(ClientId, Config),
{ok, Client3} = emqtt:start_link([{clean_start, false} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client3),
%% Only the last unacked QoS 1 message should be retransmitted.
Msgs3 = receive_messages(NPubs, _Timeout = 2000),
ct:pal("Msgs3 = ~p", [Msgs3]),
?assertMatch(
[#{topic := <<"t/100/foo">>, payload := <<"M11">>, qos := 1, dup := true}],
Msgs3
),
ok = disconnect_client(Client3).
t_clean_start_drops_subscriptions(Config) ->
%% 1. A persistent session is started and disconnected.
%% 2. While disconnected, a message is published and persisted.
@ -627,13 +946,13 @@ t_clean_start_drops_subscriptions(Config) ->
| Config
]),
{ok, _} = emqtt:ConnFun(Client1),
{ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2),
{ok, _, [1]} = emqtt:subscribe(Client1, STopic, qos1),
ok = emqtt:disconnect(Client1),
maybe_kill_connection_process(ClientId, Config),
%% 2.
ok = publish(Topic, Payload1),
ok = publish(Topic, Payload1, ?QOS_1),
%% 3.
{ok, Client2} = emqtt:start_link([
@ -645,12 +964,14 @@ t_clean_start_drops_subscriptions(Config) ->
]),
{ok, _} = emqtt:ConnFun(Client2),
?assertEqual(0, client_info(session_present, Client2)),
{ok, _, [2]} = emqtt:subscribe(Client2, STopic, qos2),
{ok, _, [1]} = emqtt:subscribe(Client2, STopic, qos1),
ok = publish(Topic, Payload2),
timer:sleep(100),
ok = publish(Topic, Payload2, ?QOS_1),
[Msg1] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg1)),
pong = emqtt:ping(Client2),
ok = emqtt:disconnect(Client2),
maybe_kill_connection_process(ClientId, Config),
@ -664,10 +985,11 @@ t_clean_start_drops_subscriptions(Config) ->
]),
{ok, _} = emqtt:ConnFun(Client3),
ok = publish(Topic, Payload3),
ok = publish(Topic, Payload3, ?QOS_1),
[Msg2] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Payload3)}, maps:find(payload, Msg2)),
pong = emqtt:ping(Client3),
ok = emqtt:disconnect(Client3).
t_unsubscribe(Config) ->
@ -731,8 +1053,32 @@ t_multiple_subscription_matches(Config) ->
?assertEqual({ok, 2}, maps:find(qos, Msg2)),
ok = emqtt:disconnect(Client2).
get_topicwise_order(Msgs) ->
maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs).
get_msgpub_topic(#mqtt_msg{topic = Topic}) ->
Topic;
get_msgpub_topic(#{topic := Topic}) ->
Topic.
get_msgpub_payload(#mqtt_msg{payload = Payload}) ->
Payload;
get_msgpub_payload(#{payload := Payload}) ->
Payload.
get_msg_essentials(Msg = #{}) ->
maps:with([packet_id, topic, payload, qos], Msg);
get_msg_essentials({pubrel, Msg}) ->
{pubrel, maps:with([packet_id, reason_code], Msg)}.
get_msgs_essentials(Msgs) ->
[get_msg_essentials(M) || M <- Msgs].
pick_respective_msgs(MsgRefs, Msgs) ->
[M || M <- Msgs, Ref <- MsgRefs, maps:get(packet_id, M) =:= maps:get(packet_id, Ref)].
skip_ds_tc(Config) ->
case ?config(persistent_store, Config) of
case ?config(persistence, Config) of
ds ->
{skip, "Testcase not yet supported under 'emqx_persistent_session_ds' implementation"};
_ ->

2
apps/emqx/test/emqx_persistent_session_ds_router_SUITE.erl
View File

@ -38,7 +38,7 @@ init_per_suite(Config) ->
AppSpecs = [
emqx_durable_storage,
{emqx, #{
config => #{persistent_session_store => #{ds => true}},
config => #{session_persistence => #{enable => true}},
override_env => [{boot_modules, [broker]}]
}}
],

8
apps/emqx/test/emqx_proper_types.erl
View File

@ -511,13 +511,7 @@ peercert() ->
conn_mod() ->
oneof([
emqx_connection,
emqx_ws_connection,
emqx_coap_mqtt_adapter,
emqx_sn_gateway,
emqx_lwm2m_protocol,
emqx_gbt32960_conn,
emqx_jt808_connection,
emqx_tcp_connection
emqx_ws_connection
]).
proto_name() ->

15
apps/emqx/test/emqx_quic_multistreams_SUITE.erl
View File

@ -669,22 +669,21 @@ t_multi_streams_packet_malform(Config) ->
case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of
{ok, 10} -> ok;
{error, cancelled} -> ok;
{error, stm_send_error, aborted} -> ok
{error, stm_send_error, aborted} -> ok;
{error, closed} -> ok
end,
?assert(is_list(emqtt:info(C))),
{error, stm_send_error, _} =
{error, closed} =
snabbkaffe:retry(
10000,
10,
fun() ->
{error, stm_send_error, _} = quicer:send(
{error, closed} = quicer:send(
MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>
)
end
),
?assert(is_list(emqtt:info(C))),
ok = emqtt:disconnect(C).
@ -770,9 +769,9 @@ t_multi_streams_packet_too_large(Config) ->
timeout = recv_pub(1),
?assert(is_list(emqtt:info(C))),
%% Connection could be kept
{error, stm_send_error, _} = quicer:send(via_stream(PubVia), <<1>>),
{error, stm_send_error, _} = quicer:send(via_stream(PubVia2), <<1>>),
%% Connection could be kept but data stream are closed!
{error, closed} = quicer:send(via_stream(PubVia), <<1>>),
{error, closed} = quicer:send(via_stream(PubVia2), <<1>>),
%% We could send data over new stream
{ok, PubVia3} = emqtt:start_data_stream(C, []),
ok = emqtt:publish_async(

6
apps/emqx/test/emqx_router_helper_SUITE.erl
View File

@ -80,7 +80,7 @@ t_mnesia(_) ->
ct:sleep(200).
t_cleanup_membership_mnesia_down(_Config) ->
Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
Slave = emqx_cth_cluster:node_name(node2),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),
@ -92,7 +92,7 @@ t_cleanup_membership_mnesia_down(_Config) ->
?assertEqual([<<"d/e/f">>], emqx_router:topics()).
t_cleanup_membership_node_down(_Config) ->
Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
Slave = emqx_cth_cluster:node_name(node3),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),
@ -104,7 +104,7 @@ t_cleanup_membership_node_down(_Config) ->
?assertEqual([<<"d/e/f">>], emqx_router:topics()).
t_cleanup_monitor_node_down(_Config) ->
Slave = emqx_cth_cluster:start_bare_node(?FUNCTION_NAME, #{driver => ct_slave}),
[Slave] = emqx_cth_cluster:start_bare_nodes([node4]),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),

9
apps/emqx/test/emqx_routing_SUITE.erl
View File

@ -218,8 +218,9 @@ t_routing_schema_switch(VFrom, VTo, Config) ->
],
#{work_dir => WorkDir}
),
% Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
Nodes = [Node1, Node2, Node3],
try
% Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
?assertEqual(
[{ok, VTo}, {ok, VTo}, {ok, VTo}],
erpc:multicall(Nodes, emqx_router, get_schema_vsn, [])
@ -248,8 +249,10 @@ t_routing_schema_switch(VFrom, VTo, Config) ->
?assertNotReceive(_),
ok = emqtt:stop(C1),
ok = emqtt:stop(C2),
ok = emqtt:stop(C3),
ok = emqx_cth_cluster:stop(Nodes).
ok = emqtt:stop(C3)
after
ok = emqx_cth_cluster:stop(Nodes)
end.
%%

76
apps/emqx/test/emqx_secret_tests.erl Normal file
View File

@ -0,0 +1,76 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_secret_tests).
-include_lib("eunit/include/eunit.hrl").
wrap_unwrap_test() ->
?assertEqual(
42,
emqx_secret:unwrap(emqx_secret:wrap(42))
).
unwrap_immediate_test() ->
?assertEqual(
42,
emqx_secret:unwrap(42)
).
wrap_unwrap_load_test_() ->
Secret = <<"foobaz">>,
{
setup,
fun() -> write_temp_file(Secret) end,
fun(Filename) -> file:delete(Filename) end,
fun(Filename) ->
?_assertEqual(
Secret,
emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename}))
)
end
}.
wrap_load_term_test() ->
?assertEqual(
{file, "no/such/file/i/swear"},
emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
).
wrap_unwrap_missing_file_test() ->
?assertThrow(
#{msg := failed_to_read_secret_file, reason := "No such file or directory"},
emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
).
wrap_term_test() ->
?assertEqual(
42,
emqx_secret:term(emqx_secret:wrap(42))
).
external_fun_term_error_test() ->
Term = {foo, bar},
?assertError(
badarg,
emqx_secret:term(fun() -> Term end)
).
write_temp_file(Bytes) ->
Ts = erlang:system_time(millisecond),
Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)),
ok = file:write_file(Filename, Bytes),
Filename.

216
apps/emqx/test/emqx_shared_sub_SUITE.erl
View File

@ -63,6 +63,7 @@ init_per_suite(Config) ->
end,
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
emqx_logger:set_log_level(debug),
[{dist_pid, DistPid} | Config].
end_per_suite(Config) ->
@ -137,7 +138,8 @@ t_random_basic(Config) when is_list(Config) ->
ClientId = <<"ClientId">>,
Topic = <<"foo">>,
Payload = <<"hello">>,
emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
Group = <<"group1">>,
emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload),
%% wait for the subscription to show up
ct:sleep(200),
@ -402,7 +404,7 @@ t_hash(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
t_hash_clinetid(Config) when is_list(Config) ->
t_hash_clientid(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
@ -528,14 +530,15 @@ last_message(ExpectedPayload, Pids, Timeout) ->
t_dispatch(Config) when is_list(Config) ->
ok = ensure_config(random),
Topic = <<"foo">>,
Group = <<"group1">>,
?assertEqual(
{error, no_subscribers},
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
),
emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
?assertEqual(
{ok, 1},
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
).
t_uncovered_func(Config) when is_list(Config) ->
@ -572,7 +575,7 @@ t_local(Config) when is_list(Config) ->
<<"sticky_group">> => sticky
},
Node = start_slave('local_shared_sub_testtesttest', 21999),
Node = start_slave('local_shared_sub_local_1', 21999),
ok = ensure_group_config(GroupConfig),
ok = ensure_group_config(Node, GroupConfig),
@ -625,7 +628,7 @@ t_remote(Config) when is_list(Config) ->
<<"sticky_group">> => sticky
},
Node = start_slave('remote_shared_sub_testtesttest', 21999),
Node = start_slave('remote_shared_sub_remote_1', 21999),
ok = ensure_group_config(GroupConfig),
ok = ensure_group_config(Node, GroupConfig),
@ -674,7 +677,7 @@ t_local_fallback(Config) when is_list(Config) ->
Topic = <<"local_foo/bar">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
Node = start_slave('local_fallback_shared_sub_test', 11888),
Node = start_slave('local_fallback_shared_sub_1', 11888),
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]),
{ok, _} = emqtt:connect(ConnPid1),
@ -991,37 +994,110 @@ t_session_kicked(Config) when is_list(Config) ->
?assertEqual([], collect_msgs(0)),
ok.
%% FIXME: currently doesn't work
%% t_different_groups_same_topic({init, Config}) ->
%% TestName = atom_to_binary(?FUNCTION_NAME),
%% ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
%% {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
%% {ok, _} = emqtt:connect(C),
%% [{client, C}, {clientid, ClientId} | Config];
%% t_different_groups_same_topic({'end', Config}) ->
%% C = ?config(client, Config),
%% emqtt:stop(C),
%% ok;
%% t_different_groups_same_topic(Config) when is_list(Config) ->
%% C = ?config(client, Config),
%% ClientId = ?config(clientid, Config),
%% %% Subscribe and unsubscribe to both $queue and $shared topics
%% Topic = <<"t/1">>,
%% SharedTopic0 = <<"$share/aa/", Topic/binary>>,
%% SharedTopic1 = <<"$share/bb/", Topic/binary>>,
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}),
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}),
-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS),
?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS}))
).
%% Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
%% emqx:publish(Message0),
%% ?assertMatch([ {publish, #{payload := <<"hi">>}}
%% , {publish, #{payload := <<"hi">>}}
%% ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}),
t_different_groups_same_topic({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
{ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config];
t_different_groups_same_topic({'end', Config}) ->
C = ?config(client, Config),
emqtt:stop(C),
ok;
t_different_groups_same_topic(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
GroupA = <<"aa">>,
GroupB = <<"bb">>,
Topic = <<"t/1">>,
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0),
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1),
SharedTopicGroupA = ?SHARE(GroupA, Topic),
?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2),
SharedTopicGroupB = ?SHARE(GroupB, Topic),
?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2),
%% ok.
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>),
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
ok.
t_different_groups_update_subopts({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
{ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config];
t_different_groups_update_subopts({'end', Config}) ->
C = ?config(client, Config),
emqtt:stop(C),
ok;
t_different_groups_update_subopts(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
Topic = <<"t/1">>,
GroupA = <<"aa">>,
GroupB = <<"bb">>,
SharedTopicGroupA = ?SHARE(GroupA, Topic),
SharedTopicGroupB = ?SHARE(GroupB, Topic),
Fun = fun(Group, QoS) ->
?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS),
?assertMatch(
#{qos := QoS},
emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic))
)
end,
[Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]],
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
ok.
t_queue_subscription({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
@ -1038,23 +1114,19 @@ t_queue_subscription({'end', Config}) ->
t_queue_subscription(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to both $queue and $shared topics
%% Subscribe and unsubscribe to both $queue share and $share/<group> with same topic
Topic = <<"t/1">>,
QueueTopic = <<"$queue/", Topic/binary>>,
SharedTopic = <<"$share/aa/", Topic/binary>>,
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}),
%% FIXME: we should actually see 2 routes, one for each group
%% ($queue and aa), but currently the latest subscription
%% overwrites the existing one.
?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2),
?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2),
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% FIXME: should ensure we have 2 subscriptions
[_] = emqx_router:lookup_routes(Topic)
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
@ -1063,37 +1135,29 @@ t_queue_subscription(Config) when is_list(Config) ->
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
%% FIXME: should receive one message from each group
%% , {publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000)
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic),
%% FIXME: return code should be success instead of 17 ("no_subscription_existed")
{ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic),
%% FIXME: this should eventually be true, but currently we leak
%% the previous group subscription...
%% ?retry(
%% _Sleep0 = 100,
%% _Attempts0 = 50,
%% begin
%% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% [] = emqx_router:lookup_routes(Topic)
%% end
%% ),
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(0, length(emqx_router:match_routes(Topic)))
end
),
ct:sleep(500),
Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>),
emqx:publish(Message1),
%% FIXME: we should *not* receive any messages...
%% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
%% This is from the leaked group...
?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{
routes => ets:tab2list(emqx_route)
}),
%% we should *not* receive any messages.
?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
ok.
@ -1190,34 +1254,24 @@ recv_msgs(Count, Msgs) ->
end.
start_slave(Name, Port) ->
{ok, Node} = ct_slave:start(
list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 10000},
{startup_timeout, 10000},
{erl_flags, ebin_path()}
]
{ok, Node} = emqx_cth_peer:start_link(
Name,
ebin_path()
),
pong = net_adm:ping(Node),
setup_node(Node, Port),
Node.
stop_slave(Node) ->
rpc:call(Node, mria, leave, []),
ct_slave:stop(Node).
emqx_cth_peer:stop(Node).
host() ->
[_, Host] = string:tokens(atom_to_list(node()), "@"),
Host.
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch.
["-pa" | code:get_path()].
setup_node(Node, Port) ->
EnvHandler =

12
apps/emqx/test/emqx_topic_SUITE.erl
View File

@ -238,11 +238,11 @@ long_topic() ->
t_parse(_) ->
?assertError(
{invalid_topic_filter, <<"$queue/t">>},
parse(<<"$queue/t">>, #{share => <<"g">>})
parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{})
),
?assertError(
{invalid_topic_filter, <<"$share/g/t">>},
parse(<<"$share/g/t">>, #{share => <<"g">>})
parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{})
),
?assertError(
{invalid_topic_filter, <<"$share/t">>},
@ -254,8 +254,12 @@ t_parse(_) ->
),
?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)),
?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})),
?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)),
?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)),
?assertEqual(
{#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>)
),
?assertEqual(
{#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>)
),
%% The '$local' and '$fastlane' topics have been deprecated.
?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)),
?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)),

3
apps/emqx/test/emqx_topic_index_SUITE.erl
View File

@ -209,9 +209,6 @@ t_match_fast_forward(Config) ->
M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab),
M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab),
M:insert(<<"a/b/c/+">>, id3, <<>>, Tab),
% dbg:tracer(),
% dbg:p(all, c),
% dbg:tpl({ets, next, '_'}, x),
?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))),
?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]).

94
apps/emqx_audit/BSL.txt Normal file
View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2023
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2027-02-01
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

5
apps/emqx_audit/README.md Normal file
View File

@ -0,0 +1,5 @@
emqx_audit
=====
Audit log for EMQX, empowers users to efficiently access the desired audit trail data
and facilitates auditing, compliance, troubleshooting, and security analysis.

26
apps/emqx_audit/include/emqx_audit.hrl Normal file
View File

@ -0,0 +1,26 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-define(AUDIT, emqx_audit).
-record(?AUDIT, {
%% basic info
created_at,
node,
from,
source,
source_ip,
%% operation info
operation_id,
operation_type,
args,
operation_result,
failure,
%% request detail
http_method,
http_request,
http_status_code,
duration_ms,
extra
}).

5
apps/emqx_audit/rebar.config Normal file
View File

@ -0,0 +1,5 @@
{erl_opts, [debug_info]}.
{deps, [
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}}
]}.

10
apps/emqx_audit/src/emqx_audit.app.src Normal file
View File

@ -0,0 +1,10 @@
{application, emqx_audit, [
{description, "Audit log for EMQX"},
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_audit_app, []}},
{applications, [kernel, stdlib, emqx]},
{env, []},
{modules, []},
{links, []}
]}.

245
apps/emqx_audit/src/emqx_audit.erl Normal file
View File

@ -0,0 +1,245 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit).
-behaviour(gen_server).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-include("emqx_audit.hrl").
%% API
-export([start_link/0]).
-export([log/3]).
-export([trans_clean_expired/2]).
%% gen_server callbacks
-export([
init/1,
handle_continue/2,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]).
-ifdef(TEST).
-define(INTERVAL, 100).
-else.
-define(INTERVAL, 10000).
-endif.
to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) ->
#?AUDIT{
operation_id = <<"">>,
operation_type = atom_to_binary(Cmd),
args = Args,
operation_result = <<"">>,
failure = <<"">>,
duration_ms = DurationMs,
from = cli,
source = <<"">>,
source_ip = <<"">>,
http_status_code = <<"">>,
http_method = <<"">>,
http_request = <<"">>
};
to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api ->
#{
source := Source,
source_ip := SourceIp,
%% operation info
operation_id := OperationId,
operation_type := OperationType,
operation_result := OperationResult,
%% request detail
http_status_code := StatusCode,
http_method := Method,
http_request := Request,
duration_ms := DurationMs
} = Log,
#?AUDIT{
from = From,
source = Source,
source_ip = SourceIp,
%% operation info
operation_id = OperationId,
operation_type = OperationType,
operation_result = OperationResult,
failure = maps:get(failure, Log, <<"">>),
%% request detail
http_status_code = StatusCode,
http_method = Method,
http_request = Request,
duration_ms = DurationMs,
args = <<"">>
};
to_audit(#{from := erlang_console, function := F, args := Args}) ->
#?AUDIT{
from = erlang_console,
source = <<"">>,
source_ip = <<"">>,
%% operation info
operation_id = <<"">>,
operation_type = <<"">>,
operation_result = <<"">>,
failure = <<"">>,
%% request detail
http_status_code = <<"">>,
http_method = <<"">>,
http_request = <<"">>,
duration_ms = 0,
args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args]))
}.
log(_Level, undefined, _Handler) ->
ok;
log(Level, Meta1, Handler) ->
Meta2 = Meta1#{time => logger:timestamp(), level => Level},
log_to_file(Level, Meta2, Handler),
log_to_db(Meta2),
remove_handler_when_disabled().
remove_handler_when_disabled() ->
case emqx_config:get([log, audit, enable], false) of
true ->
ok;
false ->
_ = logger:remove_handler(?AUDIT_HANDLER),
ok
end.
log_to_db(Log) ->
Audit0 = to_audit(Log),
Audit = Audit0#?AUDIT{
node = node(),
created_at = erlang:system_time(microsecond)
},
mria:dirty_write(?AUDIT, Audit).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init([]) ->
ok = mria:create_table(?AUDIT, [
{type, ordered_set},
{rlog_shard, ?COMMON_SHARD},
{storage, disc_copies},
{record_name, ?AUDIT},
{attributes, record_info(fields, ?AUDIT)}
]),
{ok, #{}, {continue, setup}}.
handle_continue(setup, State) ->
ok = mria:wait_for_tables([?AUDIT]),
NewState = State#{role => mria_rlog:role()},
?AUDIT(alert, #{
cmd => emqx,
args => ["start"],
version => emqx_release:version(),
from => cli,
duration_ms => 0
}),
{noreply, NewState, interval(NewState)}.
handle_call(_Request, _From, State) ->
{reply, ignore, State, interval(State)}.
handle_cast(_Request, State) ->
{noreply, State, interval(State)}.
handle_info(timeout, State) ->
ExtraWait = clean_expired_logs(),
{noreply, State, interval(State) + ExtraWait};
handle_info(_Info, State) ->
{noreply, State, interval(State)}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
%%%===================================================================
%%% Internal functions
%%%===================================================================
%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds.
clean_expired_logs() ->
MaxSize = max_size(),
Oldest = mnesia:dirty_first(?AUDIT),
CurSize = mnesia:table_info(?AUDIT, size),
case CurSize - MaxSize of
DelSize when DelSize > 0 ->
case
mria:transaction(
?COMMON_SHARD,
fun ?MODULE:trans_clean_expired/2,
[Oldest, DelSize]
)
of
{atomic, ok} ->
0;
{aborted, Reason} ->
?SLOG(error, #{
msg => "clean_expired_audit_aborted",
reason => Reason,
delete_size => DelSize,
current_size => CurSize,
max_count => MaxSize
}),
60000
end;
_ ->
0
end.
trans_clean_expired(Oldest, DelCount) ->
First = mnesia:first(?AUDIT),
%% Other node already clean from the oldest record.
%% ensure not delete twice, otherwise records that should not be deleted will be deleted.
case First =:= Oldest of
true -> do_clean_expired(First, DelCount);
false -> ok
end.
do_clean_expired(_, DelSize) when DelSize =< 0 -> ok;
do_clean_expired('$end_of_table', _DelSize) ->
ok;
do_clean_expired(CurKey, DeleteSize) ->
mnesia:delete(?AUDIT, CurKey, sticky_write),
do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1).
max_size() ->
emqx_conf:get([log, audit, max_filter_size], 5000).
interval(#{role := replicant}) -> hibernate;
interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL).
log_to_file(Level, Meta, #{module := Module} = Handler) ->
Log = #{level => Level, meta => Meta, msg => undefined},
Handler1 = maps:without(?OWN_KEYS, Handler),
try
erlang:apply(Module, log, [Log, Handler1])
catch
C:R:S ->
case logger:remove_handler(?AUDIT_HANDLER) of
ok ->
logger:internal_log(
error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S}
);
{error, {not_found, _}} ->
ok;
{error, Reason} ->
logger:internal_log(
error,
{removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S}
)
end
end.

398
apps/emqx_audit/src/emqx_audit_api.erl Normal file
View File

@ -0,0 +1,398 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_api).
-behaviour(minirest_api).
%% API
-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]).
-export([audit/2]).
-export([qs2ms/2, format/1]).
-include_lib("emqx/include/logger.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-include("emqx_audit.hrl").
-import(hoconsc, [mk/2, ref/2, array/1]).
-define(TAGS, ["Audit"]).
-define(AUDIT_QS_SCHEMA, [
{<<"node">>, atom},
{<<"from">>, atom},
{<<"source">>, binary},
{<<"source_ip">>, binary},
{<<"operation_id">>, binary},
{<<"operation_type">>, binary},
{<<"operation_result">>, atom},
{<<"http_status_code">>, integer},
{<<"http_method">>, atom},
{<<"gte_created_at">>, timestamp},
{<<"lte_created_at">>, timestamp},
{<<"gte_duration_ms">>, timestamp},
{<<"lte_duration_ms">>, timestamp}
]).
-define(DISABLE_MSG, <<"Audit is disabled">>).
namespace() -> "audit".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
["/audit"].
schema("/audit") ->
#{
'operationId' => audit,
get => #{
tags => ?TAGS,
description => ?DESC(audit_get),
parameters => [
{node,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"emqx@127.0.0.1">>,
desc => ?DESC(filter_node)
})},
{from,
?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
in => query,
required => false,
example => <<"dashboard">>,
desc => ?DESC(filter_from)
})},
{source,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"admin">>,
desc => ?DESC(filter_source)
})},
{source_ip,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"127.0.0.1">>,
desc => ?DESC(filter_source_ip)
})},
{operation_id,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"/rules/{id}">>,
desc => ?DESC(filter_operation_id)
})},
{operation_type,
?HOCON(binary(), #{
in => query,
example => <<"rules">>,
required => false,
desc => ?DESC(filter_operation_type)
})},
{operation_result,
?HOCON(?ENUM([success, failure]), #{
in => query,
example => failure,
required => false,
desc => ?DESC(filter_operation_result)
})},
{http_status_code,
?HOCON(integer(), #{
in => query,
example => 200,
required => false,
desc => ?DESC(filter_http_status_code)
})},
{http_method,
?HOCON(?ENUM([post, put, delete]), #{
in => query,
example => post,
required => false,
desc => ?DESC(filter_http_method)
})},
{gte_duration_ms,
?HOCON(integer(), #{
in => query,
example => 0,
required => false,
desc => ?DESC(filter_gte_duration_ms)
})},
{lte_duration_ms,
?HOCON(integer(), #{
in => query,
example => 1000,
required => false,
desc => ?DESC(filter_lte_duration_ms)
})},
{gte_created_at,
?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
in => query,
required => false,
example => <<"2023-10-15T00:00:00.820384+08:00">>,
desc => ?DESC(filter_gte_created_at)
})},
{lte_created_at,
?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
in => query,
example => <<"2023-10-16T00:00:00.820384+08:00">>,
required => false,
desc => ?DESC(filter_lte_created_at)
})},
ref(emqx_dashboard_swagger, page),
ref(emqx_dashboard_swagger, limit)
],
summary => <<"List audit logs">>,
responses => #{
200 =>
emqx_dashboard_swagger:schema_with_example(
array(?REF(audit_list)),
audit_log_list_example()
),
400 => emqx_dashboard_swagger:error_codes(
['BAD_REQUEST'],
?DISABLE_MSG
)
}
}
}.
fields(audit_list) ->
[
{data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})},
{meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
];
fields(audit) ->
[
{created_at,
?HOCON(
emqx_utils_calendar:epoch_millisecond(),
#{
desc => "The time when the log is created"
}
)},
{node,
?HOCON(binary(), #{
desc => "The node name to which the log is created"
})},
{from,
?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
desc => "The source type of the log"
})},
{source,
?HOCON(binary(), #{
desc => "The source of the log"
})},
{source_ip,
?HOCON(binary(), #{
desc => "The source ip of the log"
})},
{operation_id,
?HOCON(binary(), #{
desc => "The operation id of the log"
})},
{operation_type,
?HOCON(binary(), #{
desc => "The operation type of the log"
})},
{operation_result,
?HOCON(?ENUM([success, failure]), #{
desc => "The operation result of the log"
})},
{http_status_code,
?HOCON(integer(), #{
desc => "The http status code of the log"
})},
{http_method,
?HOCON(?ENUM([post, put, delete]), #{
desc => "The http method of the log"
})},
{duration_ms,
?HOCON(integer(), #{
desc => "The duration of the log"
})},
{args,
?HOCON(?ARRAY(binary()), #{
desc => "The args of the log"
})},
{failure,
?HOCON(?ARRAY(binary()), #{
desc => "The failure of the log"
})},
{http_request,
?HOCON(?REF(http_request), #{
desc => "The http request of the log"
})}
];
fields(http_request) ->
[
{bindings, ?HOCON(map(), #{})},
{body, ?HOCON(map(), #{})},
{headers, ?HOCON(map(), #{})},
{method, ?HOCON(?ENUM([post, put, delete]), #{})}
].
audit(get, #{query_string := QueryString}) ->
case emqx_config:get([log, audit, enable], false) of
false ->
{400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}};
true ->
case
emqx_mgmt_api:node_query(
node(),
?AUDIT,
QueryString,
?AUDIT_QS_SCHEMA,
fun ?MODULE:qs2ms/2,
fun ?MODULE:format/1
)
of
{error, page_limit_invalid} ->
{400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
{error, Node, Error} ->
Message = list_to_binary(
io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])
),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Result ->
{200, Result}
end
end.
qs2ms(_Tab, {Qs, _}) ->
#{
match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []),
fuzzy_fun => undefined
}.
gen_match_spec([], Audit, Conn) ->
[{Audit, Conn, ['$_']}];
gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn);
gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn);
gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn);
gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn);
gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn);
gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn);
gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn);
gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn);
gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn);
gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]);
gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [
{'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn
]);
gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]);
gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [
{'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn
]).
format(Audit) ->
#?AUDIT{
created_at = CreatedAt,
node = Node,
from = From,
source = Source,
source_ip = SourceIp,
operation_id = OperationId,
operation_type = OperationType,
operation_result = OperationResult,
http_status_code = HttpStatusCode,
http_method = HttpMethod,
duration_ms = DurationMs,
args = Args,
failure = Failure,
http_request = HttpRequest
} = Audit,
#{
created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond),
node => Node,
from => From,
source => Source,
source_ip => SourceIp,
operation_id => OperationId,
operation_type => OperationType,
operation_result => OperationResult,
http_status_code => HttpStatusCode,
http_method => HttpMethod,
duration_ms => DurationMs,
args => Args,
failure => Failure,
http_request => HttpRequest
}.
audit_log_list_example() ->
#{
data => [api_example(), cli_example()],
meta => #{
<<"count">> => 2,
<<"hasnext">> => false,
<<"limit">> => 50,
<<"page">> => 1
}
}.
api_example() ->
#{
<<"args">> => "",
<<"created_at">> => "2023-10-17T10:41:20.383993+08:00",
<<"duration_ms">> => 0,
<<"failure">> => "",
<<"from">> => "dashboard",
<<"http_method">> => "post",
<<"http_request">> => #{
<<"bindings">> => #{},
<<"body">> => #{
<<"password">> => "******",
<<"username">> => "admin"
},
<<"headers">> => #{
<<"accept">> => "*/*",
<<"authorization">> => "******",
<<"connection">> => "keep-alive",
<<"content-length">> => "45",
<<"content-type">> => "application/json"
},
<<"method">> => "post"
},
<<"http_status_code">> => 200,
<<"node">> => "emqx@127.0.0.1",
<<"operation_id">> => "/login",
<<"operation_result">> => "success",
<<"operation_type">> => "login",
<<"source">> => "admin",
<<"source_ip">> => "127.0.0.1"
}.
cli_example() ->
#{
<<"args">> => [<<"show">>, <<"log">>],
<<"created_at">> => "2023-10-17T10:45:13.100426+08:00",
<<"duration_ms">> => 7,
<<"failure">> => "",
<<"from">> => "cli",
<<"http_method">> => "",
<<"http_request">> => "",
<<"http_status_code">> => "",
<<"node">> => "emqx@127.0.0.1",
<<"operation_id">> => "",
<<"operation_result">> => "",
<<"operation_type">> => "conf",
<<"source">> => "",
<<"source_ip">> => ""
}.

15
apps/emqx_audit/src/emqx_audit_app.erl Normal file
View File

@ -0,0 +1,15 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_app).
-behaviour(application).
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
emqx_audit_sup:start_link().
stop(_State) ->
ok.

33
apps/emqx_audit/src/emqx_audit_sup.erl Normal file
View File

@ -0,0 +1,33 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_sup).
-behaviour(supervisor).
-export([start_link/0]).
-export([init/1]).
-define(SERVER, ?MODULE).
start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) ->
SupFlags = #{
strategy => one_for_all,
intensity => 10,
period => 10
},
ChildSpecs = [
#{
id => emqx_audit,
start => {emqx_audit, start_link, []},
type => worker,
restart => transient,
shutdown => 1000
}
],
{ok, {SupFlags, ChildSpecs}}.

248
apps/emqx_audit/test/emqx_audit_api_SUITE.erl Normal file
View File

@ -0,0 +1,248 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_audit_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
all() ->
[
{group, audit, [sequence]}
].
groups() ->
[
{audit, [sequence], common_tests()}
].
common_tests() ->
emqx_common_test_helpers:all(?MODULE).
-define(CONF_DEFAULT, #{
node =>
#{
name => "emqx1@127.0.0.1",
cookie => "emqxsecretcookie",
data_dir => "data"
},
log => #{
audit =>
#{
enable => true,
ignore_high_frequency_request => true,
level => info,
max_filter_size => 15,
rotation_count => 2,
rotation_size => "10MB",
time_offset => "system"
}
}
}).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_config:erase_all(),
emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
ok = emqx_config_logger:refresh_config(),
application:set_env(emqx, boot_modules, []),
emqx_conf_cli:load(),
Config.
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
t_http_api(_) ->
process_flag(trap_exit, true),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
{ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
?assertMatch(#{<<"max_qos_allowed">> := 1}, Res),
{ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
?assertMatch(
#{
<<"data">> := [
#{
<<"from">> := <<"rest_api">>,
<<"operation_id">> := <<"/configs/global_zone">>,
<<"source_ip">> := <<"127.0.0.1">>,
<<"source">> := _,
<<"http_request">> := #{
<<"method">> := <<"put">>,
<<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}},
<<"bindings">> := _,
<<"headers">> := #{<<"authorization">> := <<"******">>}
},
<<"http_status_code">> := 200,
<<"operation_result">> := <<"success">>,
<<"operation_type">> := <<"configs">>
}
]
},
emqx_utils_json:decode(Res1, [return_maps])
),
ok.
t_disabled(_) ->
Enable = [log, audit, enable],
?assertEqual(true, emqx:get_config(Enable)),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
Size1 = mnesia:table_info(emqx_audit, size),
{ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"),
Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100),
NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false),
{ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs),
{ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"),
?assertEqual(NewLogs, GetLog1),
?assertMatch(
{error, _},
emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader)
),
Size2 = mnesia:table_info(emqx_audit, size),
%% Record the audit disable action, so the size + 1
?assertEqual(Size1 + 1, Size2),
{ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111),
{ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
?assertMatch(#{<<"max_topic_levels">> := 111}, Res),
Size3 = mnesia:table_info(emqx_audit, size),
%% Don't record mqtt update request.
?assertEqual(Size2, Size3),
%% enabled again
{ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1),
{ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"),
?assertEqual(Logs1, GetLog2),
Size4 = mnesia:table_info(emqx_audit, size),
?assertEqual(Size3 + 1, Size4),
ok.
t_cli(_Config) ->
ok = emqx_ctl:run_command(["conf", "show", "log"]),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
#{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
?assertMatch(
[
#{
<<"from">> := <<"cli">>,
<<"operation_id">> := <<"">>,
<<"source_ip">> := <<"">>,
<<"operation_type">> := <<"conf">>,
<<"args">> := [<<"show">>, <<"log">>],
<<"node">> := _,
<<"source">> := <<"">>,
<<"http_request">> := <<"">>
}
],
Data
),
%% check filter
{ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
#{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
?assertEqual(Data, Data1),
{ok, Res2} = emqx_mgmt_api_test_util:request_api(
get, AuditPath, "from=erlang_console", AuthHeader
),
?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])),
ok.
t_max_size(_Config) ->
{ok, _} = emqx:update_config([log, audit, max_filter_size], 1000),
SizeFun =
fun() ->
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Limit = "limit=1000",
{ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader),
#{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
erlang:length(Data)
end,
InitSize = SizeFun(),
lists:foreach(
fun(_) ->
ok = emqx_ctl:run_command(["conf", "show", "log"])
end,
lists:duplicate(100, 1)
),
timer:sleep(110),
Size1 = SizeFun(),
?assert(Size1 - InitSize >= 100, {Size1, InitSize}),
{ok, _} = emqx:update_config([log, audit, max_filter_size], 10),
%% wait for clean_expired
timer:sleep(250),
ExpectSize = emqx:get_config([log, audit, max_filter_size]),
Size2 = SizeFun(),
?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}),
ok.
t_kickout_clients_without_log(_) ->
process_flag(trap_exit, true),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
{ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
kickout_clients(),
{ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
?assertEqual(AuditLogs1, AuditLogs2),
ok.
kickout_clients() ->
ClientId1 = <<"client1">>,
ClientId2 = <<"client2">>,
ClientId3 = <<"client3">>,
{ok, C1} = emqtt:start_link(#{
clientid => ClientId1,
proto_ver => v5,
properties => #{'Session-Expiry-Interval' => 120}
}),
{ok, _} = emqtt:connect(C1),
{ok, C2} = emqtt:start_link(#{clientid => ClientId2}),
{ok, _} = emqtt:connect(C2),
{ok, C3} = emqtt:start_link(#{clientid => ClientId3}),
{ok, _} = emqtt:connect(C3),
timer:sleep(300),
%% get /clients
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
{ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]),
ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
ClientsPage = maps:get(<<"page">>, ClientsMeta),
ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
ClientsCount = maps:get(<<"count">>, ClientsMeta),
?assertEqual(ClientsPage, 1),
?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()),
?assertEqual(ClientsCount, 3),
%% kickout clients
KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]),
KickoutBody = [ClientId1, ClientId2, ClientId3],
{ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody),
{ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]),
?assertMatch(#{<<"data">> := []}, ClientsResponse2).

4
apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl
View File

@ -26,10 +26,6 @@
-define(AUTHN_BACKEND, ldap).
-define(AUTHN_BACKEND_BIN, <<"ldap">>).
-define(AUTHN_BACKEND_BIND, ldap_bind).
-define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>).
-define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}).
-define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}).
-endif.

2
apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth_ldap, [
{description, "EMQX LDAP Authentication and Authorization"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auth_ldap_app, []}},
{applications, [

2
apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl
View File

@ -25,12 +25,10 @@
start(_StartType, _StartArgs) ->
ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap),
ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap),
ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind),
{ok, Sup} = emqx_auth_ldap_sup:start_link(),
{ok, Sup}.
stop(_State) ->
ok = emqx_authn:deregister_provider(?AUTHN_TYPE),
ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND),
ok = emqx_authz:unregister_source(?AUTHZ_TYPE),
ok.

185
apps/emqx_auth_ldap/src/emqx_authn_ldap.erl
View File

@ -16,19 +16,10 @@
-module(emqx_authn_ldap).
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
-behaviour(emqx_authn_provider).
%% a compatible attribute for version 4.x
-define(ISENABLED_ATTR, "isEnabled").
-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
%% TODO
%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-export([
create/2,
update/2,
@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) ->
ignore;
authenticate(#{password := undefined}, _) ->
{error, bad_username_or_password};
authenticate(
#{password := Password} = Credential,
#{
password_attribute := PasswordAttr,
is_superuser_attribute := IsSuperuserAttr,
query_timeout := Timeout,
resource_id := ResourceId
} = State
) ->
case
emqx_resource:simple_sync_query(
ResourceId,
{query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
)
of
{ok, []} ->
ignore;
{ok, [Entry]} ->
is_enabled(Password, Entry, State);
{error, Reason} ->
?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
resource => ResourceId,
timeout => Timeout,
reason => Reason
}),
ignore
authenticate(Credential, #{method := #{type := Type}} = State) ->
case Type of
hash ->
emqx_authn_ldap_hash:authenticate(Credential, State);
bind ->
emqx_authn_ldap_bind:authenticate(Credential, State)
end.
%% it used the deprecated config form
parse_config(
#{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0
) ->
Config = maps:without([password_attribute, is_superuser_attribute], Config0),
parse_config(Config#{
method => #{
type => hash,
password_attribute => PasswordAttr,
is_superuser_attribute => IsSuperuserAttr
}
});
parse_config(Config) ->
maps:with([query_timeout, password_attribute, is_superuser_attribute], Config).
%% To compatible v4.x
is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
case emqx_authn_utils:to_bool(IsEnabled) of
true ->
ensure_password(Password, Entry, State);
_ ->
{error, user_disabled}
end.
ensure_password(
Password,
#eldap_entry{attributes = Attributes} = Entry,
#{password_attribute := PasswordAttr} = State
) ->
case get_value(PasswordAttr, Attributes) of
undefined ->
{error, no_password};
[LDAPPassword | _] ->
extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
end.
%% RFC 2307 format password
%% https://datatracker.ietf.org/doc/html/rfc2307
extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
case
re:run(
LDAPPassword,
"{([^{}]+)}(.+)",
[{capture, all_but_first, list}, global]
)
of
{match, [[HashTypeStr, PasswordHashStr]]} ->
case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
{ok, HashType} ->
PasswordHash = to_binary(PasswordHashStr),
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
_Error ->
{error, invalid_hash_type}
end;
_ ->
OnFail(LDAPPassword, Password, Entry, State)
end.
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
case lists:member(HashType, ?VALID_ALGORITHMS) of
true ->
verify_password(HashType, PasswordHash, Password, Entry, State);
_ ->
{error, {invalid_hash_type, HashType}}
end.
%% this password is in LDIF format which is base64 encoding
try_decode_password(LDAPPassword, Password, Entry, State) ->
case safe_base64_decode(LDAPPassword) of
{ok, Decode} ->
extract_hash_algorithm(
Decode,
Password,
fun(_, _, _, _) ->
{error, invalid_password}
end,
Entry,
State
);
{error, Reason} ->
{error, {invalid_password, Reason}}
end.
%% sha with salt
%% https://www.openldap.org/faq/data/cache/347.html
verify_password(ssha, PasswordData, Password, Entry, State) ->
case safe_base64_decode(PasswordData) of
{ok, <<PasswordHash:20/binary, Salt/binary>>} ->
verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
{ok, _} ->
{error, invalid_ssha_password};
{error, Reason} ->
{error, {invalid_password, Reason}}
end;
verify_password(
Algorithm,
Base64HashData,
Password,
Entry,
State
) ->
verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
PasswordHash = hash_password(Algorithm, Salt, Position, Password),
case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
true ->
{ok, is_superuser(Entry, State)};
_ ->
{error, bad_username_or_password}
end.
is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) ->
Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
#{is_superuser => emqx_authn_utils:to_bool(Value)}.
safe_base64_decode(Data) ->
try
{ok, base64:decode(Data)}
catch
_:Reason ->
{error, {invalid_base64_data, Reason}}
end.
get_lower_bin_value(Key, Proplists, Default) ->
[Value | _] = get_value(Key, Proplists, [Default]),
to_binary(string:to_lower(Value)).
to_binary(Value) ->
erlang:list_to_binary(Value).
hash_password(Algorithm, _Salt, disable, Password) ->
hash_password(Algorithm, Password);
hash_password(Algorithm, Salt, suffix, Password) ->
hash_password(Algorithm, <<Password/binary, Salt/binary>>).
hash_password(Algorithm, Data) ->
crypto:hash(Algorithm, Data).
compare_password(hash, LDAPPasswordHash, PasswordHash) ->
emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
compare_password(base64, Base64HashData, PasswordHash) ->
emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).
maps:with([query_timeout, method], Config).

21
apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl
View File

@ -20,32 +20,13 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
-behaviour(emqx_authn_provider).
-export([
create/2,
update/2,
authenticate/2,
destroy/1
authenticate/2
]).
%%------------------------------------------------------------------------------
%% APIs
%%------------------------------------------------------------------------------
create(_AuthenticatorID, Config) ->
emqx_authn_ldap:do_create(?MODULE, Config).
update(Config, State) ->
emqx_authn_ldap:update(Config, State).
destroy(State) ->
emqx_authn_ldap:destroy(State).
authenticate(#{auth_method := _}, _) ->
ignore;
authenticate(#{password := undefined}, _) ->
{error, bad_username_or_password};
authenticate(
#{password := _Password} = Credential,
#{

66
apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl
View File

@ -1,66 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_ldap_bind_schema).
-behaviour(emqx_authn_schema).
-export([
fields/1,
desc/1,
refs/0,
select_union_member/1,
namespace/0
]).
-include("emqx_auth_ldap.hrl").
-include_lib("hocon/include/hoconsc.hrl").
namespace() -> "authn".
refs() ->
[?R_REF(ldap_bind)].
select_union_member(#{
<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN
}) ->
refs();
select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) ->
throw(#{
reason => "unknown_mechanism",
expected => ?AUTHN_MECHANISM
});
select_union_member(_) ->
undefined.
fields(ldap_bind) ->
[
{mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
{backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)},
{query_timeout, fun query_timeout/1}
] ++
emqx_authn_schema:common_fields() ++
emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts).
desc(ldap_bind) ->
?DESC(ldap_bind);
desc(_) ->
undefined.
query_timeout(type) -> emqx_schema:timeout_duration_ms();
query_timeout(desc) -> ?DESC(?FUNCTION_NAME);
query_timeout(default) -> <<"5s">>;
query_timeout(_) -> undefined.

197
apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl Normal file
View File

@ -0,0 +1,197 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_ldap_hash).
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
%% a compatible attribute for version 4.x
-define(ISENABLED_ATTR, "isEnabled").
-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
%% TODO
%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-export([
authenticate/2
]).
-import(proplists, [get_value/2, get_value/3]).
%%------------------------------------------------------------------------------
%% APIs
%%------------------------------------------------------------------------------
authenticate(
#{password := Password} = Credential,
#{
method := #{
password_attribute := PasswordAttr,
is_superuser_attribute := IsSuperuserAttr
},
query_timeout := Timeout,
resource_id := ResourceId
} = State
) ->
case
emqx_resource:simple_sync_query(
ResourceId,
{query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
)
of
{ok, []} ->
ignore;
{ok, [Entry]} ->
is_enabled(Password, Entry, State);
{error, Reason} ->
?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
resource => ResourceId,
timeout => Timeout,
reason => Reason
}),
ignore
end.
%% To compatible v4.x
is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
case emqx_authn_utils:to_bool(IsEnabled) of
true ->
ensure_password(Password, Entry, State);
_ ->
{error, user_disabled}
end.
ensure_password(
Password,
#eldap_entry{attributes = Attributes} = Entry,
#{method := #{password_attribute := PasswordAttr}} = State
) ->
case get_value(PasswordAttr, Attributes) of
undefined ->
{error, no_password};
[LDAPPassword | _] ->
extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
end.
%% RFC 2307 format password
%% https://datatracker.ietf.org/doc/html/rfc2307
extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
case
re:run(
LDAPPassword,
"{([^{}]+)}(.+)",
[{capture, all_but_first, list}, global]
)
of
{match, [[HashTypeStr, PasswordHashStr]]} ->
case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
{ok, HashType} ->
PasswordHash = to_binary(PasswordHashStr),
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
_Error ->
{error, invalid_hash_type}
end;
_ ->
OnFail(LDAPPassword, Password, Entry, State)
end.
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
case lists:member(HashType, ?VALID_ALGORITHMS) of
true ->
verify_password(HashType, PasswordHash, Password, Entry, State);
_ ->
{error, {invalid_hash_type, HashType}}
end.
%% this password is in LDIF format which is base64 encoding
try_decode_password(LDAPPassword, Password, Entry, State) ->
case safe_base64_decode(LDAPPassword) of
{ok, Decode} ->
extract_hash_algorithm(
Decode,
Password,
fun(_, _, _, _) ->
{error, invalid_password}
end,
Entry,
State
);
{error, Reason} ->
{error, {invalid_password, Reason}}
end.
%% sha with salt
%% https://www.openldap.org/faq/data/cache/347.html
verify_password(ssha, PasswordData, Password, Entry, State) ->
case safe_base64_decode(PasswordData) of
{ok, <<PasswordHash:20/binary, Salt/binary>>} ->
verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
{ok, _} ->
{error, invalid_ssha_password};
{error, Reason} ->
{error, {invalid_password, Reason}}
end;
verify_password(
Algorithm,
Base64HashData,
Password,
Entry,
State
) ->
verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
PasswordHash = hash_password(Algorithm, Salt, Position, Password),
case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
true ->
{ok, is_superuser(Entry, State)};
_ ->
{error, bad_username_or_password}
end.
is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) ->
Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
#{is_superuser => emqx_authn_utils:to_bool(Value)}.
safe_base64_decode(Data) ->
try
{ok, base64:decode(Data)}
catch
_:Reason ->
{error, {invalid_base64_data, Reason}}
end.
get_lower_bin_value(Key, Proplists, Default) ->
[Value | _] = get_value(Key, Proplists, [Default]),
to_binary(string:to_lower(Value)).
to_binary(Value) ->
erlang:list_to_binary(Value).
hash_password(Algorithm, _Salt, disable, Password) ->
hash_password(Algorithm, Password);
hash_password(Algorithm, Salt, suffix, Password) ->
hash_password(Algorithm, <<Password/binary, Salt/binary>>).
hash_password(Algorithm, Data) ->
crypto:hash(Algorithm, Data).
compare_password(hash, LDAPPasswordHash, PasswordHash) ->
emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
compare_password(base64, Base64HashData, PasswordHash) ->
emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

61
apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl
View File

@ -32,7 +32,7 @@
namespace() -> "authn".
refs() ->
[?R_REF(ldap)].
[?R_REF(ldap), ?R_REF(ldap_deprecated)].
select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) ->
refs();
@ -44,12 +44,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) ->
select_union_member(_) ->
undefined.
fields(ldap_deprecated) ->
common_fields() ++
[
{password_attribute, password_attribute()},
{is_superuser_attribute, is_superuser_attribute()}
];
fields(ldap) ->
common_fields() ++
[
{method,
?HOCON(
hoconsc:union([?R_REF(hash_method), ?R_REF(bind_method)]),
#{desc => ?DESC(method)}
)}
];
fields(hash_method) ->
[
{type, method_type(hash)},
{password_attribute, password_attribute()},
{is_superuser_attribute, is_superuser_attribute()}
];
fields(bind_method) ->
[{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts).
common_fields() ->
[
{mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
{backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
{password_attribute, fun password_attribute/1},
{is_superuser_attribute, fun is_superuser_attribute/1},
{query_timeout, fun query_timeout/1}
] ++
emqx_authn_schema:common_fields() ++
@ -57,18 +79,35 @@ fields(ldap) ->
desc(ldap) ->
?DESC(ldap);
desc(ldap_deprecated) ->
?DESC(ldap_deprecated);
desc(hash_method) ->
?DESC(hash_method);
desc(bind_method) ->
?DESC(bind_method);
desc(_) ->
undefined.
password_attribute(type) -> string();
password_attribute(desc) -> ?DESC(?FUNCTION_NAME);
password_attribute(default) -> <<"userPassword">>;
password_attribute(_) -> undefined.
method_type(Type) ->
?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}).
is_superuser_attribute(type) -> string();
is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME);
is_superuser_attribute(default) -> <<"isSuperuser">>;
is_superuser_attribute(_) -> undefined.
password_attribute() ->
?HOCON(
string(),
#{
desc => ?DESC(?FUNCTION_NAME),
default => <<"userPassword">>
}
).
is_superuser_attribute() ->
?HOCON(
string(),
#{
desc => ?DESC(?FUNCTION_NAME),
default => <<"isSuperuser">>
}
).
query_timeout(type) -> emqx_schema:timeout_duration_ms();
query_timeout(desc) -> ?DESC(?FUNCTION_NAME);

36
apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl
View File

@ -70,6 +70,29 @@ end_per_suite(Config) ->
%% Tests
%%------------------------------------------------------------------------------
t_create_with_deprecated_cfg(_Config) ->
AuthConfig = deprecated_raw_ldap_auth_config(),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(
?GLOBAL
),
?assertMatch(
#{
method := #{
type := hash,
is_superuser_attribute := _,
password_attribute := "not_the_default_value"
}
},
State
),
emqx_authn_test_lib:delete_config(?ResourceID).
t_create(_Config) ->
AuthConfig = raw_ldap_auth_config(),
@ -225,6 +248,19 @@ raw_ldap_auth_config() ->
<<"pool_size">> => 8
}.
deprecated_raw_ldap_auth_config() ->
#{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap">>,
<<"server">> => ldap_server(),
<<"is_superuser_attribute">> => <<"isSuperuser">>,
<<"password_attribute">> => <<"not_the_default_value">>,
<<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>,
<<"pool_size">> => 8
}.
user_seeds() ->
New = fun(Username, Password, Result) ->
#{

17
apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl
View File

@ -27,7 +27,7 @@
-define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>).
-define(PATH, [authentication]).
-define(ResourceID, <<"password_based:ldap_bind">>).
-define(ResourceID, <<"password_based:ldap">>).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -78,7 +78,7 @@ t_create(_Config) ->
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
{ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
emqx_authn_test_lib:delete_config(?ResourceID).
t_create_invalid(_Config) ->
@ -146,10 +146,10 @@ t_destroy(_Config) ->
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap_bind, state := State}]} =
{ok, [#{provider := emqx_authn_ldap, state := State}]} =
emqx_authn_chains:list_authenticators(?GLOBAL),
{ok, _} = emqx_authn_ldap_bind:authenticate(
{ok, _} = emqx_authn_ldap:authenticate(
#{
username => <<"mqttuser0001">>,
password => <<"mqttuser0001">>
@ -165,7 +165,7 @@ t_destroy(_Config) ->
% Authenticator should not be usable anymore
?assertMatch(
ignore,
emqx_authn_ldap_bind:authenticate(
emqx_authn_ldap:authenticate(
#{
username => <<"mqttuser0001">>,
password => <<"mqttuser0001">>
@ -199,7 +199,7 @@ t_update(_Config) ->
% We update with config with correct query, provider should update and work properly
{ok, _} = emqx:update_config(
?PATH,
{update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig}
{update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig}
),
{ok, _} = emqx_access_control:authenticate(
@ -218,14 +218,17 @@ t_update(_Config) ->
raw_ldap_auth_config() ->
#{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap_bind">>,
<<"backend">> => <<"ldap">>,
<<"server">> => ldap_server(),
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(uid=${username})">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>,
<<"pool_size">> => 8,
<<"method">> => #{
<<"type">> => <<"bind">>,
<<"bind_password">> => <<"${password}">>
}
}.
user_seeds() ->

16
apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl
View File

@ -278,6 +278,10 @@ raw_mongo_auth_config() ->
<<"server">> => mongo_server(),
<<"w_mode">> => <<"unsafe">>,
<<"auth_source">> => mongo_authsource(),
<<"username">> => mongo_username(),
<<"password">> => mongo_password(),
<<"filter">> => #{<<"username">> => <<"${username}">>},
<<"password_hash_field">> => <<"password_hash">>,
<<"salt_field">> => <<"salt">>,
@ -464,9 +468,21 @@ mongo_config() ->
{database, <<"mqtt">>},
{host, ?MONGO_HOST},
{port, ?MONGO_DEFAULT_PORT},
{auth_source, mongo_authsource()},
{login, mongo_username()},
{password, mongo_password()},
{register, ?MONGO_CLIENT}
].
mongo_authsource() ->
iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
mongo_username() ->
iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
mongo_password() ->
iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).

16
apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl
View File

@ -397,6 +397,10 @@ raw_mongo_authz_config() ->
<<"collection">> => <<"acl">>,
<<"server">> => mongo_server(),
<<"auth_source">> => mongo_authsource(),
<<"username">> => mongo_username(),
<<"password">> => mongo_password(),
<<"filter">> => #{<<"username">> => <<"${username}">>}
}.
@ -408,9 +412,21 @@ mongo_config() ->
{database, <<"mqtt">>},
{host, ?MONGO_HOST},
{port, ?MONGO_DEFAULT_PORT},
{auth_source, mongo_authsource()},
{login, mongo_username()},
{password, mongo_password()},
{register, ?MONGO_CLIENT}
].
mongo_authsource() ->
iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
mongo_username() ->
iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
mongo_password() ->
iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).

2
apps/emqx_auth_redis/src/emqx_auth_redis.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth_redis, [
{description, "EMQX Redis Authentication and Authorization"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auth_redis_app, []}},
{applications, [

8
apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl
View File

@ -64,12 +64,8 @@ refs(_) ->
expected => "single | cluster | sentinel"
}).
fields(redis_single) ->
common_fields() ++ emqx_redis:fields(single);
fields(redis_cluster) ->
common_fields() ++ emqx_redis:fields(cluster);
fields(redis_sentinel) ->
common_fields() ++ emqx_redis:fields(sentinel).
fields(Type) ->
common_fields() ++ emqx_redis:fields(Type).
desc(redis_single) ->
?DESC(single);

12
apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl
View File

@ -34,17 +34,9 @@ namespace() -> "authz".
type() -> ?AUTHZ_TYPE.
fields(redis_single) ->
fields(Type) ->
emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
emqx_redis:fields(single) ++
[{cmd, cmd()}];
fields(redis_sentinel) ->
emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
emqx_redis:fields(sentinel) ++
[{cmd, cmd()}];
fields(redis_cluster) ->
emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
emqx_redis:fields(cluster) ++
emqx_redis:fields(Type) ++
[{cmd, cmd()}].
desc(redis_single) ->

108
apps/emqx_bridge/src/emqx_action_info.erl
View File

@ -22,7 +22,7 @@
-export([
action_type_to_connector_type/1,
action_type_to_bridge_v1_type/1,
action_type_to_bridge_v1_type/2,
bridge_v1_type_to_action_type/1,
is_action_type/1,
registered_schema_modules/0,
@ -35,7 +35,12 @@
transform_bridge_v1_config_to_action_config/4
]).
-callback bridge_v1_type_name() -> atom().
-callback bridge_v1_type_name() ->
atom()
| {
fun(({ActionConfig :: map(), ConnectorConfig :: map()}) -> Type :: atom()),
TypeList :: [atom()]
}.
-callback action_type_name() -> atom().
-callback connector_type_name() -> atom().
-callback schema_module() -> atom().
@ -68,8 +73,16 @@
-if(?EMQX_RELEASE_EDITION == ee).
hard_coded_action_info_modules_ee() ->
[
emqx_bridge_azure_event_hub_action_info,
emqx_bridge_confluent_producer_action_info,
emqx_bridge_gcp_pubsub_producer_action_info,
emqx_bridge_kafka_action_info,
emqx_bridge_azure_event_hub_action_info
emqx_bridge_matrix_action_info,
emqx_bridge_mongodb_action_info,
emqx_bridge_pgsql_action_info,
emqx_bridge_syskeeper_action_info,
emqx_bridge_timescale_action_info,
emqx_bridge_redis_action_info
].
-else.
hard_coded_action_info_modules_ee() ->
@ -106,16 +119,30 @@ bridge_v1_type_to_action_type(Type) ->
ActionType -> ActionType
end.
action_type_to_bridge_v1_type(Bin) when is_binary(Bin) ->
action_type_to_bridge_v1_type(binary_to_existing_atom(Bin));
action_type_to_bridge_v1_type(Type) ->
action_type_to_bridge_v1_type(Bin, Conf) when is_binary(Bin) ->
action_type_to_bridge_v1_type(binary_to_existing_atom(Bin), Conf);
action_type_to_bridge_v1_type(ActionType, ActionConf) ->
ActionInfoMap = info_map(),
ActionTypeToBridgeV1Type = maps:get(action_type_to_bridge_v1_type, ActionInfoMap),
case maps:get(Type, ActionTypeToBridgeV1Type, undefined) of
undefined -> Type;
BridgeV1Type -> BridgeV1Type
case maps:get(ActionType, ActionTypeToBridgeV1Type, undefined) of
undefined ->
ActionType;
BridgeV1TypeFun when is_function(BridgeV1TypeFun) ->
case get_confs(ActionType, ActionConf) of
{ConnectorConfig, ActionConfig} -> BridgeV1TypeFun({ConnectorConfig, ActionConfig});
undefined -> ActionType
end;
BridgeV1Type ->
BridgeV1Type
end.
get_confs(ActionType, #{<<"connector">> := ConnectorName} = ActionConfig) ->
ConnectorType = action_type_to_connector_type(ActionType),
ConnectorConfig = emqx_conf:get_raw([connectors, ConnectorType, ConnectorName]),
{ConnectorConfig, ActionConfig};
get_confs(_, _) ->
undefined.
%% This function should return true for all inputs that are bridge V1 types for
%% bridges that have been refactored to bridge V2s, and for all all bridge V2
%% types. For everything else the function should return false.
@ -232,37 +259,56 @@ get_info_map(Module) ->
%% Force the module to get loaded
_ = code:ensure_loaded(Module),
ActionType = Module:action_type_name(),
BridgeV1Type =
{BridgeV1TypeOrFun, BridgeV1Types} =
case erlang:function_exported(Module, bridge_v1_type_name, 0) of
true ->
Module:bridge_v1_type_name();
case Module:bridge_v1_type_name() of
{_BridgeV1TypeFun, _BridgeV1Types} = BridgeV1TypeTuple ->
BridgeV1TypeTuple;
BridgeV1Type0 ->
{BridgeV1Type0, [BridgeV1Type0]}
end;
false ->
Module:action_type_name()
{ActionType, [ActionType]}
end,
#{
action_type_names => #{
ActionType => true,
BridgeV1Type => true
},
bridge_v1_type_to_action_type => #{
BridgeV1Type => ActionType,
action_type_names =>
lists:foldl(
fun(BridgeV1Type, M) ->
M#{BridgeV1Type => true}
end,
#{ActionType => true},
BridgeV1Types
),
bridge_v1_type_to_action_type =>
lists:foldl(
fun(BridgeV1Type, M) ->
%% Alias the bridge V1 type to the action type
ActionType => ActionType
},
M#{BridgeV1Type => ActionType}
end,
#{ActionType => ActionType},
BridgeV1Types
),
action_type_to_bridge_v1_type => #{
ActionType => BridgeV1Type
},
action_type_to_connector_type => #{
ActionType => Module:connector_type_name(),
%% Alias the bridge V1 type to the action type
BridgeV1Type => Module:connector_type_name()
ActionType => BridgeV1TypeOrFun
},
action_type_to_connector_type =>
lists:foldl(
fun(BridgeV1Type, M) ->
M#{BridgeV1Type => Module:connector_type_name()}
end,
#{ActionType => Module:connector_type_name()},
BridgeV1Types
),
action_type_to_schema_module => #{
ActionType => Module:schema_module()
},
action_type_to_info_module => #{
ActionType => Module,
%% Alias the bridge V1 type to the action type
BridgeV1Type => Module
}
action_type_to_info_module =>
lists:foldl(
fun(BridgeV1Type, M) ->
M#{BridgeV1Type => Module}
end,
#{ActionType => Module},
BridgeV1Types
)
}.

2
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge, [
{description, "EMQX bridges"},
{vsn, "0.1.30"},
{vsn, "0.1.31"},
{registered, [emqx_bridge_sup]},
{mod, {emqx_bridge_app, []}},
{applications, [

Some files were not shown because too many files have changed in this diff Show More