Merge remote-tracking branch 'origin/master' into release-53

.ci/docker-compose-file/credentials.env

@@ -0,0 +1,7 @@
+MONGO_USERNAME=emqx
+MONGO_PASSWORD=passw0rd
+MONGO_AUTHSOURCE=admin
+
+# See "Environment Variables" @ https://hub.docker.com/_/mongo
+MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
+MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}

.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml

@@ -9,6 +9,9 @@ services:
       - emqx_bridge
     ports:
       - "27017:27017"
+    env_file:
+      - .env
+      - credentials.env
     command:
       --ipv6
       --bind_ip_all

.ci/docker-compose-file/docker-compose.yaml

@@ -5,6 +5,7 @@ services:
     container_name: erlang
     image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
     env_file:
+      - credentials.env
       - conf.env
     environment:
       GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}

.ci/docker-compose-file/redis/sentinel-tcp/sentinel-base.conf

@@ -1,7 +1,7 @@
 sentinel resolve-hostnames yes
 bind :: 0.0.0.0
 
-sentinel monitor mymaster redis-sentinel-master 6379 1
+sentinel monitor mytcpmaster redis-sentinel-master 6379 1
-sentinel auth-pass mymaster public
+sentinel auth-pass mytcpmaster public
-sentinel down-after-milliseconds mymaster 10000
+sentinel down-after-milliseconds mytcpmaster 10000
-sentinel failover-timeout mymaster 20000
+sentinel failover-timeout mytcpmaster 20000

.ci/docker-compose-file/redis/sentinel-tls/sentinel-base.conf

@@ -8,7 +8,7 @@ tls-key-file /etc/certs/key.pem
 tls-ca-cert-file /etc/certs/cacert.pem
 tls-auth-clients no
 
-sentinel monitor mymaster redis-sentinel-tls-master 6389 1
+sentinel monitor mytlsmaster redis-sentinel-tls-master 6389 1
-sentinel auth-pass mymaster public
+sentinel auth-pass mytlsmaster public
-sentinel down-after-milliseconds mymaster 10000
+sentinel down-after-milliseconds mytlsmaster 10000
-sentinel failover-timeout mymaster 20000
+sentinel failover-timeout mytlsmaster 20000

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -61,10 +61,6 @@ body:
         # paste output here
         $ uname -a
         # paste output here
-
-        # On Windows:
-        C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
-        # paste output here
         ```
 
         </details>

.github/pull_request_template.md

@@ -1,9 +1,8 @@
 Fixes <issue-or-jira-number>
 
-<!-- Make sure to target release-52 branch if this PR is intended to fix the issues for the release candidate. -->
+Release version: v/e5.?
 
 ## Summary
-copilot:summary
 
 ## PR Checklist
 Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked:

.github/workflows/build_packages.yaml

@@ -65,58 +65,6 @@ on:
         default: '5.2-3'
 
 jobs:
-  windows:
-    runs-on: windows-2019
-    if: inputs.profile == 'emqx'
-    strategy:
-      fail-fast: false
-      matrix:
-        profile: # for now only CE for windows
-          - emqx
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.inputs.ref }}
-        fetch-depth: 0
-
-    - uses: ilammy/msvc-dev-cmd@v1.12.0
-    - uses: erlef/setup-beam@v1.16.0
-      with:
-        otp-version: 25.3.2
-    - name: build
-      env:
-        PYTHON: python
-        DIAGNOSTIC: 1
-      run: |
-        # ensure crypto app (openssl)
-        erl -eval "erlang:display(crypto:info_lib())" -s init stop
-        make ${{ matrix.profile }}-tgz
-    - name: run emqx
-      timeout-minutes: 5
-      run: |
-        $ErrorActionPreference = "Stop"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
-        Start-Sleep -s 10
-        $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping
-        if ($pingOutput = 'pong') {
-          echo "EMQX started OK"
-        } else {
-          echo "Failed to ping EMQX $pingOutput"
-          Exit 1
-        }
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
-        echo "EMQX stopped"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
-        echo "EMQX installed"
-        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
-        echo "EMQX uninstalled"
-    - uses: actions/upload-artifact@v3
-      if: success()
-      with:
-        name: ${{ matrix.profile }}
-        path: _packages/${{ matrix.profile }}/
-        retention-days: 7
-
   mac:
     strategy:
       fail-fast: false
@@ -126,9 +74,9 @@ jobs:
         otp:
           - ${{ inputs.otp_vsn }}
         os:
-          - macos-11
           - macos-12
           - macos-12-arm64
+          - macos-13
     runs-on: ${{ matrix.os }}
     steps:
     - uses: emqx/self-hosted-cleanup-action@v1.0.3

.github/workflows/build_packages_cron.yaml

@@ -130,59 +130,3 @@ jobs:
         with:
           payload: |
             {"text": "Scheduled build of ${{ matrix.profile }} package for ${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}
-
-  windows:
-    if: github.repository_owner == 'emqx'
-    runs-on: windows-2019
-    strategy:
-      fail-fast: false
-      matrix:
-        profile:
-          - emqx
-        otp:
-          - 25.3.2
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ilammy/msvc-dev-cmd@v1.12.0
-      - uses: erlef/setup-beam@v1.16.0
-        with:
-          otp-version: ${{ matrix.otp }}
-      - name: build
-        env:
-          PYTHON: python
-          DIAGNOSTIC: 1
-        run: |
-          # ensure crypto app (openssl)
-          erl -eval "erlang:display(crypto:info_lib())" -s init stop
-          make ${{ matrix.profile }}-tgz
-      - name: run emqx
-        timeout-minutes: 5
-        run: |
-          ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
-          Start-Sleep -s 10
-          $pingOutput = ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping
-          if ($pingOutput = 'pong') {
-            echo "EMQX started OK"
-          } else {
-            echo "Failed to ping EMQX $pingOutput"
-            Exit 1
-          }
-          ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
-          echo "EMQX stopped"
-          ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
-          echo "EMQX installed"
-          ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
-          echo "EMQX uninstalled"
-      - uses: actions/upload-artifact@v3
-        with:
-          name: windows
-          path: _packages/${{ matrix.profile }}/*
-          retention-days: 7
-      - name: Send notification to Slack
-        uses: slackapi/slack-github-action@v1.23.0
-        if: failure()
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        with:
-          payload: |
-            {"text": "Scheduled build of ${{ matrix.profile }} package for Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}

.tool-versions

@@ -1,2 +1,2 @@
-erlang 25.3.2-2
+erlang 26.1.2-1
-elixir 1.14.5-otp-25
+elixir 1.15.7-otp-26

apps/emqx/include/emqx.hrl

@@ -39,9 +39,6 @@
 %% System topic
 -define(SYSTOP, <<"$SYS/">>).
 
-%% Queue topic
--define(QUEUE, <<"$queue/">>).
-
 %%--------------------------------------------------------------------
 %% alarms
 %%--------------------------------------------------------------------

apps/emqx/include/emqx_mqtt.hrl

@@ -55,6 +55,17 @@
 %% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
 -define(MAX_TOPIC_LEN, 65535).
 
+%%--------------------------------------------------------------------
+%% MQTT Share-Sub Internal
+%%--------------------------------------------------------------------
+
+-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}).
+
+%% guards
+-define(IS_TOPIC(T),
+    (is_binary(T) orelse is_record(T, share))
+).
+
 %%--------------------------------------------------------------------
 %% MQTT QoS Levels
 %%--------------------------------------------------------------------
@@ -661,13 +672,10 @@ end).
 -define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}).
 
 -define(SHARE, "$share").
+-define(QUEUE, "$queue").
 -define(SHARE(Group, Topic), emqx_topic:join([<<?SHARE>>, Group, Topic])).
--define(IS_SHARE(Topic),
+
-    case Topic of
+-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-        <<?SHARE, _/binary>> -> true;
-        _ -> false
-    end
-).
 
 -define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
 -define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).

apps/emqx/include/emqx_trace.hrl

@@ -32,6 +32,5 @@
 
 -define(SHARD, ?COMMON_SHARD).
 -define(MAX_SIZE, 30).
--define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -endif.

apps/emqx/include/http_api.hrl

@@ -17,6 +17,7 @@
 %% HTTP API Auth
 -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
 -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
+-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
 
 %% Bad Request
 -define(BAD_REQUEST, 'BAD_REQUEST').

apps/emqx/include/logger.hrl

@@ -40,7 +40,9 @@
     end
 ).
 
+-define(AUDIT_HANDLER, emqx_audit).
 -define(TRACE_FILTER, emqx_trace_filter).
+-define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)).
 
@@ -61,25 +63,35 @@
     )
 end).
 
--define(AUDIT(_Level_, _From_, _Meta_), begin
+-ifdef(EMQX_RELEASE_EDITION).
-    case emqx_config:get([log, audit], #{enable => false}) of
+
-        #{enable := false} ->
+-if(?EMQX_RELEASE_EDITION == ee).
+
+-define(AUDIT(_LevelFun_, _MetaFun_), begin
+    case logger_config:get(logger, ?AUDIT_HANDLER) of
+        {error, {not_found, _}} ->
             ok;
-        #{enable := true, level := _AllowLevel_} ->
+        {ok, Handler = #{level := _AllowLevel_}} ->
+            _Level_ = _LevelFun_,
             case logger:compare_levels(_AllowLevel_, _Level_) of
                 _R_ when _R_ == lt; _R_ == eq ->
-                    emqx_trace:log(
+                    emqx_audit:log(_Level_, _MetaFun_, Handler);
-                        _Level_,
+                _ ->
-                        [{emqx_audit, fun(L, _) -> L end, undefined, undefined}],
-                        _Msg = undefined,
-                        _Meta_#{from => _From_}
-                    );
-                gt ->
                     ok
             end
     end
 end).
 
+-else.
+%% Only for compile pass, ce edition will not call it
+-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
+-endif.
+
+-else.
+%% Only for compile pass, ce edition will not call it
+-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
+-endif.
+
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
 -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).

apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl

@@ -9,14 +9,9 @@
 -include_lib("stdlib/include/assert.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
--include_lib("emqx/src/emqx_persistent_session_ds.hrl").
-
--define(DEFAULT_KEYSPACE, default).
--define(DS_SHARD_ID, <<"local">>).
--define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}).
-
 -import(emqx_common_test_helpers, [on_exit/1]).
 
 %%------------------------------------------------------------------------------
@@ -45,7 +40,7 @@ init_per_testcase(TestCase, Config) when
     Cluster = cluster(#{n => 1}),
     ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
     NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
-    Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
+    Nodes = emqx_cth_cluster:start(NodeSpecs),
     [
         {cluster, Cluster},
         {node_specs, NodeSpecs},
@@ -53,12 +48,36 @@ init_per_testcase(TestCase, Config) when
         {nodes, Nodes}
         | Config
     ];
+init_per_testcase(t_session_gc = TestCase, Config) ->
+    Opts = #{
+        n => 3,
+        roles => [core, core, replicant],
+        extra_emqx_conf =>
+            "\n session_persistence {"
+            "\n   last_alive_update_interval = 500ms "
+            "\n   session_gc_interval = 2s "
+            "\n   session_gc_batch_size = 1 "
+            "\n }"
+    },
+    Cluster = cluster(Opts),
+    ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
+    NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
+    Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
+    [
+        {cluster, Cluster},
+        {node_specs, NodeSpecs},
+        {cluster_opts, ClusterOpts},
+        {nodes, Nodes},
+        {gc_interval, timer:seconds(2)}
+        | Config
+    ];
 init_per_testcase(_TestCase, Config) ->
     Config.
 
 end_per_testcase(TestCase, Config) when
     TestCase =:= t_session_subscription_idempotency;
-    TestCase =:= t_session_unsubscription_idempotency
+    TestCase =:= t_session_unsubscription_idempotency;
+    TestCase =:= t_session_gc
 ->
     Nodes = ?config(nodes, Config),
     emqx_common_test_helpers:call_janitor(60_000),
@@ -72,32 +91,38 @@ end_per_testcase(_TestCase, _Config) ->
 %% Helper fns
 %%------------------------------------------------------------------------------
 
-cluster(#{n := N}) ->
+cluster(#{n := N} = Opts) ->
-    Spec = #{role => core, apps => app_specs()},
+    MkRole = fun(M) ->
+        case maps:get(roles, Opts, undefined) of
+            undefined ->
+                core;
+            Roles ->
+                lists:nth(M, Roles)
+        end
+    end,
+    MkSpec = fun(M) -> #{role => MkRole(M), apps => app_specs(Opts)} end,
     lists:map(
         fun(M) ->
             Name = list_to_atom("ds_SUITE" ++ integer_to_list(M)),
-            {Name, Spec}
+            {Name, MkSpec(M)}
         end,
         lists:seq(1, N)
     ).
 
 app_specs() ->
+    app_specs(_Opts = #{}).
+
+app_specs(Opts) ->
+    ExtraEMQXConf = maps:get(extra_emqx_conf, Opts, ""),
     [
         emqx_durable_storage,
-        {emqx, "persistent_session_store = {ds = true}"}
+        {emqx, "session_persistence = {enable = true}" ++ ExtraEMQXConf}
     ].
 
 get_mqtt_port(Node, Type) ->
     {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]),
     Port.
 
-get_all_iterator_ids(Node) ->
-    Fn = fun(K, _V, Acc) -> [K | Acc] end,
-    erpc:call(Node, fun() ->
-        emqx_ds_storage_layer:foldl_iterator_prefix(?DS_SHARD, <<>>, Fn, [])
-    end).
-
 wait_nodeup(Node) ->
     ?retry(
         _Sleep0 = 500,
@@ -127,33 +152,37 @@ start_client(Opts0 = #{}) ->
 
 restart_node(Node, NodeSpec) ->
     ?tp(will_restart_node, #{}),
-    ?tp(notice, "restarting node", #{node => Node}),
+    emqx_cth_cluster:restart(Node, NodeSpec),
-    true = monitor_node(Node, true),
-    ok = erpc:call(Node, init, restart, []),
-    receive
-        {nodedown, Node} ->
-            ok
-    after 10_000 ->
-        ct:fail("node ~p didn't stop", [Node])
-    end,
-    ?tp(notice, "waiting for nodeup", #{node => Node}),
     wait_nodeup(Node),
-    wait_gen_rpc_down(NodeSpec),
-    ?tp(notice, "restarting apps", #{node => Node}),
-    Apps = maps:get(apps, NodeSpec),
-    ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]),
-    _ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]),
-    %% have to re-inject this so that we may stop the node succesfully at the
-    %% end....
-    ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec),
-    ok = snabbkaffe:forward_trace(Node),
-    ?tp(notice, "node restarted", #{node => Node}),
     ?tp(restarted_node, #{}),
     ok.
 
 is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}}) ->
     EI > 0.
 
+list_all_sessions(Node) ->
+    erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []).
+
+list_all_subscriptions(Node) ->
+    erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []).
+
+list_all_pubranges(Node) ->
+    erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []).
+
+prop_only_cores_run_gc(CoreNodes) ->
+    {"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}.
+prop_only_cores_run_gc(Trace, CoreNodes) ->
+    GCNodes = lists:usort([
+        N
+     || #{
+            ?snk_kind := K,
+            ?snk_meta := #{node := N}
+        } <- Trace,
+        lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]),
+        N =/= node()
+    ]),
+    ?assertEqual(lists:usort(CoreNodes), GCNodes).
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -233,10 +262,10 @@ t_session_subscription_idempotency(Config) ->
         end,
         fun(Trace) ->
             ct:pal("trace:\n  ~p", [Trace]),
-            SubTopicFilterWords = emqx_topic:words(SubTopicFilter),
+            ConnInfo = #{},
             ?assertMatch(
-                {ok, #{}, #{SubTopicFilterWords := #{}}},
+                #{subscriptions := #{SubTopicFilter := #{}}},
-                erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
+                erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
             )
         end
     ),
@@ -307,9 +336,10 @@ t_session_unsubscription_idempotency(Config) ->
         end,
         fun(Trace) ->
             ct:pal("trace:\n  ~p", [Trace]),
+            ConnInfo = #{},
             ?assertMatch(
-                {ok, #{}, Subs = #{}} when map_size(Subs) =:= 0,
+                #{subscriptions := Subs = #{}} when map_size(Subs) =:= 0,
-                erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
+                erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
             ),
             ok
         end
@@ -370,18 +400,12 @@ do_t_session_discard(Params) ->
                 _Attempts0 = 50,
                 true = map_size(emqx_persistent_session_ds:list_all_streams()) > 0
             ),
-            ?retry(
-                _Sleep0 = 100,
-                _Attempts0 = 50,
-                true = map_size(emqx_persistent_session_ds:list_all_iterators()) > 0
-            ),
             ok = emqtt:stop(Client0),
             ?tp(notice, "disconnected", #{}),
 
             ?tp(notice, "reconnecting", #{}),
-            %% we still have iterators and streams
+            %% we still have streams
             ?assert(map_size(emqx_persistent_session_ds:list_all_streams()) > 0),
-            ?assert(map_size(emqx_persistent_session_ds:list_all_iterators()) > 0),
             Client1 = start_client(ReconnectOpts),
             {ok, _} = emqtt:connect(Client1),
             ?assertEqual([], emqtt:subscriptions(Client1)),
@@ -394,7 +418,7 @@ do_t_session_discard(Params) ->
             ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
             ?assertEqual([], emqx_persistent_session_ds_router:topics()),
             ?assertEqual(#{}, emqx_persistent_session_ds:list_all_streams()),
-            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_iterators()),
+            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_pubranges()),
             ok = emqtt:stop(Client1),
             ?tp(notice, "disconnected", #{}),
 
@@ -406,3 +430,201 @@ do_t_session_discard(Params) ->
         end
     ),
     ok.
+
+t_session_expiration1(Config) ->
+    ClientId = atom_to_binary(?FUNCTION_NAME),
+    Opts = #{
+        clientid => ClientId,
+        sequence => [
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 1}}, #{}},
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
+        ]
+    },
+    do_t_session_expiration(Config, Opts).
+
+t_session_expiration2(Config) ->
+    ClientId = atom_to_binary(?FUNCTION_NAME),
+    Opts = #{
+        clientid => ClientId,
+        sequence => [
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{
+                'Session-Expiry-Interval' => 1
+            }},
+            {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
+        ]
+    },
+    do_t_session_expiration(Config, Opts).
+
+do_t_session_expiration(_Config, Opts) ->
+    #{
+        clientid := ClientId,
+        sequence := [
+            {FirstConn, FirstDisconn},
+            {SecondConn, SecondDisconn},
+            {ThirdConn, ThirdDisconn}
+        ]
+    } = Opts,
+    CommonParams = #{proto_ver => v5, clientid => ClientId},
+    ?check_trace(
+        begin
+            Topic = <<"some/topic">>,
+            Params0 = maps:merge(CommonParams, FirstConn),
+            Client0 = start_client(Params0),
+            {ok, _} = emqtt:connect(Client0),
+            {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2),
+            Subs0 = emqx_persistent_session_ds:list_all_subscriptions(),
+            ?assertEqual(1, map_size(Subs0), #{subs => Subs0}),
+            Info0 = maps:from_list(emqtt:info(Client0)),
+            ?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}),
+            emqtt:disconnect(Client0, ?RC_NORMAL_DISCONNECTION, FirstDisconn),
+
+            Params1 = maps:merge(CommonParams, SecondConn),
+            Client1 = start_client(Params1),
+            {ok, _} = emqtt:connect(Client1),
+            Info1 = maps:from_list(emqtt:info(Client1)),
+            ?assertEqual(1, maps:get(session_present, Info1), #{info => Info1}),
+            Subs1 = emqtt:subscriptions(Client1),
+            ?assertEqual([], Subs1),
+            emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn),
+
+            ct:sleep(1_500),
+
+            Params2 = maps:merge(CommonParams, ThirdConn),
+            Client2 = start_client(Params2),
+            {ok, _} = emqtt:connect(Client2),
+            Info2 = maps:from_list(emqtt:info(Client2)),
+            ?assertEqual(0, maps:get(session_present, Info2), #{info => Info2}),
+            Subs2 = emqtt:subscriptions(Client2),
+            ?assertEqual([], Subs2),
+            emqtt:publish(Client2, Topic, <<"payload">>),
+            ?assertNotReceive({publish, #{topic := Topic}}),
+            %% ensure subscriptions are absent from table.
+            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
+            emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn),
+
+            ok
+        end,
+        []
+    ),
+    ok.
+
+t_session_gc(Config) ->
+    GCInterval = ?config(gc_interval, Config),
+    [Node1, Node2, Node3] = Nodes = ?config(nodes, Config),
+    CoreNodes = [Node1, Node2],
+    [
+        Port1,
+        Port2,
+        Port3
+    ] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes),
+    CommonParams = #{
+        clean_start => false,
+        proto_ver => v5
+    },
+    StartClient = fun(ClientId, Port, ExpiryInterval) ->
+        Params = maps:merge(CommonParams, #{
+            clientid => ClientId,
+            port => Port,
+            properties => #{'Session-Expiry-Interval' => ExpiryInterval}
+        }),
+        Client = start_client(Params),
+        {ok, _} = emqtt:connect(Client),
+        Client
+    end,
+
+    ?check_trace(
+        begin
+            ClientId0 = <<"session_gc0">>,
+            Client0 = StartClient(ClientId0, Port1, 30),
+
+            ClientId1 = <<"session_gc1">>,
+            Client1 = StartClient(ClientId1, Port2, 1),
+
+            ClientId2 = <<"session_gc2">>,
+            Client2 = StartClient(ClientId2, Port3, 1),
+
+            lists:foreach(
+                fun(Client) ->
+                    Topic = <<"some/topic">>,
+                    Payload = <<"hi">>,
+                    {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client, Topic, ?QOS_1),
+                    {ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1),
+                    ok
+                end,
+                [Client0, Client1, Client2]
+            ),
+
+            %% Clients are still alive; no session is garbage collected.
+            Res0 = ?block_until(
+                #{
+                    ?snk_kind := ds_session_gc,
+                    ?snk_span := {complete, _},
+                    ?snk_meta := #{node := N}
+                } when
+                    N =/= node(),
+                3 * GCInterval + 1_000
+            ),
+            ?assertMatch({ok, _}, Res0),
+            {ok, #{?snk_meta := #{time := T0}}} = Res0,
+            Sessions0 = list_all_sessions(Node1),
+            Subs0 = list_all_subscriptions(Node1),
+            ?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}),
+            ?assertEqual(3, map_size(Subs0), #{subs => Subs0}),
+
+            %% Now we disconnect 2 of them; only those should be GC'ed.
+            ?assertMatch(
+                {ok, {ok, _}},
+                ?wait_async_action(
+                    emqtt:stop(Client1),
+                    #{?snk_kind := terminate},
+                    1_000
+                )
+            ),
+            ct:pal("disconnected client1"),
+            ?assertMatch(
+                {ok, {ok, _}},
+                ?wait_async_action(
+                    emqtt:stop(Client2),
+                    #{?snk_kind := terminate},
+                    1_000
+                )
+            ),
+            ct:pal("disconnected client2"),
+            ?assertMatch(
+                {ok, _},
+                ?block_until(
+                    #{
+                        ?snk_kind := ds_session_gc_cleaned,
+                        ?snk_meta := #{node := N, time := T},
+                        session_ids := [ClientId1]
+                    } when
+                        N =/= node() andalso T > T0,
+                    4 * GCInterval + 1_000
+                )
+            ),
+            ?assertMatch(
+                {ok, _},
+                ?block_until(
+                    #{
+                        ?snk_kind := ds_session_gc_cleaned,
+                        ?snk_meta := #{node := N, time := T},
+                        session_ids := [ClientId2]
+                    } when
+                        N =/= node() andalso T > T0,
+                    4 * GCInterval + 1_000
+                )
+            ),
+            Sessions1 = list_all_sessions(Node1),
+            Subs1 = list_all_subscriptions(Node1),
+            ?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}),
+            ?assertEqual(1, map_size(Subs1), #{subs => Subs1}),
+
+            ok
+        end,
+        [
+            prop_only_cores_run_gc(CoreNodes)
+        ]
+    ),
+    ok.

apps/emqx/priv/bpapi.versions

@@ -39,10 +39,12 @@
 {emqx_mgmt_api_plugins,2}.
 {emqx_mgmt_cluster,1}.
 {emqx_mgmt_cluster,2}.
+{emqx_mgmt_data_backup,1}.
 {emqx_mgmt_trace,1}.
 {emqx_mgmt_trace,2}.
 {emqx_node_rebalance,1}.
 {emqx_node_rebalance,2}.
+{emqx_node_rebalance,3}.
 {emqx_node_rebalance_api,1}.
 {emqx_node_rebalance_api,2}.
 {emqx_node_rebalance_evacuation,1}.

apps/emqx/rebar.config

@@ -27,9 +27,9 @@
     {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}},
     {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
-    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
+    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
-    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}},
+    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
@@ -45,7 +45,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.1"}}}
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]},
                           {"integration_test", [recursive]}]}
@@ -55,7 +55,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.1"}}}
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}

apps/emqx/rebar.config.script

@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.202"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.303"}}}.
 
 Dialyzer = fun(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),

apps/emqx/src/bhvrs/emqx_db_backup.erl

@@ -16,4 +16,21 @@
 
 -module(emqx_db_backup).
 
+-type traverse_break_reason() :: over | migrate.
+
 -callback backup_tables() -> [mria:table()].
+
+%% validate the backup
+%% return `ok` to traverse the next item
+%% return `{ok, over}` to finish the traverse
+%% return `{ok, migrate}` to call the migration callback
+-callback validate_mnesia_backup(tuple()) ->
+    ok
+    | {ok, traverse_break_reason()}
+    | {error, term()}.
+
+-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
+
+-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
+
+-export_type([traverse_break_reason/0]).

apps/emqx/src/config/emqx_config_logger.erl

@@ -23,8 +23,9 @@
 -export([post_config_update/5]).
 -export([filter_audit/2]).
 
+-include("logger.hrl").
+
 -define(LOG, [log]).
--define(AUDIT_HANDLER, emqx_audit).
 
 add_handler() ->
     ok = emqx_config_handler:add_handler(?LOG, ?MODULE),
@@ -95,6 +96,10 @@ update_log_handlers(NewHandlers) ->
     ok = application:set_env(kernel, logger, NewHandlers),
     ok.
 
+%% Don't remove audit log handler here, we need record this removed action into audit log file.
+%% we will remove audit log handler after audit log is record in emqx_audit:log/3.
+update_log_handler({removed, ?AUDIT_HANDLER}) ->
+    ok;
 update_log_handler({removed, Id}) ->
     log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]),
     logger:remove_handler(Id);

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.1.14"},
+    {vsn, "5.1.15"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_broker.erl

@@ -118,18 +118,20 @@ create_tabs() ->
 %% Subscribe API
 %%------------------------------------------------------------------------------
 
--spec subscribe(emqx_types:topic()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok.
-subscribe(Topic) when is_binary(Topic) ->
+subscribe(Topic) when ?IS_TOPIC(Topic) ->
     subscribe(Topic, undefined).
 
--spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) ->
-subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) ->
+    ok.
+subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) ->
     subscribe(Topic, SubId, ?DEFAULT_SUBOPTS);
-subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) ->
+subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) ->
     subscribe(Topic, undefined, SubOpts).
 
--spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) ->
-subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
+    ok.
+subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
     SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0),
     _ = emqx_trace:subscribe(Topic, SubId, SubOpts),
     SubPid = self(),
@@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) ->
 with_subid(SubId, SubOpts) ->
     maps:put(subid, SubId, SubOpts).
 
-%% @private
 do_subscribe(Topic, SubPid, SubOpts) ->
     true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}),
-    Group = maps:get(share, SubOpts, undefined),
+    do_subscribe2(Topic, SubPid, SubOpts).
-    do_subscribe(Group, Topic, SubPid, SubOpts).
 
-do_subscribe(undefined, Topic, SubPid, SubOpts) ->
+do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) ->
+    %% FIXME: subscribe shard bug
+    %% https://emqx.atlassian.net/browse/EMQX-10214
     case emqx_broker_helper:get_sub_shard(SubPid, Topic) of
         0 ->
             true = ets:insert(?SUBSCRIBER, {Topic, SubPid}),
@@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) ->
             true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}),
             call(pick({Topic, I}), {subscribe, Topic, I})
     end;
-%% Shared subscription
+do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when
-do_subscribe(Group, Topic, SubPid, SubOpts) ->
+    is_binary(RealTopic)
+->
     true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}),
-    emqx_shared_sub:subscribe(Group, Topic, SubPid).
+    emqx_shared_sub:subscribe(Group, RealTopic, SubPid).
 
 %%--------------------------------------------------------------------
 %% Unsubscribe API
 %%--------------------------------------------------------------------
 
--spec unsubscribe(emqx_types:topic()) -> ok.
+-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok.
-unsubscribe(Topic) when is_binary(Topic) ->
+unsubscribe(Topic) when ?IS_TOPIC(Topic) ->
     SubPid = self(),
     case ets:lookup(?SUBOPTION, {Topic, SubPid}) of
         [{_, SubOpts}] ->
-            _ = emqx_broker_helper:reclaim_seq(Topic),
             _ = emqx_trace:unsubscribe(Topic, SubOpts),
             do_unsubscribe(Topic, SubPid, SubOpts);
         [] ->
             ok
     end.
 
+-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
+    ok.
 do_unsubscribe(Topic, SubPid, SubOpts) ->
     true = ets:delete(?SUBOPTION, {Topic, SubPid}),
     true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}),
-    Group = maps:get(share, SubOpts, undefined),
+    do_unsubscribe2(Topic, SubPid, SubOpts).
-    do_unsubscribe(Group, Topic, SubPid, SubOpts).
 
-do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
+-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
+    ok.
+do_unsubscribe2(Topic, SubPid, SubOpts) when
+    is_binary(Topic), is_pid(SubPid), is_map(SubOpts)
+->
+    _ = emqx_broker_helper:reclaim_seq(Topic),
     case maps:get(shard, SubOpts, 0) of
         0 ->
             true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
@@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
             true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
             cast(pick({Topic, I}), {unsubscribed, Topic, I})
     end;
-do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
+do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when
+    is_binary(Group), is_binary(Topic), is_pid(SubPid)
+->
     emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
 
 %%--------------------------------------------------------------------
@@ -306,7 +316,9 @@ aggre([], true, Acc) ->
     lists:usort(Acc).
 
 %% @doc Forward message to another node.
--spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) ->
+-spec forward(
+    node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async
+) ->
     emqx_types:deliver_result().
 forward(Node, To, Delivery, async) ->
     true = emqx_broker_proto_v1:forward_async(Node, To, Delivery),
@@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) ->
             Result
     end.
 
--spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
+-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
+    emqx_types:deliver_result().
 dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) ->
     case emqx:is_running() of
         true ->
@@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) ->
     end.
 
 -compile({inline, [subscribers/1]}).
--spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) ->
+-spec subscribers(
+    emqx_types:topic()
+    | emqx_types:share()
+    | {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()}
+) ->
     [pid()].
 subscribers(Topic) when is_binary(Topic) ->
     lookup_value(?SUBSCRIBER, Topic, []);
@@ -372,7 +389,7 @@ subscriber_down(SubPid) ->
                 SubOpts when is_map(SubOpts) ->
                     _ = emqx_broker_helper:reclaim_seq(Topic),
                     true = ets:delete(?SUBOPTION, {Topic, SubPid}),
-                    do_unsubscribe(undefined, Topic, SubPid, SubOpts);
+                    do_unsubscribe2(Topic, SubPid, SubOpts);
                 undefined ->
                     ok
             end
@@ -386,7 +403,7 @@ subscriber_down(SubPid) ->
 %%--------------------------------------------------------------------
 
 -spec subscriptions(pid() | emqx_types:subid()) ->
-    [{emqx_types:topic(), emqx_types:subopts()}].
+    [{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}].
 subscriptions(SubPid) when is_pid(SubPid) ->
     [
         {Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})}
@@ -400,20 +417,22 @@ subscriptions(SubId) ->
             []
     end.
 
--spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()].
+-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()].
 subscriptions_via_topic(Topic) ->
     MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}],
     ets:select(?SUBOPTION, MatchSpec).
 
--spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean().
+-spec subscribed(
+    pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share()
+) -> boolean().
 subscribed(SubPid, Topic) when is_pid(SubPid) ->
     ets:member(?SUBOPTION, {Topic, SubPid});
 subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
     SubPid = emqx_broker_helper:lookup_subpid(SubId),
     ets:member(?SUBOPTION, {Topic, SubPid}).
 
--spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()).
+-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
-get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) ->
+get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
     lookup_value(?SUBOPTION, {Topic, SubPid});
 get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
     case emqx_broker_helper:lookup_subpid(SubId) of
@@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
             undefined
     end.
 
--spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean().
+-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean().
 set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) ->
     set_subopts(self(), Topic, NewOpts).
 
@@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) ->
             false
     end.
 
--spec topics() -> [emqx_types:topic()].
+-spec topics() -> [emqx_types:topic() | emqx_types:share()].
 topics() ->
     emqx_router:topics().
 
@@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
--spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
+-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
+    emqx_types:deliver_result().
 do_dispatch(Topic, #delivery{message = Msg}) ->
     DispN = lists:foldl(
         fun(Sub, N) ->
@@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) ->
             {ok, DispN}
     end.
 
+%% Donot dispatch to share subscriber here.
+%% we do it in `emqx_shared_sub.erl` with configured strategy
 do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
     case erlang:is_process_alive(SubPid) of
         true ->

apps/emqx/src/emqx_channel.erl

@@ -423,6 +423,7 @@ handle_in(
             {ok, Channel}
     end;
 handle_in(
+    %% TODO: Why discard the Reason Code?
     ?PUBREC_PACKET(PacketId, _ReasonCode, Properties),
     Channel =
         #channel{clientinfo = ClientInfo, session = Session}
@@ -476,60 +477,27 @@ handle_in(
             ok = emqx_metrics:inc('packets.pubcomp.missed'),
             {ok, Channel}
     end;
-handle_in(
+handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) ->
-    SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
+    Pipe = pipeline(
-    Channel = #channel{clientinfo = ClientInfo}
+        [
-) ->
+            fun check_subscribe/2,
-    case emqx_packet:check(SubPkt) of
+            fun enrich_subscribe/2,
-        ok ->
+            %% TODO && FIXME (EMQX-10786): mount topic before authz check.
-            TopicFilters0 = parse_topic_filters(TopicFilters),
+            fun check_sub_authzs/2,
-            TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
+            fun check_sub_caps/2
-            TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
-            HasAuthzDeny = lists:any(
-                fun({_TopicFilter, ReasonCode}) ->
-                    ReasonCode =:= ?RC_NOT_AUTHORIZED
-                end,
-                TupleTopicFilters0
-            ),
-            DenyAction = emqx:get_config([authorization, deny_action], ignore),
-            case DenyAction =:= disconnect andalso HasAuthzDeny of
-                true ->
-                    handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
-                false ->
-                    TopicFilters2 = [
-                        TopicFilter
-                     || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
         ],
-                    TopicFilters3 = run_hooks(
+        SubPkt,
-                        'client.subscribe',
+        Channel0
-                        [ClientInfo, Properties],
-                        TopicFilters2
     ),
-                    {TupleTopicFilters1, NChannel} = process_subscribe(
+    case Pipe of
-                        TopicFilters3,
+        {ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} ->
-                        Properties,
+            {TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel),
-                        Channel
+            ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC),
-                    ),
+            handle_out(suback, {PacketId, ReasonCodes}, NChannel);
-                    TupleTopicFilters2 =
+        {error, {disconnect, RC}, Channel} ->
-                        lists:foldl(
+            %% funcs in pipeline always cause action: `disconnect`
-                            fun
+            %% And Only one ReasonCode in DISCONNECT packet
-                                ({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) ->
+            handle_out(disconnect, RC, Channel)
-                                    Key = {Topic, maps:without([deny_subscription], Opts)},
-                                    lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR});
-                                (Tuple = {Key, _Value}, Acc) ->
-                                    lists:keyreplace(Key, 1, Acc, Tuple)
-                            end,
-                            TupleTopicFilters0,
-                            TupleTopicFilters1
-                        ),
-                    ReasonCodes2 = [
-                        ReasonCode
-                     || {_TopicFilter, ReasonCode} <- TupleTopicFilters2
-                    ],
-                    handle_out(suback, {PacketId, ReasonCodes2}, NChannel)
-            end;
-        {error, ReasonCode} ->
-            handle_out(disconnect, ReasonCode, Channel)
     end;
 handle_in(
     Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
@@ -540,7 +508,7 @@ handle_in(
             TopicFilters1 = run_hooks(
                 'client.unsubscribe',
                 [ClientInfo, Properties],
-                parse_topic_filters(TopicFilters)
+                parse_raw_topic_filters(TopicFilters)
             ),
             {ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel),
             handle_out(unsuback, {PacketId, ReasonCodes}, NChannel);
@@ -782,32 +750,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) ->
 %% Process Subscribe
 %%--------------------------------------------------------------------
 
--compile({inline, [process_subscribe/3]}).
+process_subscribe(TopicFilters, Channel) ->
-process_subscribe(TopicFilters, SubProps, Channel) ->
+    process_subscribe(TopicFilters, Channel, []).
-    process_subscribe(TopicFilters, SubProps, Channel, []).
 
-process_subscribe([], _SubProps, Channel, Acc) ->
+process_subscribe([], Channel, Acc) ->
     {lists:reverse(Acc), Channel};
-process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) ->
+process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) ->
-    case check_sub_caps(TopicFilter, SubOpts, Channel) of
+    {NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel),
-        ok ->
+    process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]).
-            {ReasonCode, NChannel} = do_subscribe(
-                TopicFilter,
-                SubOpts#{sub_props => SubProps},
-                Channel
-            ),
-            process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]);
-        {error, ReasonCode} ->
-            ?SLOG(
-                warning,
-                #{
-                    msg => "cannot_subscribe_topic_filter",
-                    reason => emqx_reason_codes:name(ReasonCode)
-                },
-                #{topic => TopicFilter}
-            ),
-            process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
-    end.
 
 do_subscribe(
     TopicFilter,
@@ -818,11 +768,13 @@ do_subscribe(
             session = Session
         }
 ) ->
+    %% TODO && FIXME (EMQX-10786): mount topic before authz check.
     NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter),
-    NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel),
+    case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of
-    case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of
         {ok, NSession} ->
-            {QoS, Channel#channel{session = NSession}};
+            %% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now
+            RC = QoS,
+            {RC, Channel#channel{session = NSession}};
         {error, RC} ->
             ?SLOG(
                 warning,
@@ -835,6 +787,30 @@ do_subscribe(
             {RC, Channel}
     end.
 
+gen_reason_codes(TFChecked, TFSubedWitNhRC) ->
+    do_gen_reason_codes([], TFChecked, TFSubedWitNhRC).
+
+%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2
+%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and
+%% NRC should override the initial RC.
+do_gen_reason_codes(Acc, [], []) ->
+    lists:reverse(Acc);
+do_gen_reason_codes(
+    Acc,
+    [{_, ?RC_SUCCESS} | RestTF],
+    [{_, NRC} | RestWithNRC]
+) ->
+    %% will passing through `process_subscribe/2`
+    %% use NRC to override IintialRC
+    do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC);
+do_gen_reason_codes(
+    Acc,
+    [{_, InitialRC} | Rest],
+    RestWithNRC
+) ->
+    %% InitialRC is not `RC_SUCCESS`, use it.
+    do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC).
+
 %%--------------------------------------------------------------------
 %% Process Unsubscribe
 %%--------------------------------------------------------------------
@@ -1213,13 +1189,8 @@ handle_call(Req, Channel) ->
     ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}.
 
 handle_info({subscribe, TopicFilters}, Channel) ->
-    {_, NChannel} = lists:foldl(
+    NTopicFilters = enrich_subscribe(TopicFilters, Channel),
-        fun({TopicFilter, SubOpts}, {_, ChannelAcc}) ->
+    {_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel),
-            do_subscribe(TopicFilter, SubOpts, ChannelAcc)
-        end,
-        {[], Channel},
-        parse_topic_filters(TopicFilters)
-    ),
     {ok, NChannel};
 handle_info({unsubscribe, TopicFilters}, Channel) ->
     {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel),
@@ -1234,12 +1205,13 @@ handle_info(
         #channel{
             conn_state = ConnState,
             clientinfo = ClientInfo,
+            conninfo = ConnInfo,
             session = Session
         }
 ) when
     ConnState =:= connected orelse ConnState =:= reauthenticating
 ->
-    {Intent, Session1} = emqx_session:disconnect(ClientInfo, Session),
+    {Intent, Session1} = emqx_session:disconnect(ClientInfo, ConnInfo, Session),
     Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
     Channel2 = Channel1#channel{session = Session1},
     case maybe_shutdown(Reason, Intent, Channel2) of
@@ -1351,7 +1323,8 @@ handle_timeout(
         {ok, Replies, NSession} ->
             handle_out(publish, Replies, Channel#channel{session = NSession})
     end;
-handle_timeout(_TRef, expire_session, Channel) ->
+handle_timeout(_TRef, expire_session, Channel = #channel{session = Session}) ->
+    ok = emqx_session:destroy(Session),
     shutdown(expired, Channel);
 handle_timeout(
     _TRef,
@@ -1859,49 +1832,156 @@ check_pub_caps(
 ) ->
     emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}).
 
+%%--------------------------------------------------------------------
+%% Check Subscribe Packet
+
+check_subscribe(SubPkt, _Channel) ->
+    case emqx_packet:check(SubPkt) of
+        ok -> ok;
+        {error, RC} -> {error, {disconnect, RC}}
+    end.
+
 %%--------------------------------------------------------------------
 %% Check Sub Authorization
 
-check_sub_authzs(TopicFilters, Channel) ->
-    check_sub_authzs(TopicFilters, Channel, []).
-
 check_sub_authzs(
-    [TopicFilter = {Topic, _} | More],
+    ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0),
-    Channel = #channel{clientinfo = ClientInfo},
+    Channel = #channel{clientinfo = ClientInfo}
-    Acc
 ) ->
+    CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo),
+    HasAuthzDeny = lists:any(
+        fun({{_TopicFilter, _SubOpts}, ReasonCode}) ->
+            ReasonCode =:= ?RC_NOT_AUTHORIZED
+        end,
+        CheckResult
+    ),
+    DenyAction = emqx:get_config([authorization, deny_action], ignore),
+    case DenyAction =:= disconnect andalso HasAuthzDeny of
+        true ->
+            {error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel};
+        false ->
+            {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}
+    end.
+
+do_check_sub_authzs(TopicFilters, ClientInfo) ->
+    do_check_sub_authzs(ClientInfo, TopicFilters, []).
+
+do_check_sub_authzs(_ClientInfo, [], Acc) ->
+    lists:reverse(Acc);
+do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) ->
+    %% subsclibe authz check only cares the real topic filter when shared-sub
+    %% e.g. only check <<"t/#">> for <<"$share/g/t/#">>
     Action = authz_action(TopicFilter),
-    case emqx_access_control:authorize(ClientInfo, Action, Topic) of
+    case
+        emqx_access_control:authorize(
+            ClientInfo,
+            Action,
+            emqx_topic:get_shared_real_topic(Topic)
+        )
+    of
+        %% TODO: support maximum QoS granted
+        %% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
+        %% Not implemented yet:
+        %% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]);
         allow ->
-            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
+            do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]);
         deny ->
-            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
+            do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
-    end;
+    end.
-check_sub_authzs([], _Channel, Acc) ->
-    lists:reverse(Acc).
 
 %%--------------------------------------------------------------------
 %% Check Sub Caps
 
-check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
+check_sub_caps(
-    emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts).
+    ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters),
+    Channel = #channel{clientinfo = ClientInfo}
+) ->
+    CheckResult = do_check_sub_caps(ClientInfo, TopicFilters),
+    {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}.
+
+do_check_sub_caps(ClientInfo, TopicFilters) ->
+    do_check_sub_caps(ClientInfo, TopicFilters, []).
+
+do_check_sub_caps(_ClientInfo, [], Acc) ->
+    lists:reverse(Acc);
+do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) ->
+    case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of
+        ok ->
+            do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]);
+        {error, NRC} ->
+            ?SLOG(
+                warning,
+                #{
+                    msg => "cannot_subscribe_topic_filter",
+                    reason => emqx_reason_codes:name(NRC)
+                },
+                #{topic => Topic}
+            ),
+            do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc])
+    end;
+do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) ->
+    do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]).
 
 %%--------------------------------------------------------------------
-%% Enrich SubId
+%% Run Subscribe Hooks
 
-enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
+run_sub_hooks(
-    [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
+    ?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0),
-enrich_subopts_subid(_Properties, TopicFilters) ->
+    _Channel = #channel{clientinfo = ClientInfo}
-    TopicFilters.
+) ->
+    TopicFilters = [
+        TopicFilter
+     || {TopicFilter, ?RC_SUCCESS} <- TopicFilters0
+    ],
+    _NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters).
 
 %%--------------------------------------------------------------------
 %% Enrich SubOpts
 
-enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) ->
+%% for api subscribe without sub-authz check and sub-caps check.
-    SubOpts;
+enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) ->
-enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
+    do_enrich_subscribe(#{}, TopicFilters, Channel);
+%% for mqtt clients sent subscribe packet.
+enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) ->
+    NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel),
+    {ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}.
+
+do_enrich_subscribe(Properties, TopicFilters, Channel) ->
+    _NTopicFilters = run_fold(
+        [
+            %% TODO: do try catch with reason code here
+            fun(TFs, _) -> parse_raw_topic_filters(TFs) end,
+            fun enrich_subopts_subid/2,
+            fun enrich_subopts_porps/2,
+            fun enrich_subopts_flags/2
+        ],
+        TopicFilters,
+        #{sub_props => Properties, channel => Channel}
+    ).
+
+enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) ->
+    [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
+enrich_subopts_subid(TopicFilters, _State) ->
+    TopicFilters.
+
+enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) ->
+    [{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters].
+
+enrich_subopts_flags(TopicFilters, #{channel := Channel}) ->
+    do_enrich_subopts_flags(TopicFilters, Channel).
+
+do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) ->
+    [{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters];
+do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
+    Rap = flag(IsBridge),
     NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)),
-    SubOpts#{rap => flag(IsBridge), nl => NL}.
+    [
+        {Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}}
+     || {Topic, SubOpts} <- TopicFilters
+    ].
+
+merge_default_subopts(SubOpts) ->
+    maps:merge(?DEFAULT_SUBOPTS, SubOpts).
 
 %%--------------------------------------------------------------------
 %% Enrich ConnAck Caps
@@ -2091,8 +2171,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) ->
 %%--------------------------------------------------------------------
 %% Parse Topic Filters
 
--compile({inline, [parse_topic_filters/1]}).
+%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _]
-parse_topic_filters(TopicFilters) ->
+parse_raw_topic_filters(TopicFilters) ->
     lists:map(fun emqx_topic:parse/1, TopicFilters).
 
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_cm.erl

@@ -91,6 +91,7 @@
     clean_down/1,
     mark_channel_connected/1,
     mark_channel_disconnected/1,
+    is_channel_connected/1,
     get_connected_client_count/0
 ]).
 

apps/emqx/src/emqx_cm_sup.erl

@@ -47,7 +47,17 @@ init([]) ->
     Locker = child_spec(emqx_cm_locker, 5000, worker),
     Registry = child_spec(emqx_cm_registry, 5000, worker),
     Manager = child_spec(emqx_cm, 5000, worker),
-    {ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}.
+    DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
+    Children =
+        [
+            Banned,
+            Flapping,
+            Locker,
+            Registry,
+            Manager,
+            DSSessionGCSup
+        ],
+    {ok, {SupFlags, Children}}.
 
 %%--------------------------------------------------------------------
 %% Internal functions

apps/emqx/src/emqx_message.erl

@@ -301,7 +301,9 @@ update_expiry(Msg) ->
     Msg.
 
 %% @doc Message to PUBLISH Packet.
--spec to_packet(emqx_types:packet_id(), emqx_types:message()) ->
+%%
+%% When QoS=0 then packet id must be `undefined'
+-spec to_packet(emqx_types:packet_id() | undefined, emqx_types:message()) ->
     emqx_types:packet().
 to_packet(
     PacketId,

apps/emqx/src/emqx_mountpoint.erl

@@ -17,6 +17,7 @@
 -module(emqx_mountpoint).
 
 -include("emqx.hrl").
+-include("emqx_mqtt.hrl").
 -include("emqx_placeholder.hrl").
 -include("types.hrl").
 
@@ -34,38 +35,54 @@
 -spec mount(maybe(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
+        | emqx_types:share()
         | emqx_types:message()
         | emqx_types:topic_filters().
 mount(undefined, Any) ->
     Any;
-mount(MountPoint, Topic) when is_binary(Topic) ->
+mount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
-    prefix(MountPoint, Topic);
+    prefix_maybe_share(MountPoint, Topic);
-mount(MountPoint, Msg = #message{topic = Topic}) ->
+mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
-    Msg#message{topic = prefix(MountPoint, Topic)};
+    Msg#message{topic = prefix_maybe_share(MountPoint, Topic)};
 mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
-    [{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
+    [{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
 
-%% @private
+-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
--compile({inline, [prefix/2]}).
+    Any ::
-prefix(MountPoint, Topic) ->
+        emqx_types:topic()
-    <<MountPoint/binary, Topic/binary>>.
+        | emqx_types:share().
+prefix_maybe_share(MountPoint, Topic) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
+    <<MountPoint/binary, Topic/binary>>;
+prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
+    #share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
 
 -spec unmount(maybe(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
+        | emqx_types:share()
         | emqx_types:message().
 unmount(undefined, Any) ->
     Any;
-unmount(MountPoint, Topic) when is_binary(Topic) ->
+unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
+    unmount_maybe_share(MountPoint, Topic);
+unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
+    Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}.
+
+unmount_maybe_share(MountPoint, Topic) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
     case string:prefix(Topic, MountPoint) of
         nomatch -> Topic;
         Topic1 -> Topic1
     end;
-unmount(MountPoint, Msg = #message{topic = Topic}) ->
+unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
-    case string:prefix(Topic, MountPoint) of
+    is_binary(MountPoint) andalso is_binary(Topic)
-        nomatch -> Msg;
+->
-        Topic1 -> Msg#message{topic = Topic1}
+    TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
-    end.
 
 -spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
 replvar(undefined, _Vars) ->

apps/emqx/src/emqx_mqtt_caps.erl

@@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) ->
 
 -spec check_sub(
     emqx_types:clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts()
 ) ->
     ok_or_error(emqx_types:reason_code()).
 check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) ->
     Caps = emqx_config:get_zone_conf(Zone, [mqtt]),
     Flags = #{
+        %% TODO: qos check
+        %% (max_qos_allowed, Map) ->
+        %% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2),
         topic_levels => emqx_topic:levels(Topic),
         is_wildcard => emqx_topic:wildcard(Topic),
-        is_shared => maps:is_key(share, SubOpts),
+        is_shared => erlang:is_record(Topic, share),
         is_exclusive => maps:get(is_exclusive, SubOpts, false)
     },
     do_check_sub(Flags, Caps, ClientInfo, Topic).
@@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) ->
     {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
 do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) ->
     {error, ?RC_TOPIC_FILTER_INVALID};
-do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) ->
+do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when
+    is_binary(Topic)
+->
     case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of
         deny ->
             {error, ?RC_QUOTA_EXCEEDED};
         _ ->
             ok
     end;
+%% for max_qos_allowed
+%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2
+%% do_check_sub(_, _) ->
+%%     {ok, RC};
 do_check_sub(_Flags, _Caps, _, _) ->
     ok.
 

apps/emqx/src/emqx_persistent_message.erl

@@ -19,7 +19,7 @@
 -include("emqx.hrl").
 
 -export([init/0]).
--export([is_store_enabled/0]).
+-export([is_persistence_enabled/0, force_ds/0]).
 
 %% Message persistence
 -export([
@@ -28,9 +28,8 @@
 
 -define(PERSISTENT_MESSAGE_DB, emqx_persistent_message).
 
-%% FIXME
 -define(WHEN_ENABLED(DO),
-    case is_store_enabled() of
+    case is_persistence_enabled() of
         true -> DO;
         false -> {skipped, disabled}
     end
@@ -40,18 +39,40 @@
 
 init() ->
     ?WHEN_ENABLED(begin
-        ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{
+        Backend = storage_backend(),
-            backend => builtin,
+        ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, Backend),
-            storage => {emqx_ds_storage_bitfield_lts, #{}}
-        }),
         ok = emqx_persistent_session_ds_router:init_tables(),
         ok = emqx_persistent_session_ds:create_tables(),
         ok
     end).
 
--spec is_store_enabled() -> boolean().
+-spec is_persistence_enabled() -> boolean().
-is_store_enabled() ->
+is_persistence_enabled() ->
-    emqx_config:get([persistent_session_store, ds]).
+    emqx_config:get([session_persistence, enable]).
+
+-spec storage_backend() -> emqx_ds:create_db_opts().
+storage_backend() ->
+    storage_backend(emqx_config:get([session_persistence, storage])).
+
+%% Dev-only option: force all messages to go through
+%% `emqx_persistent_session_ds':
+-spec force_ds() -> boolean().
+force_ds() ->
+    emqx_config:get([session_persistence, force_persistence]).
+
+storage_backend(#{
+    builtin := #{enable := true, n_shards := NShards, replication_factor := ReplicationFactor}
+}) ->
+    #{
+        backend => builtin,
+        storage => {emqx_ds_storage_bitfield_lts, #{}},
+        n_shards => NShards,
+        replication_factor => ReplicationFactor
+    };
+storage_backend(#{
+    fdb := #{enable := true} = FDBConfig
+}) ->
+    FDBConfig#{backend => fdb}.
 
 %%--------------------------------------------------------------------
 

apps/emqx/src/emqx_persistent_message_ds_replayer.erl

@@ -19,12 +19,18 @@
 -module(emqx_persistent_message_ds_replayer).
 
 %% API:
--export([new/0, next_packet_id/1, replay/2, commit_offset/3, poll/3, n_inflight/1]).
+-export([new/0, open/1, next_packet_id/1, n_inflight/1]).
+
+-export([poll/4, replay/2, commit_offset/4]).
+
+-export([seqno_to_packet_id/1, packet_id_to_seqno/2]).
+
+-export([committed_until/2]).
 
 %% internal exports:
 -export([]).
 
--export_type([inflight/0]).
+-export_type([inflight/0, seqno/0]).
 
 -include_lib("emqx/include/logger.hrl").
 -include("emqx_persistent_session_ds.hrl").
@@ -34,6 +40,13 @@
 -include_lib("eunit/include/eunit.hrl").
 -endif.
 
+-define(EPOCH_SIZE, 16#10000).
+
+-define(ACK, 0).
+-define(COMP, 1).
+
+-define(TRACK_FLAG(WHICH), (1 bsl WHICH)).
+
 %%================================================================================
 %% Type declarations
 %%================================================================================
@@ -41,23 +54,23 @@
 %% Note: sequence numbers are monotonic; they don't wrap around:
 -type seqno() :: non_neg_integer().
 
--record(range, {
+-type track() :: ack | comp.
-    stream :: emqx_ds:stream(),
+-type commit_type() :: rec.
-    first :: seqno(),
-    last :: seqno(),
-    iterator_next :: emqx_ds:iterator() | undefined
-}).
-
--type range() :: #range{}.
 
 -record(inflight, {
-    next_seqno = 0 :: seqno(),
+    next_seqno = 1 :: seqno(),
-    acked_seqno = 0 :: seqno(),
+    commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()},
-    offset_ranges = [] :: [range()]
+    %% Ranges are sorted in ascending order of their sequence numbers.
+    offset_ranges = [] :: [ds_pubrange()]
 }).
 
 -opaque inflight() :: #inflight{}.
 
+-type reply_fun() :: fun(
+    (seqno(), emqx_types:message()) ->
+        emqx_session:replies() | {_AdvanceSeqno :: false, emqx_session:replies()}
+).
+
 %%================================================================================
 %% API funcions
 %%================================================================================
@@ -66,85 +79,98 @@
 new() ->
     #inflight{}.
 
+-spec open(emqx_persistent_session_ds:id()) -> inflight().
+open(SessionId) ->
+    {Ranges, RecUntil} = ro_transaction(
+        fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end
+    ),
+    {Commits, NextSeqno} = compute_inflight_range(Ranges),
+    #inflight{
+        commits = Commits#{rec => RecUntil},
+        next_seqno = NextSeqno,
+        offset_ranges = Ranges
+    }.
+
 -spec next_packet_id(inflight()) -> {emqx_types:packet_id(), inflight()}.
-next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqNo}) ->
+next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) ->
-    Inflight = Inflight0#inflight{next_seqno = LastSeqNo + 1},
+    Inflight = Inflight0#inflight{next_seqno = next_seqno(LastSeqno)},
-    case LastSeqNo rem 16#10000 of
+    {seqno_to_packet_id(LastSeqno), Inflight}.
-        0 ->
-            %% We skip sequence numbers that lead to PacketId = 0 to
-            %% simplify math. Note: it leads to occasional gaps in the
-            %% sequence numbers.
-            next_packet_id(Inflight);
-        PacketId ->
-            {PacketId, Inflight}
-    end.
 
 -spec n_inflight(inflight()) -> non_neg_integer().
-n_inflight(#inflight{next_seqno = NextSeqNo, acked_seqno = AckedSeqno}) ->
+n_inflight(#inflight{offset_ranges = Ranges}) ->
-    %% NOTE: this function assumes that gaps in the sequence ID occur
+    %% TODO
-    %% _only_ when the packet ID wraps:
+    %% This is not very efficient. Instead, we can take the maximum of
-    case AckedSeqno >= ((NextSeqNo bsr 16) bsl 16) of
+    %% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`.
-        true ->
+    %% This won't be exact number but a pessimistic estimate, but this way we
-            NextSeqNo - AckedSeqno;
+    %% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2
-        false ->
+    %% messages for some reason. For that to work, we need to additionally track
-            NextSeqNo - AckedSeqno - 1
+    %% actual `AckedUntil` / `CompUntil` during `commit_offset/4`.
-    end.
+    lists:foldl(
-
+        fun
--spec replay(emqx_persistent_session_ds:id(), inflight()) ->
+            (#ds_pubrange{type = ?T_CHECKPOINT}, N) ->
-    emqx_session:replies().
-replay(_SessionId, _Inflight = #inflight{offset_ranges = _Ranges}) ->
-    [].
-
--spec commit_offset(emqx_persistent_session_ds:id(), emqx_types:packet_id(), inflight()) ->
-    {_IsValidOffset :: boolean(), inflight()}.
-commit_offset(
-    SessionId,
-    PacketId,
-    Inflight0 = #inflight{
-        acked_seqno = AckedSeqno0, next_seqno = NextSeqNo, offset_ranges = Ranges0
-    }
-) ->
-    AckedSeqno =
-        case packet_id_to_seqno(NextSeqNo, PacketId) of
-            N when N > AckedSeqno0; AckedSeqno0 =:= 0 ->
                 N;
-            OutOfRange ->
+            (#ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until}, N) ->
-                ?SLOG(warning, #{
+                N + range_size(First, Until)
-                    msg => "out-of-order_ack",
-                    prev_seqno => AckedSeqno0,
-                    acked_seqno => OutOfRange,
-                    next_seqno => NextSeqNo,
-                    packet_id => PacketId
-                }),
-                AckedSeqno0
         end,
-    Ranges = lists:filter(
+        0,
-        fun(#range{stream = Stream, last = LastSeqno, iterator_next = ItNext}) ->
+        Ranges
-            case LastSeqno =< AckedSeqno of
+    ).
-                true ->
+
-                    %% This range has been fully
+-spec replay(reply_fun(), inflight()) -> {emqx_session:replies(), inflight()}.
-                    %% acked. Remove it and replace saved
+replay(ReplyFun, Inflight0 = #inflight{offset_ranges = Ranges0}) ->
-                    %% iterator with the trailing iterator.
+    {Ranges, Replies} = lists:mapfoldr(
-                    update_iterator(SessionId, Stream, ItNext),
+        fun(Range, Acc) ->
-                    false;
+            replay_range(ReplyFun, Range, Acc)
-                false ->
-                    %% This range still has unacked
-                    %% messages:
-                    true
-            end
         end,
+        [],
         Ranges0
     ),
-    Inflight = Inflight0#inflight{acked_seqno = AckedSeqno, offset_ranges = Ranges},
+    Inflight = Inflight0#inflight{offset_ranges = Ranges},
-    {true, Inflight}.
+    {Replies, Inflight}.
 
--spec poll(emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
+-spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) ->
+    {_IsValidOffset :: boolean(), inflight()}
+when
+    Offset :: track() | commit_type().
+commit_offset(
+    SessionId,
+    Track,
+    PacketId,
+    Inflight0 = #inflight{commits = Commits}
+) when Track == ack orelse Track == comp ->
+    case validate_commit(Track, PacketId, Inflight0) of
+        CommitUntil when is_integer(CommitUntil) ->
+            %% TODO
+            %% We do not preserve `CommitUntil` in the database. Instead, we discard
+            %% fully acked ranges from the database. In effect, this means that the
+            %% most recent `CommitUntil` the client has sent may be lost in case of a
+            %% crash or client loss.
+            Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}},
+            Inflight = discard_committed(SessionId, Inflight1),
+            {true, Inflight};
+        false ->
+            {false, Inflight0}
+    end;
+commit_offset(
+    SessionId,
+    CommitType = rec,
+    PacketId,
+    Inflight0 = #inflight{commits = Commits}
+) ->
+    case validate_commit(CommitType, PacketId, Inflight0) of
+        CommitUntil when is_integer(CommitUntil) ->
+            update_committed_offset(SessionId, CommitType, CommitUntil),
+            Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}},
+            {true, Inflight};
+        false ->
+            {false, Inflight0}
+    end.
+
+-spec poll(reply_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
     {emqx_session:replies(), inflight()}.
-poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff ->
+poll(ReplyFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE ->
-    #inflight{next_seqno = NextSeqNo0, acked_seqno = AckedSeqno} =
-        Inflight0,
     FetchThreshold = max(1, WindowSize div 2),
-    FreeSpace = AckedSeqno + WindowSize - NextSeqNo0,
+    FreeSpace = WindowSize - n_inflight(Inflight0),
     case FreeSpace >= FetchThreshold of
         false ->
             %% TODO: this branch is meant to avoid fetching data from
@@ -153,10 +179,27 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
             %% client get stuck even?
             {[], Inflight0};
         true ->
+            %% TODO: Wrap this in `mria:async_dirty/2`?
             Streams = shuffle(get_streams(SessionId)),
-            fetch(SessionId, Inflight0, Streams, FreeSpace, [])
+            fetch(ReplyFun, SessionId, Inflight0, Streams, FreeSpace, [])
     end.
 
+%% Which seqno this track is committed until.
+%% "Until" means this is first seqno that is _not yet committed_ for this track.
+-spec committed_until(track() | commit_type(), inflight()) -> seqno().
+committed_until(Track, #inflight{commits = Commits}) ->
+    maps:get(Track, Commits).
+
+-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
+seqno_to_packet_id(Seqno) ->
+    Seqno rem ?EPOCH_SIZE.
+
+%% Reconstruct session counter by adding most significant bits from
+%% the current counter to the packet id.
+-spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno().
+packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) ->
+    packet_id_to_seqno_(NextSeqno, PacketId).
+
 %%================================================================================
 %% Internal exports
 %%================================================================================
@@ -165,87 +208,329 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
 %% Internal functions
 %%================================================================================
 
-fetch(_SessionId, Inflight, _Streams = [], _N, Acc) ->
+compute_inflight_range([]) ->
-    {lists:reverse(Acc), Inflight};
+    {#{ack => 1, comp => 1}, 1};
-fetch(_SessionId, Inflight, _Streams, 0, Acc) ->
+compute_inflight_range(Ranges) ->
-    {lists:reverse(Acc), Inflight};
+    _RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges),
-fetch(SessionId, Inflight0, [Stream | Streams], N, Publishes0) ->
+    AckedUntil = find_committed_until(ack, Ranges),
-    #inflight{next_seqno = FirstSeqNo, offset_ranges = Ranges0} = Inflight0,
+    CompUntil = find_committed_until(comp, Ranges),
-    ItBegin = get_last_iterator(SessionId, Stream, Ranges0),
+    Commits = #{
-    {ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N),
+        ack => emqx_maybe:define(AckedUntil, LastSeqno),
-    {NMessages, Publishes, Inflight1} =
+        comp => emqx_maybe:define(CompUntil, LastSeqno)
-        lists:foldl(
-            fun(Msg, {N0, PubAcc0, InflightAcc0}) ->
-                {PacketId, InflightAcc} = next_packet_id(InflightAcc0),
-                PubAcc = [{PacketId, Msg} | PubAcc0],
-                {N0 + 1, PubAcc, InflightAcc}
-            end,
-            {0, Publishes0, Inflight0},
-            Messages
-        ),
-    #inflight{next_seqno = LastSeqNo} = Inflight1,
-    case NMessages > 0 of
-        true ->
-            Range = #range{
-                first = FirstSeqNo,
-                last = LastSeqNo - 1,
-                stream = Stream,
-                iterator_next = ItEnd
     },
-            Inflight = Inflight1#inflight{offset_ranges = Ranges0 ++ [Range]},
+    {Commits, LastSeqno}.
-            fetch(SessionId, Inflight, Streams, N - NMessages, Publishes);
-        false ->
-            fetch(SessionId, Inflight1, Streams, N, Publishes)
-    end.
 
--spec update_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream(), emqx_ds:iterator()) -> ok.
+find_committed_until(Track, Ranges) ->
-update_iterator(DSSessionId, Stream, Iterator) ->
+    RangesUncommitted = lists:dropwhile(
-    %% Workaround: we convert `Stream' to a binary before attempting to store it in
+        fun(Range) ->
-    %% mnesia(rocksdb) because of a bug in `mnesia_rocksdb' when trying to do
+            case Range of
-    %% `mnesia:dirty_all_keys' later.
+                #ds_pubrange{type = ?T_CHECKPOINT} ->
-    StreamBin = term_to_binary(Stream),
+                    true;
-    mria:dirty_write(?SESSION_ITER_TAB, #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator}).
+                #ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} ->
-
+                    not has_track(Track, Tracks)
-get_last_iterator(SessionId, Stream, Ranges) ->
+            end
-    case lists:keyfind(Stream, #range.stream, lists:reverse(Ranges)) of
-        false ->
-            get_iterator(SessionId, Stream);
-        #range{iterator_next = Next} ->
-            Next
-    end.
-
--spec get_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream()) -> emqx_ds:iterator().
-get_iterator(DSSessionId, Stream) ->
-    %% See comment in `update_iterator'.
-    StreamBin = term_to_binary(Stream),
-    Id = {DSSessionId, StreamBin},
-    [#ds_iter{iter = It}] = mnesia:dirty_read(?SESSION_ITER_TAB, Id),
-    It.
-
--spec get_streams(emqx_persistent_session_ds:id()) -> [emqx_ds:stream()].
-get_streams(SessionId) ->
-    lists:map(
-        fun(#ds_stream{stream = Stream}) ->
-            Stream
         end,
-        mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId)
+        Ranges
+    ),
+    case RangesUncommitted of
+        [#ds_pubrange{id = {_, CommittedUntil}} | _] ->
+            CommittedUntil;
+        [] ->
+            undefined
+    end.
+
+-spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()].
+get_ranges(SessionId) ->
+    Pat = erlang:make_tuple(
+        record_info(size, ds_pubrange),
+        '_',
+        [{1, ds_pubrange}, {#ds_pubrange.id, {SessionId, '_'}}]
+    ),
+    mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read).
+
+fetch(ReplyFun, SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 ->
+    #inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0,
+    ItBegin = get_last_iterator(DSStream, Ranges),
+    {ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N),
+    case Messages of
+        [] ->
+            fetch(ReplyFun, SessionId, Inflight0, Streams, N, Acc);
+        _ ->
+            %% We need to preserve the iterator pointing to the beginning of the
+            %% range, so that we can replay it if needed.
+            {Publishes, {UntilSeqno, Tracks}} = publish(ReplyFun, FirstSeqno, Messages),
+            Size = range_size(FirstSeqno, UntilSeqno),
+            Range0 = #ds_pubrange{
+                id = {SessionId, FirstSeqno},
+                type = ?T_INFLIGHT,
+                tracks = Tracks,
+                until = UntilSeqno,
+                stream = DSStream#ds_stream.ref,
+                iterator = ItBegin
+            },
+            ok = preserve_range(Range0),
+            %% ...Yet we need to keep the iterator pointing past the end of the
+            %% range, so that we can pick up where we left off: it will become
+            %% `ItBegin` of the next range for this stream.
+            Range = keep_next_iterator(ItEnd, Range0),
+            Inflight = Inflight0#inflight{
+                next_seqno = UntilSeqno,
+                offset_ranges = Ranges ++ [Range]
+            },
+            fetch(ReplyFun, SessionId, Inflight, Streams, N - Size, [Publishes | Acc])
+    end;
+fetch(_ReplyFun, _SessionId, Inflight, _Streams, _N, Acc) ->
+    Publishes = lists:append(lists:reverse(Acc)),
+    {Publishes, Inflight}.
+
+discard_committed(
+    SessionId,
+    Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0}
+) ->
+    %% TODO: This could be kept and incrementally updated in the inflight state.
+    Checkpoints = find_checkpoints(Ranges0),
+    %% TODO: Wrap this in `mria:async_dirty/2`?
+    Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0),
+    Inflight0#inflight{offset_ranges = Ranges}.
+
+find_checkpoints(Ranges) ->
+    lists:foldl(
+        fun(#ds_pubrange{stream = StreamRef, until = Until}, Acc) ->
+            %% For each stream, remember the last range over this stream.
+            Acc#{StreamRef => Until}
+        end,
+        #{},
+        Ranges
     ).
 
-%% Reconstruct session counter by adding most significant bits from
+discard_committed_ranges(
-%% the current counter to the packet id.
+    SessionId,
--spec packet_id_to_seqno(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer().
+    Commits,
-packet_id_to_seqno(NextSeqNo, PacketId) ->
+    Checkpoints,
-    Epoch = NextSeqNo bsr 16,
+    Ranges = [Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest]
-    case packet_id_to_seqno_(Epoch, PacketId) of
+) ->
-        N when N =< NextSeqNo ->
+    case discard_committed_range(Commits, Range) of
-            N;
+        discard ->
-        _ ->
+            %% This range has been fully committed.
-            packet_id_to_seqno_(Epoch - 1, PacketId)
+            %% Either discard it completely, or preserve the iterator for the next range
+            %% over this stream (i.e. a checkpoint).
+            RangeKept =
+                case maps:get(StreamRef, Checkpoints) of
+                    CP when CP > Until ->
+                        discard_range(Range),
+                        [];
+                    Until ->
+                        [checkpoint_range(Range)]
+                end,
+            %% Since we're (intentionally) not using transactions here, it's important to
+            %% issue database writes in the same order in which ranges are stored: from
+            %% the oldest to the newest. This is also why we need to compute which ranges
+            %% should become checkpoints before we start writing anything.
+            RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest);
+        keep ->
+            %% This range has not been fully committed.
+            [Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)];
+        keep_all ->
+            %% The rest of ranges (if any) still have uncommitted messages.
+            Ranges;
+        TracksLeft ->
+            %% Only some track has been committed.
+            %% Preserve the uncommitted tracks in the database.
+            RangeKept = Range#ds_pubrange{tracks = TracksLeft},
+            preserve_range(restore_first_iterator(RangeKept)),
+            [RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]
+    end;
+discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) ->
+    [].
+
+discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) ->
+    discard;
+discard_committed_range(
+    #{ack := AckedUntil, comp := CompUntil},
+    #ds_pubrange{until = Until}
+) when Until > AckedUntil andalso Until > CompUntil ->
+    keep_all;
+discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) ->
+    case discard_tracks(Commits, Until, Tracks) of
+        0 ->
+            discard;
+        Tracks ->
+            keep;
+        TracksLeft ->
+            TracksLeft
     end.
 
--spec packet_id_to_seqno_(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer().
+discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) ->
-packet_id_to_seqno_(Epoch, PacketId) ->
+    TAck =
-    (Epoch bsl 16) + PacketId.
+        case Until > AckedUntil of
+            true -> ?TRACK_FLAG(?ACK) band Tracks;
+            false -> 0
+        end,
+    TComp =
+        case Until > CompUntil of
+            true -> ?TRACK_FLAG(?COMP) band Tracks;
+            false -> 0
+        end,
+    TAck bor TComp.
+
+replay_range(
+    ReplyFun,
+    Range0 = #ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until, iterator = It},
+    Acc
+) ->
+    Size = range_size(First, Until),
+    {ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
+    %% Asserting that range is consistent with the message storage state.
+    {Replies, {Until, _TracksInitial}} = publish(ReplyFun, First, MessagesUnacked),
+    %% Again, we need to keep the iterator pointing past the end of the
+    %% range, so that we can pick up where we left off.
+    Range = keep_next_iterator(ItNext, Range0),
+    {Range, Replies ++ Acc};
+replay_range(_ReplyFun, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) ->
+    {Range0, Acc}.
+
+validate_commit(
+    Track,
+    PacketId,
+    Inflight = #inflight{commits = Commits, next_seqno = NextSeqno}
+) ->
+    Seqno = packet_id_to_seqno_(NextSeqno, PacketId),
+    CommittedUntil = maps:get(Track, Commits),
+    CommitNext = get_commit_next(Track, Inflight),
+    case Seqno >= CommittedUntil andalso Seqno < CommitNext of
+        true ->
+            next_seqno(Seqno);
+        false ->
+            ?SLOG(warning, #{
+                msg => "out-of-order_commit",
+                track => Track,
+                packet_id => PacketId,
+                commit_seqno => Seqno,
+                committed_until => CommittedUntil,
+                commit_next => CommitNext
+            }),
+            false
+    end.
+
+get_commit_next(ack, #inflight{next_seqno = NextSeqno}) ->
+    NextSeqno;
+get_commit_next(rec, #inflight{next_seqno = NextSeqno}) ->
+    NextSeqno;
+get_commit_next(comp, #inflight{commits = Commits}) ->
+    maps:get(rec, Commits).
+
+publish(ReplyFun, FirstSeqno, Messages) ->
+    lists:mapfoldl(
+        fun(Message, {Seqno, TAcc}) ->
+            case ReplyFun(Seqno, Message) of
+                {_Advance = false, Reply} ->
+                    {Reply, {Seqno, TAcc}};
+                Reply ->
+                    NextSeqno = next_seqno(Seqno),
+                    NextTAcc = add_msg_track(Message, TAcc),
+                    {Reply, {NextSeqno, NextTAcc}}
+            end
+        end,
+        {FirstSeqno, 0},
+        Messages
+    ).
+
+add_msg_track(Message, Tracks) ->
+    case emqx_message:qos(Message) of
+        1 -> ?TRACK_FLAG(?ACK) bor Tracks;
+        2 -> ?TRACK_FLAG(?COMP) bor Tracks;
+        _ -> Tracks
+    end.
+
+keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) ->
+    Range#ds_pubrange{
+        iterator = ItNext,
+        %% We need to keep the first iterator around, in case we need to preserve
+        %% this range again, updating still uncommitted tracks it's part of.
+        misc = Misc#{iterator_first => ItFirst}
+    }.
+
+restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) ->
+    Range#ds_pubrange{
+        iterator = ItFirst,
+        misc = maps:remove(iterator_first, Misc)
+    }.
+
+-spec preserve_range(ds_pubrange()) -> ok.
+preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) ->
+    mria:dirty_write(?SESSION_PUBRANGE_TAB, Range).
+
+has_track(ack, Tracks) ->
+    (?TRACK_FLAG(?ACK) band Tracks) > 0;
+has_track(comp, Tracks) ->
+    (?TRACK_FLAG(?COMP) band Tracks) > 0.
+
+-spec discard_range(ds_pubrange()) -> ok.
+discard_range(#ds_pubrange{id = RangeId}) ->
+    mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId).
+
+-spec checkpoint_range(ds_pubrange()) -> ds_pubrange().
+checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) ->
+    Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}},
+    ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range),
+    Range;
+checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) ->
+    %% This range should have been checkpointed already.
+    Range.
+
+get_last_iterator(DSStream = #ds_stream{ref = StreamRef}, Ranges) ->
+    case lists:keyfind(StreamRef, #ds_pubrange.stream, lists:reverse(Ranges)) of
+        false ->
+            DSStream#ds_stream.beginning;
+        #ds_pubrange{iterator = ItNext} ->
+            ItNext
+    end.
+
+-spec get_streams(emqx_persistent_session_ds:id()) -> [ds_stream()].
+get_streams(SessionId) ->
+    mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId).
+
+-spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno().
+get_committed_offset(SessionId, Name) ->
+    case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of
+        [] ->
+            1;
+        [#ds_committed_offset{until = Seqno}] ->
+            Seqno
+    end.
+
+-spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok.
+update_committed_offset(SessionId, Name, Until) ->
+    mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{
+        id = {SessionId, Name}, until = Until
+    }).
+
+next_seqno(Seqno) ->
+    NextSeqno = Seqno + 1,
+    case seqno_to_packet_id(NextSeqno) of
+        0 ->
+            %% We skip sequence numbers that lead to PacketId = 0 to
+            %% simplify math. Note: it leads to occasional gaps in the
+            %% sequence numbers.
+            NextSeqno + 1;
+        _ ->
+            NextSeqno
+    end.
+
+packet_id_to_seqno_(NextSeqno, PacketId) ->
+    Epoch = NextSeqno bsr 16,
+    case (Epoch bsl 16) + PacketId of
+        N when N =< NextSeqno ->
+            N;
+        N ->
+            N - ?EPOCH_SIZE
+    end.
+
+range_size(FirstSeqno, UntilSeqno) ->
+    %% This function assumes that gaps in the sequence ID occur _only_ when the
+    %% packet ID wraps.
+    Size = UntilSeqno - FirstSeqno,
+    Size + (FirstSeqno bsr 16) - (UntilSeqno bsr 16).
 
 -spec shuffle([A]) -> [A].
 shuffle(L0) ->
@@ -259,24 +544,28 @@ shuffle(L0) ->
     {_, L} = lists:unzip(L2),
     L.
 
+ro_transaction(Fun) ->
+    {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
+    Res.
+
 -ifdef(TEST).
 
 %% This test only tests boundary conditions (to make sure property-based test didn't skip them):
 packet_id_to_seqno_test() ->
     %% Packet ID = 1; first epoch:
-    ?assertEqual(1, packet_id_to_seqno(1, 1)),
+    ?assertEqual(1, packet_id_to_seqno_(1, 1)),
-    ?assertEqual(1, packet_id_to_seqno(10, 1)),
+    ?assertEqual(1, packet_id_to_seqno_(10, 1)),
-    ?assertEqual(1, packet_id_to_seqno(1 bsl 16 - 1, 1)),
+    ?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)),
-    ?assertEqual(1, packet_id_to_seqno(1 bsl 16, 1)),
+    ?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)),
     %% Packet ID = 1; second and 3rd epochs:
-    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(1 bsl 16 + 1, 1)),
+    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)),
-    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(2 bsl 16, 1)),
+    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)),
-    ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno(2 bsl 16 + 1, 1)),
+    ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)),
     %% Packet ID = 16#ffff:
     PID = 1 bsl 16 - 1,
-    ?assertEqual(PID, packet_id_to_seqno(PID, PID)),
+    ?assertEqual(PID, packet_id_to_seqno_(PID, PID)),
-    ?assertEqual(PID, packet_id_to_seqno(1 bsl 16, PID)),
+    ?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)),
-    ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno(2 bsl 16, PID)),
+    ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)),
     ok.
 
 packet_id_to_seqno_test_() ->
@@ -291,8 +580,8 @@ packet_id_to_seqno_prop() ->
             SeqNo,
             seqno_gen(NextSeqNo),
             begin
-                PacketId = SeqNo rem 16#10000,
+                PacketId = seqno_to_packet_id(SeqNo),
-                ?assertEqual(SeqNo, packet_id_to_seqno(NextSeqNo, PacketId)),
+                ?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)),
                 true
             end
         )
@@ -311,4 +600,55 @@ seqno_gen(NextSeqNo) ->
     Max = max(0, NextSeqNo - 1),
     range(Min, Max).
 
+range_size_test_() ->
+    [
+        ?_assertEqual(0, range_size(42, 42)),
+        ?_assertEqual(1, range_size(42, 43)),
+        ?_assertEqual(1, range_size(16#ffff, 16#10001)),
+        ?_assertEqual(16#ffff - 456 + 123, range_size(16#1f0000 + 456, 16#200000 + 123))
+    ].
+
+compute_inflight_range_test_() ->
+    [
+        ?_assertEqual(
+            {#{ack => 1, comp => 1}, 1},
+            compute_inflight_range([])
+        ),
+        ?_assertEqual(
+            {#{ack => 12, comp => 13}, 42},
+            compute_inflight_range([
+                #ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
+                #ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
+                #ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
+                #ds_pubrange{
+                    id = {<<>>, 12},
+                    until = 13,
+                    type = ?T_INFLIGHT,
+                    tracks = ?TRACK_FLAG(?ACK)
+                },
+                #ds_pubrange{
+                    id = {<<>>, 13},
+                    until = 20,
+                    type = ?T_INFLIGHT,
+                    tracks = ?TRACK_FLAG(?COMP)
+                },
+                #ds_pubrange{
+                    id = {<<>>, 20},
+                    until = 42,
+                    type = ?T_INFLIGHT,
+                    tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)
+                }
+            ])
+        ),
+        ?_assertEqual(
+            {#{ack => 13, comp => 13}, 13},
+            compute_inflight_range([
+                #ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
+                #ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
+                #ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
+                #ds_pubrange{id = {<<>>, 12}, until = 13, type = ?T_CHECKPOINT}
+            ])
+        )
+    ].
+
 -endif.

apps/emqx/src/emqx_persistent_session_ds.erl

@@ -56,13 +56,16 @@
     deliver/3,
     replay/3,
     handle_timeout/3,
-    disconnect/1,
+    disconnect/2,
     terminate/2
 ]).
 
 %% session table operations
 -export([create_tables/0]).
 
+%% internal export used by session GC process
+-export([destroy_session/1]).
+
 %% Remove me later (satisfy checks for an unused BPAPI)
 -export([
     do_open_iterator/3,
@@ -70,24 +73,27 @@
     do_ensure_all_iterators_closed/1
 ]).
 
+-export([print_session/1]).
+
 -ifdef(TEST).
 -export([
-    session_open/1,
+    session_open/2,
     list_all_sessions/0,
     list_all_subscriptions/0,
     list_all_streams/0,
-    list_all_iterators/0
+    list_all_pubranges/0
 ]).
 -endif.
 
 %% Currently, this is the clientid.  We avoid `emqx_types:clientid()' because that can be
 %% an atom, in theory (?).
 -type id() :: binary().
--type topic_filter() :: emqx_ds:topic_filter().
+-type topic_filter() :: emqx_types:topic().
+-type topic_filter_words() :: emqx_ds:topic_filter().
 -type subscription_id() :: {id(), topic_filter()}.
 -type subscription() :: #{
     start_time := emqx_ds:time(),
-    propts := map(),
+    props := map(),
     extra := map()
 }.
 -type session() :: #{
@@ -95,23 +101,26 @@
     id := id(),
     %% When the session was created
     created_at := timestamp(),
-    %% When the session should expire
+    %% When the client was last considered alive
-    expires_at := timestamp() | never,
+    last_alive_at := timestamp(),
     %% Client’s Subscriptions.
-    iterators := #{topic() => subscription()},
+    subscriptions := #{topic_filter() => subscription()},
     %% Inflight messages
     inflight := emqx_persistent_message_ds_replayer:inflight(),
     %% Receive maximum
     receive_maximum := pos_integer(),
+    %% Connection Info
+    conninfo := emqx_types:conninfo(),
     %%
     props := map()
 }.
 
 -type timestamp() :: emqx_utils_calendar:epoch_millisecond().
--type topic() :: emqx_types:topic().
+-type millisecond() :: non_neg_integer().
 -type clientinfo() :: emqx_types:clientinfo().
 -type conninfo() :: emqx_session:conninfo().
 -type replies() :: emqx_session:replies().
+-type timer() :: pull | get_streams | bump_last_alive_at.
 
 -define(STATS_KEYS, [
     subscriptions_cnt,
@@ -121,6 +130,12 @@
     next_pkt_id
 ]).
 
+-define(IS_EXPIRED(NOW_MS, LAST_ALIVE_AT, EI),
+    (is_number(LAST_ALIVE_AT) andalso
+        is_number(EI) andalso
+        (NOW_MS >= LAST_ALIVE_AT + EI))
+).
+
 -export_type([id/0]).
 
 %%
@@ -142,7 +157,7 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) ->
     %% somehow isolate those idling not-yet-expired sessions into a separate process
     %% space, and move this call back into `emqx_cm` where it belongs.
     ok = emqx_cm:discard_session(ClientID),
-    case open_session(ClientID) of
+    case session_open(ClientID, ConnInfo) of
         Session0 = #{} ->
             ensure_timers(),
             ReceiveMaximum = receive_maximum(ConnInfo),
@@ -153,24 +168,13 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) ->
     end.
 
 ensure_session(ClientID, ConnInfo, Conf) ->
-    {ok, Session, #{}} = session_ensure_new(ClientID, Conf),
+    Session = session_ensure_new(ClientID, ConnInfo, Conf),
     ReceiveMaximum = receive_maximum(ConnInfo),
-    Session#{iterators => #{}, receive_maximum => ReceiveMaximum}.
+    Session#{
-
+        conninfo => ConnInfo,
-open_session(ClientID) ->
+        receive_maximum => ReceiveMaximum,
-    case session_open(ClientID) of
+        subscriptions => #{}
-        {ok, Session, Subscriptions} ->
+    }.
-            Session#{iterators => prep_subscriptions(Subscriptions)};
-        false ->
-            false
-    end.
-
-prep_subscriptions(Subscriptions) ->
-    maps:fold(
-        fun(Topic, Subscription, Acc) -> Acc#{emqx_topic:join(Topic) => Subscription} end,
-        #{},
-        Subscriptions
-    ).
 
 -spec destroy(session() | clientinfo()) -> ok.
 destroy(#{id := ClientID}) ->
@@ -195,9 +199,9 @@ info(created_at, #{created_at := CreatedAt}) ->
     CreatedAt;
 info(is_persistent, #{}) ->
     true;
-info(subscriptions, #{iterators := Iters}) ->
+info(subscriptions, #{subscriptions := Iters}) ->
     maps:map(fun(_, #{props := SubOpts}) -> SubOpts end, Iters);
-info(subscriptions_cnt, #{iterators := Iters}) ->
+info(subscriptions_cnt, #{subscriptions := Iters}) ->
     maps:size(Iters);
 info(subscriptions_max, #{props := Conf}) ->
     maps:get(max_subscriptions, Conf);
@@ -235,51 +239,71 @@ info(await_rel_timeout, #{props := Conf}) ->
 stats(Session) ->
     info(?STATS_KEYS, Session).
 
+%% Debug/troubleshooting
+-spec print_session(emqx_types:client_id()) -> map() | undefined.
+print_session(ClientId) ->
+    catch ro_transaction(
+        fun() ->
+            case mnesia:read(?SESSION_TAB, ClientId) of
+                [Session] ->
+                    #{
+                        session => Session,
+                        streams => mnesia:read(?SESSION_STREAM_TAB, ClientId),
+                        pubranges => session_read_pubranges(ClientId),
+                        offsets => session_read_offsets(ClientId),
+                        subscriptions => session_read_subscriptions(ClientId)
+                    };
+                [] ->
+                    undefined
+            end
+        end
+    ).
+
 %%--------------------------------------------------------------------
 %% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE
 %%--------------------------------------------------------------------
 
--spec subscribe(topic(), emqx_types:subopts(), session()) ->
+-spec subscribe(topic_filter(), emqx_types:subopts(), session()) ->
     {ok, session()} | {error, emqx_types:reason_code()}.
 subscribe(
     TopicFilter,
     SubOpts,
-    Session = #{id := ID, iterators := Iters}
+    Session = #{id := ID, subscriptions := Subs}
-) when is_map_key(TopicFilter, Iters) ->
+) when is_map_key(TopicFilter, Subs) ->
-    Iterator = maps:get(TopicFilter, Iters),
+    Subscription = maps:get(TopicFilter, Subs),
-    NIterator = update_subscription(TopicFilter, Iterator, SubOpts, ID),
+    NSubscription = update_subscription(TopicFilter, Subscription, SubOpts, ID),
-    {ok, Session#{iterators := Iters#{TopicFilter => NIterator}}};
+    {ok, Session#{subscriptions := Subs#{TopicFilter => NSubscription}}};
 subscribe(
     TopicFilter,
     SubOpts,
-    Session = #{id := ID, iterators := Iters}
+    Session = #{id := ID, subscriptions := Subs}
 ) ->
     % TODO: max_subscriptions
-    Iterator = add_subscription(TopicFilter, SubOpts, ID),
+    Subscription = add_subscription(TopicFilter, SubOpts, ID),
-    {ok, Session#{iterators := Iters#{TopicFilter => Iterator}}}.
+    {ok, Session#{subscriptions := Subs#{TopicFilter => Subscription}}}.
 
--spec unsubscribe(topic(), session()) ->
+-spec unsubscribe(topic_filter(), session()) ->
     {ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}.
 unsubscribe(
     TopicFilter,
-    Session = #{id := ID, iterators := Iters}
+    Session = #{id := ID, subscriptions := Subs}
-) when is_map_key(TopicFilter, Iters) ->
+) when is_map_key(TopicFilter, Subs) ->
-    Iterator = maps:get(TopicFilter, Iters),
+    Subscription = maps:get(TopicFilter, Subs),
-    SubOpts = maps:get(props, Iterator),
+    SubOpts = maps:get(props, Subscription),
     ok = del_subscription(TopicFilter, ID),
-    {ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts};
+    {ok, Session#{subscriptions := maps:remove(TopicFilter, Subs)}, SubOpts};
 unsubscribe(
     _TopicFilter,
     _Session = #{}
 ) ->
     {error, ?RC_NO_SUBSCRIPTION_EXISTED}.
 
--spec get_subscription(topic(), session()) ->
+-spec get_subscription(topic_filter(), session()) ->
     emqx_types:subopts() | undefined.
-get_subscription(TopicFilter, #{iterators := Iters}) ->
+get_subscription(TopicFilter, #{subscriptions := Subs}) ->
-    case maps:get(TopicFilter, Iters, undefined) of
+    case maps:get(TopicFilter, Subs, undefined) of
-        Iterator = #{} ->
+        Subscription = #{} ->
-            maps:get(props, Iterator);
+            maps:get(props, Subscription);
         undefined ->
             undefined
     end.
@@ -289,12 +313,12 @@ get_subscription(TopicFilter, #{iterators := Iters}) ->
 %%--------------------------------------------------------------------
 
 -spec publish(emqx_types:packet_id(), emqx_types:message(), session()) ->
-    {ok, emqx_types:publish_result(), replies(), session()}
+    {ok, emqx_types:publish_result(), session()}
     | {error, emqx_types:reason_code()}.
 publish(_PacketId, Msg, Session) ->
-    %% TODO:
+    %% TODO: QoS2
     Result = emqx_broker:publish(Msg),
-    {ok, Result, [], Session}.
+    {ok, Result, Session}.
 
 %%--------------------------------------------------------------------
 %% Client -> Broker: PUBACK
@@ -307,12 +331,13 @@ publish(_PacketId, Msg, Session) ->
     {ok, emqx_types:message(), replies(), session()}
     | {error, emqx_types:reason_code()}.
 puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
-    case emqx_persistent_message_ds_replayer:commit_offset(Id, PacketId, Inflight0) of
+    case emqx_persistent_message_ds_replayer:commit_offset(Id, ack, PacketId, Inflight0) of
         {true, Inflight} ->
             %% TODO
-            Msg = #message{},
+            Msg = emqx_message:make(Id, <<>>, <<>>),
             {ok, Msg, [], Session#{inflight => Inflight}};
         {false, _} ->
+            %% Invalid Packet Id
             {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
     end.
 
@@ -323,9 +348,16 @@ puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
 -spec pubrec(emqx_types:packet_id(), session()) ->
     {ok, emqx_types:message(), session()}
     | {error, emqx_types:reason_code()}.
-pubrec(_PacketId, _Session = #{}) ->
+pubrec(PacketId, Session = #{id := Id, inflight := Inflight0}) ->
-    % TODO: stub
+    case emqx_persistent_message_ds_replayer:commit_offset(Id, rec, PacketId, Inflight0) of
-    {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
+        {true, Inflight} ->
+            %% TODO
+            Msg = emqx_message:make(Id, <<>>, <<>>),
+            {ok, Msg, Session#{inflight => Inflight}};
+        {false, _} ->
+            %% Invalid Packet Id
+            {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
+    end.
 
 %%--------------------------------------------------------------------
 %% Client -> Broker: PUBREL
@@ -344,16 +376,23 @@ pubrel(_PacketId, Session = #{}) ->
 -spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) ->
     {ok, emqx_types:message(), replies(), session()}
     | {error, emqx_types:reason_code()}.
-pubcomp(_ClientInfo, _PacketId, _Session = #{}) ->
+pubcomp(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
-    % TODO: stub
+    case emqx_persistent_message_ds_replayer:commit_offset(Id, comp, PacketId, Inflight0) of
-    {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
+        {true, Inflight} ->
+            %% TODO
+            Msg = emqx_message:make(Id, <<>>, <<>>),
+            {ok, Msg, [], Session#{inflight => Inflight}};
+        {false, _} ->
+            %% Invalid Packet Id
+            {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
+    end.
 
 %%--------------------------------------------------------------------
 
 -spec deliver(clientinfo(), [emqx_types:deliver()], session()) ->
     {ok, replies(), session()}.
 deliver(_ClientInfo, _Delivers, Session) ->
-    %% TODO: QoS0 and system messages end up here.
+    %% TODO: system messages end up here.
     {ok, [], Session}.
 
 -spec handle_timeout(clientinfo(), _Timeout, session()) ->
@@ -363,43 +402,82 @@ handle_timeout(
     pull,
     Session = #{id := Id, inflight := Inflight0, receive_maximum := ReceiveMaximum}
 ) ->
-    {Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(Id, Inflight0, ReceiveMaximum),
+    {Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(
-    %% TODO: make these values configurable:
+        fun
+            (_Seqno, Message = #message{qos = ?QOS_0}) ->
+                {false, {undefined, Message}};
+            (Seqno, Message) ->
+                PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
+                {PacketId, Message}
+        end,
+        Id,
+        Inflight0,
+        ReceiveMaximum
+    ),
+    IdlePollInterval = emqx_config:get([session_persistence, idle_poll_interval]),
     Timeout =
         case Publishes of
             [] ->
-                100;
+                IdlePollInterval;
             [_ | _] ->
                 0
         end,
     ensure_timer(pull, Timeout),
-    {ok, Publishes, Session#{inflight => Inflight}};
+    {ok, Publishes, Session#{inflight := Inflight}};
-handle_timeout(_ClientInfo, get_streams, Session = #{id := Id}) ->
+handle_timeout(_ClientInfo, get_streams, Session) ->
-    renew_streams(Id),
+    renew_streams(Session),
     ensure_timer(get_streams),
+    {ok, [], Session};
+handle_timeout(_ClientInfo, bump_last_alive_at, Session0) ->
+    %% Note: we take a pessimistic approach here and assume that the client will be alive
+    %% until the next bump timeout.  With this, we avoid garbage collecting this session
+    %% too early in case the session/connection/node crashes earlier without having time
+    %% to commit the time.
+    BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
+    EstimatedLastAliveAt = now_ms() + BumpInterval,
+    Session = session_set_last_alive_at_trans(Session0, EstimatedLastAliveAt),
+    ensure_timer(bump_last_alive_at),
     {ok, [], Session}.
 
 -spec replay(clientinfo(), [], session()) ->
     {ok, replies(), session()}.
-replay(_ClientInfo, [], Session = #{}) ->
+replay(_ClientInfo, [], Session = #{inflight := Inflight0}) ->
-    {ok, [], Session}.
+    AckedUntil = emqx_persistent_message_ds_replayer:committed_until(ack, Inflight0),
+    RecUntil = emqx_persistent_message_ds_replayer:committed_until(rec, Inflight0),
+    CompUntil = emqx_persistent_message_ds_replayer:committed_until(comp, Inflight0),
+    ReplyFun = fun
+        (_Seqno, #message{qos = ?QOS_0}) ->
+            {false, []};
+        (Seqno, #message{qos = ?QOS_1}) when Seqno < AckedUntil ->
+            [];
+        (Seqno, #message{qos = ?QOS_2}) when Seqno < CompUntil ->
+            [];
+        (Seqno, #message{qos = ?QOS_2}) when Seqno < RecUntil ->
+            PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
+            {pubrel, PacketId};
+        (Seqno, Message) ->
+            PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
+            {PacketId, emqx_message:set_flag(dup, true, Message)}
+    end,
+    {Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(ReplyFun, Inflight0),
+    {ok, Replies, Session#{inflight := Inflight}}.
 
 %%--------------------------------------------------------------------
 
--spec disconnect(session()) -> {shutdown, session()}.
+-spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}.
-disconnect(Session = #{}) ->
+disconnect(Session0, ConnInfo) ->
+    Session = session_set_last_alive_at_trans(Session0, ConnInfo, now_ms()),
     {shutdown, Session}.
 
 -spec terminate(Reason :: term(), session()) -> ok.
 terminate(_Reason, _Session = #{}) ->
-    % TODO: close iterators
     ok.
 
 %%--------------------------------------------------------------------
 
--spec add_subscription(topic(), emqx_types:subopts(), id()) ->
+-spec add_subscription(topic_filter(), emqx_types:subopts(), id()) ->
     subscription().
-add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
+add_subscription(TopicFilter, SubOpts, DSSessionID) ->
     %% N.B.: we chose to update the router before adding the subscription to the
     %% session/iterator table.  The reasoning for this is as follows:
     %%
@@ -418,8 +496,7 @@ add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
     %% since it is guarded by a transaction context: we consider a subscription
     %% operation to be successful if it ended up changing this table.  Both router
     %% and iterator information can be reconstructed from this table, if needed.
-    ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID),
+    ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, DSSessionID),
-    TopicFilter = emqx_topic:words(TopicFilterBin),
     {ok, DSSubExt, IsNew} = session_add_subscription(
         DSSessionID, TopicFilter, SubOpts
     ),
@@ -427,20 +504,19 @@ add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
     %% we'll list streams and open iterators when implementing message replay.
     DSSubExt.
 
--spec update_subscription(topic(), subscription(), emqx_types:subopts(), id()) ->
+-spec update_subscription(topic_filter(), subscription(), emqx_types:subopts(), id()) ->
     subscription().
-update_subscription(TopicFilterBin, DSSubExt, SubOpts, DSSessionID) ->
+update_subscription(TopicFilter, DSSubExt, SubOpts, DSSessionID) ->
-    TopicFilter = emqx_topic:words(TopicFilterBin),
     {ok, NDSSubExt, false} = session_add_subscription(
         DSSessionID, TopicFilter, SubOpts
     ),
     ok = ?tp(persistent_session_ds_iterator_updated, #{sub => DSSubExt}),
     NDSSubExt.
 
--spec del_subscription(topic(), id()) ->
+-spec del_subscription(topic_filter(), id()) ->
     ok.
-del_subscription(TopicFilterBin, DSSessionId) ->
+del_subscription(TopicFilter, DSSessionId) ->
-    TopicFilter = emqx_topic:words(TopicFilterBin),
+    %% TODO: transaction?
     ?tp_span(
         persistent_session_ds_subscription_delete,
         #{session_id => DSSessionId},
@@ -449,7 +525,7 @@ del_subscription(TopicFilterBin, DSSessionId) ->
     ?tp_span(
         persistent_session_ds_subscription_route_delete,
         #{session_id => DSSessionId},
-        ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId)
+        ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, DSSessionId)
     ).
 
 %%--------------------------------------------------------------------
@@ -457,10 +533,6 @@ del_subscription(TopicFilterBin, DSSessionId) ->
 %%--------------------------------------------------------------------
 
 create_tables() ->
-    ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{
-        backend => builtin,
-        storage => {emqx_ds_storage_bitfield_lts, #{}}
-    }),
     ok = mria:create_table(
         ?SESSION_TAB,
         [
@@ -492,17 +564,31 @@ create_tables() ->
         ]
     ),
     ok = mria:create_table(
-        ?SESSION_ITER_TAB,
+        ?SESSION_PUBRANGE_TAB,
+        [
+            {rlog_shard, ?DS_MRIA_SHARD},
+            {type, ordered_set},
+            {storage, storage()},
+            {record_name, ds_pubrange},
+            {attributes, record_info(fields, ds_pubrange)}
+        ]
+    ),
+    ok = mria:create_table(
+        ?SESSION_COMMITTED_OFFSET_TAB,
         [
             {rlog_shard, ?DS_MRIA_SHARD},
             {type, set},
             {storage, storage()},
-            {record_name, ds_iter},
+            {record_name, ds_committed_offset},
-            {attributes, record_info(fields, ds_iter)}
+            {attributes, record_info(fields, ds_committed_offset)}
         ]
     ),
     ok = mria:wait_for_tables([
-        ?SESSION_TAB, ?SESSION_SUBSCRIPTIONS_TAB, ?SESSION_STREAM_TAB, ?SESSION_ITER_TAB
+        ?SESSION_TAB,
+        ?SESSION_SUBSCRIPTIONS_TAB,
+        ?SESSION_STREAM_TAB,
+        ?SESSION_PUBRANGE_TAB,
+        ?SESSION_COMMITTED_OFFSET_TAB
     ]),
     ok.
 
@@ -521,60 +607,103 @@ storage() ->
 %%
 %% Note: session API doesn't handle session takeovers, it's the job of
 %% the broker.
--spec session_open(id()) ->
+-spec session_open(id(), emqx_types:conninfo()) ->
-    {ok, session(), #{topic() => subscription()}} | false.
+    session() | false.
-session_open(SessionId) ->
+session_open(SessionId, NewConnInfo) ->
+    NowMS = now_ms(),
     transaction(fun() ->
         case mnesia:read(?SESSION_TAB, SessionId, write) of
-            [Record = #session{}] ->
+            [Record0 = #session{last_alive_at = LastAliveAt, conninfo = ConnInfo}] ->
+                EI = expiry_interval(ConnInfo),
+                case ?IS_EXPIRED(NowMS, LastAliveAt, EI) of
+                    true ->
+                        session_drop(SessionId),
+                        false;
+                    false ->
+                        %% new connection being established
+                        Record1 = Record0#session{conninfo = NewConnInfo},
+                        Record = session_set_last_alive_at(Record1, NowMS),
                         Session = export_session(Record),
                         DSSubs = session_read_subscriptions(SessionId),
                         Subscriptions = export_subscriptions(DSSubs),
-                {ok, Session, Subscriptions};
+                        Inflight = emqx_persistent_message_ds_replayer:open(SessionId),
-            [] ->
+                        Session#{
+                            conninfo => NewConnInfo,
+                            inflight => Inflight,
+                            subscriptions => Subscriptions
+                        }
+                end;
+            _ ->
                 false
         end
     end).
 
--spec session_ensure_new(id(), _Props :: map()) ->
+-spec session_ensure_new(id(), emqx_types:conninfo(), _Props :: map()) ->
-    {ok, session(), #{topic() => subscription()}}.
+    session().
-session_ensure_new(SessionId, Props) ->
+session_ensure_new(SessionId, ConnInfo, Props) ->
     transaction(fun() ->
         ok = session_drop_subscriptions(SessionId),
-        Session = export_session(session_create(SessionId, Props)),
+        Session = export_session(session_create(SessionId, ConnInfo, Props)),
-        {ok, Session, #{}}
+        Session#{
+            subscriptions => #{},
+            inflight => emqx_persistent_message_ds_replayer:new()
+        }
     end).
 
-session_create(SessionId, Props) ->
+session_create(SessionId, ConnInfo, Props) ->
     Session = #session{
         id = SessionId,
-        created_at = erlang:system_time(millisecond),
+        created_at = now_ms(),
-        expires_at = never,
+        last_alive_at = now_ms(),
-        props = Props,
+        conninfo = ConnInfo,
-        inflight = emqx_persistent_message_ds_replayer:new()
+        props = Props
     },
     ok = mnesia:write(?SESSION_TAB, Session, write),
     Session.
 
+session_set_last_alive_at_trans(Session, LastAliveAt) ->
+    #{conninfo := ConnInfo} = Session,
+    session_set_last_alive_at_trans(Session, ConnInfo, LastAliveAt).
+
+session_set_last_alive_at_trans(Session, NewConnInfo, LastAliveAt) ->
+    #{id := SessionId} = Session,
+    transaction(fun() ->
+        case mnesia:read(?SESSION_TAB, SessionId, write) of
+            [#session{} = SessionRecord0] ->
+                SessionRecord = SessionRecord0#session{conninfo = NewConnInfo},
+                _ = session_set_last_alive_at(SessionRecord, LastAliveAt),
+                ok;
+            _ ->
+                %% log and crash?
+                ok
+        end
+    end),
+    Session#{conninfo := NewConnInfo, last_alive_at := LastAliveAt}.
+
+session_set_last_alive_at(SessionRecord0, LastAliveAt) ->
+    SessionRecord = SessionRecord0#session{last_alive_at = LastAliveAt},
+    ok = mnesia:write(?SESSION_TAB, SessionRecord, write),
+    SessionRecord.
+
 %% @doc Called when a client reconnects with `clean session=true' or
 %% during session GC
 -spec session_drop(id()) -> ok.
 session_drop(DSSessionId) ->
     transaction(fun() ->
         ok = session_drop_subscriptions(DSSessionId),
-        ok = session_drop_iterators(DSSessionId),
+        ok = session_drop_pubranges(DSSessionId),
+        ok = session_drop_offsets(DSSessionId),
         ok = session_drop_streams(DSSessionId),
         ok = mnesia:delete(?SESSION_TAB, DSSessionId, write)
     end).
 
 -spec session_drop_subscriptions(id()) -> ok.
 session_drop_subscriptions(DSSessionId) ->
-    Subscriptions = session_read_subscriptions(DSSessionId),
+    Subscriptions = session_read_subscriptions(DSSessionId, write),
     lists:foreach(
         fun(#ds_sub{id = DSSubId} = DSSub) ->
             TopicFilter = subscription_id_to_topic_filter(DSSubId),
-            TopicFilterBin = emqx_topic:join(TopicFilter),
+            ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, DSSessionId),
-            ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId),
             ok = session_del_subscription(DSSub)
         end,
         Subscriptions
@@ -633,19 +762,44 @@ session_del_subscription(DSSessionId, TopicFilter) ->
 session_del_subscription(#ds_sub{id = DSSubId}) ->
     mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write).
 
-session_read_subscriptions(DSSessionId) ->
+session_read_subscriptions(DSSessionID) ->
+    session_read_subscriptions(DSSessionID, read).
+
+session_read_subscriptions(DSSessionId, LockKind) ->
     MS = ets:fun2ms(
         fun(Sub = #ds_sub{id = {Sess, _}}) when Sess =:= DSSessionId ->
             Sub
         end
     ),
-    mnesia:select(?SESSION_SUBSCRIPTIONS_TAB, MS, read).
+    mnesia:select(?SESSION_SUBSCRIPTIONS_TAB, MS, LockKind).
+
+session_read_pubranges(DSSessionID) ->
+    session_read_pubranges(DSSessionID, read).
+
+session_read_pubranges(DSSessionId, LockKind) ->
+    MS = ets:fun2ms(
+        fun(#ds_pubrange{id = {Sess, First}}) when Sess =:= DSSessionId ->
+            {DSSessionId, First}
+        end
+    ),
+    mnesia:select(?SESSION_PUBRANGE_TAB, MS, LockKind).
+
+session_read_offsets(DSSessionID) ->
+    session_read_offsets(DSSessionID, read).
+
+session_read_offsets(DSSessionId, LockKind) ->
+    MS = ets:fun2ms(
+        fun(#ds_committed_offset{id = {Sess, Type}}) when Sess =:= DSSessionId ->
+            {DSSessionId, Type}
+        end
+    ),
+    mnesia:select(?SESSION_COMMITTED_OFFSET_TAB, MS, LockKind).
 
 -spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), integer()}.
 new_subscription_id(DSSessionId, TopicFilter) ->
     %% Note: here we use _milliseconds_ to match with the timestamp
     %% field of `#message' record.
-    NowMS = erlang:system_time(millisecond),
+    NowMS = now_ms(),
     DSSubId = {DSSessionId, TopicFilter},
     {DSSubId, NowMS}.
 
@@ -653,6 +807,9 @@ new_subscription_id(DSSessionId, TopicFilter) ->
 subscription_id_to_topic_filter({_DSSessionId, TopicFilter}) ->
     TopicFilter.
 
+now_ms() ->
+    erlang:system_time(millisecond).
+
 %%--------------------------------------------------------------------
 %% RPC targets (v1)
 %%--------------------------------------------------------------------
@@ -677,84 +834,100 @@ do_ensure_all_iterators_closed(_DSSessionID) ->
 %% Reading batches
 %%--------------------------------------------------------------------
 
--spec renew_streams(id()) -> ok.
+-spec renew_streams(session()) -> ok.
-renew_streams(DSSessionId) ->
+renew_streams(#{id := SessionId, subscriptions := Subscriptions}) ->
-    Subscriptions = ro_transaction(fun() -> session_read_subscriptions(DSSessionId) end),
+    transaction(fun() ->
-    ExistingStreams = ro_transaction(fun() -> mnesia:read(?SESSION_STREAM_TAB, DSSessionId) end),
+        ExistingStreams = mnesia:read(?SESSION_STREAM_TAB, SessionId, write),
-    lists:foreach(
+        maps:fold(
-        fun(#ds_sub{id = {_, TopicFilter}, start_time = StartTime}) ->
+            fun(TopicFilter, #{start_time := StartTime}, Streams) ->
-            renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime)
+                TopicFilterWords = emqx_topic:words(TopicFilter),
+                renew_topic_streams(SessionId, TopicFilterWords, StartTime, Streams)
             end,
+            ExistingStreams,
             Subscriptions
+        )
+    end),
+    ok.
+
+-spec renew_topic_streams(id(), topic_filter_words(), emqx_ds:time(), _Acc :: [ds_stream()]) -> ok.
+renew_topic_streams(DSSessionId, TopicFilter, StartTime, ExistingStreams) ->
+    TopicStreams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
+    lists:foldl(
+        fun({Rank, Stream}, Streams) ->
+            case lists:keymember(Stream, #ds_stream.stream, Streams) of
+                true ->
+                    Streams;
+                false ->
+                    StreamRef = length(Streams) + 1,
+                    DSStream = session_store_stream(
+                        DSSessionId,
+                        StreamRef,
+                        Stream,
+                        Rank,
+                        TopicFilter,
+                        StartTime
+                    ),
+                    [DSStream | Streams]
+            end
+        end,
+        ExistingStreams,
+        TopicStreams
     ).
 
--spec renew_streams(id(), [ds_stream()], emqx_ds:topic_filter(), emqx_ds:time()) -> ok.
+session_store_stream(DSSessionId, StreamRef, Stream, Rank, TopicFilter, StartTime) ->
-renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime) ->
+    {ok, ItBegin} = emqx_ds:make_iterator(
-    AllStreams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
+        ?PERSISTENT_MESSAGE_DB,
-    transaction(
+        Stream,
-        fun() ->
+        TopicFilter,
-            lists:foreach(
+        StartTime
-                fun({Rank, Stream}) ->
-                    Rec = #ds_stream{
-                        session = DSSessionId,
-                        topic_filter = TopicFilter,
-                        stream = Stream,
-                        rank = Rank
-                    },
-                    case lists:member(Rec, ExistingStreams) of
-                        true ->
-                            ok;
-                        false ->
-                            mnesia:write(?SESSION_STREAM_TAB, Rec, write),
-                            {ok, Iterator} = emqx_ds:make_iterator(
-                                ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime
     ),
-                            %% Workaround: we convert `Stream' to a binary before
+    DSStream = #ds_stream{
-                            %% attempting to store it in mnesia(rocksdb) because of a bug
+        session = DSSessionId,
-                            %% in `mnesia_rocksdb' when trying to do
+        ref = StreamRef,
-                            %% `mnesia:dirty_all_keys' later.
+        stream = Stream,
-                            StreamBin = term_to_binary(Stream),
+        rank = Rank,
-                            IterRec = #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator},
+        beginning = ItBegin
-                            mnesia:write(?SESSION_ITER_TAB, IterRec, write)
+    },
-                    end
+    mnesia:write(?SESSION_STREAM_TAB, DSStream, write),
-                end,
+    DSStream.
-                AllStreams
-            )
-        end
-    ).
 
 %% must be called inside a transaction
 -spec session_drop_streams(id()) -> ok.
 session_drop_streams(DSSessionId) ->
-    MS = ets:fun2ms(
+    mnesia:delete(?SESSION_STREAM_TAB, DSSessionId, write).
-        fun(#ds_stream{session = DSSessionId0}) when DSSessionId0 =:= DSSessionId ->
-            DSSessionId0
-        end
-    ),
-    StreamIDs = mnesia:select(?SESSION_STREAM_TAB, MS, write),
-    lists:foreach(fun(Key) -> mnesia:delete(?SESSION_STREAM_TAB, Key, write) end, StreamIDs).
 
 %% must be called inside a transaction
--spec session_drop_iterators(id()) -> ok.
+-spec session_drop_pubranges(id()) -> ok.
-session_drop_iterators(DSSessionId) ->
+session_drop_pubranges(DSSessionId) ->
-    MS = ets:fun2ms(
+    RangeIds = session_read_pubranges(DSSessionId, write),
-        fun(#ds_iter{id = {DSSessionId0, StreamBin}}) when DSSessionId0 =:= DSSessionId ->
-            StreamBin
-        end
-    ),
-    StreamBins = mnesia:select(?SESSION_ITER_TAB, MS, write),
     lists:foreach(
-        fun(StreamBin) ->
+        fun(RangeId) ->
-            mnesia:delete(?SESSION_ITER_TAB, {DSSessionId, StreamBin}, write)
+            mnesia:delete(?SESSION_PUBRANGE_TAB, RangeId, write)
         end,
-        StreamBins
+        RangeIds
+    ).
+
+%% must be called inside a transaction
+-spec session_drop_offsets(id()) -> ok.
+session_drop_offsets(DSSessionId) ->
+    OffsetIds = session_read_offsets(DSSessionId, write),
+    lists:foreach(
+        fun(OffsetId) ->
+            mnesia:delete(?SESSION_COMMITTED_OFFSET_TAB, OffsetId, write)
+        end,
+        OffsetIds
     ).
 
 %%--------------------------------------------------------------------------------
 
 transaction(Fun) ->
+    case mnesia:is_transaction() of
+        true ->
+            Fun();
+        false ->
             {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
-    Res.
+            Res
+    end.
 
 ro_transaction(Fun) ->
     {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
@@ -772,7 +945,7 @@ export_subscriptions(DSSubs) ->
     ).
 
 export_session(#session{} = Record) ->
-    export_record(Record, #session.id, [id, created_at, expires_at, inflight, props], #{}).
+    export_record(Record, #session.id, [id, created_at, last_alive_at, conninfo, props], #{}).
 
 export_subscription(#ds_sub{} = Record) ->
     export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}).
@@ -786,13 +959,17 @@ export_record(_, _, [], Acc) ->
 %% effects. Add `CBM:init' callback to the session behavior?
 ensure_timers() ->
     ensure_timer(pull),
-    ensure_timer(get_streams).
+    ensure_timer(get_streams),
+    ensure_timer(bump_last_alive_at).
 
--spec ensure_timer(pull | get_streams) -> ok.
+-spec ensure_timer(timer()) -> ok.
+ensure_timer(bump_last_alive_at = Type) ->
+    BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
+    ensure_timer(Type, BumpInterval);
 ensure_timer(Type) ->
     ensure_timer(Type, 100).
 
--spec ensure_timer(pull | get_streams, non_neg_integer()) -> ok.
+-spec ensure_timer(timer(), non_neg_integer()) -> ok.
 ensure_timer(Type, Timeout) ->
     _ = emqx_utils:start_timer(Timeout, {emqx_session, Type}),
     ok.
@@ -804,13 +981,23 @@ receive_maximum(ConnInfo) ->
     %% indicates that it's optional.
     maps:get(receive_maximum, ConnInfo, 65_535).
 
+-spec expiry_interval(conninfo()) -> millisecond().
+expiry_interval(ConnInfo) ->
+    maps:get(expiry_interval, ConnInfo, 0).
+
 -ifdef(TEST).
 list_all_sessions() ->
     DSSessionIds = mnesia:dirty_all_keys(?SESSION_TAB),
-    Sessions = lists:map(
+    ConnInfo = #{},
+    Sessions = lists:filtermap(
         fun(SessionID) ->
-            {ok, Session, Subscriptions} = session_open(SessionID),
+            Sess = session_open(SessionID, ConnInfo),
-            {SessionID, #{session => Session, subscriptions => Subscriptions}}
+            case Sess of
+                false ->
+                    false;
+                _ ->
+                    {true, {SessionID, Sess}}
+            end
         end,
         DSSessionIds
     ),
@@ -850,16 +1037,18 @@ list_all_streams() ->
     ),
     maps:from_list(DSStreams).
 
-list_all_iterators() ->
+list_all_pubranges() ->
-    DSIterIds = mnesia:dirty_all_keys(?SESSION_ITER_TAB),
+    DSPubranges = mnesia:dirty_match_object(?SESSION_PUBRANGE_TAB, #ds_pubrange{_ = '_'}),
-    DSIters = lists:map(
+    lists:foldl(
-        fun(DSIterId) ->
+        fun(Record = #ds_pubrange{id = {SessionId, First}}, Acc) ->
-            [Record] = mnesia:dirty_read(?SESSION_ITER_TAB, DSIterId),
+            Range = export_record(
-            {DSIterId, export_record(Record, #ds_iter.id, [id, iter], #{})}
+                Record, #ds_pubrange.until, [until, stream, type, iterator], #{first => First}
-        end,
-        DSIterIds
             ),
-    maps:from_list(DSIters).
+            maps:put(SessionId, maps:get(SessionId, Acc, []) ++ [Range], Acc)
+        end,
+        #{},
+        DSPubranges
+    ).
 
 %% ifdef(TEST)
 -endif.

apps/emqx/src/emqx_persistent_session_ds.hrl

@@ -21,9 +21,13 @@
 -define(SESSION_TAB, emqx_ds_session).
 -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
 -define(SESSION_STREAM_TAB, emqx_ds_stream_tab).
--define(SESSION_ITER_TAB, emqx_ds_iter_tab).
+-define(SESSION_PUBRANGE_TAB, emqx_ds_pubrange_tab).
+-define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab).
 -define(DS_MRIA_SHARD, emqx_ds_session_shard).
 
+-define(T_INFLIGHT, 1).
+-define(T_CHECKPOINT, 2).
+
 -record(ds_sub, {
     id :: emqx_persistent_session_ds:subscription_id(),
     start_time :: emqx_ds:time(),
@@ -34,16 +38,53 @@
 
 -record(ds_stream, {
     session :: emqx_persistent_session_ds:id(),
-    topic_filter :: emqx_ds:topic_filter(),
+    ref :: _StreamRef,
     stream :: emqx_ds:stream(),
-    rank :: emqx_ds:stream_rank()
+    rank :: emqx_ds:stream_rank(),
+    beginning :: emqx_ds:iterator()
 }).
 -type ds_stream() :: #ds_stream{}.
--type ds_stream_bin() :: binary().
 
--record(ds_iter, {
+-record(ds_pubrange, {
-    id :: {emqx_persistent_session_ds:id(), ds_stream_bin()},
+    id :: {
-    iter :: emqx_ds:iterator()
+        %% What session this range belongs to.
+        _Session :: emqx_persistent_session_ds:id(),
+        %% Where this range starts.
+        _First :: emqx_persistent_message_ds_replayer:seqno()
+    },
+    %% Where this range ends: the first seqno that is not included in the range.
+    until :: emqx_persistent_message_ds_replayer:seqno(),
+    %% Which stream this range is over.
+    stream :: _StreamRef,
+    %% Type of a range:
+    %% * Inflight range is a range of yet unacked messages from this stream.
+    %% * Checkpoint range was already acked, its purpose is to keep track of the
+    %%   very last iterator for this stream.
+    type :: ?T_INFLIGHT | ?T_CHECKPOINT,
+    %% What commit tracks this range is part of.
+    %% This is rarely stored: we only need to persist it when the range contains
+    %% QoS 2 messages.
+    tracks = 0 :: non_neg_integer(),
+    %% Meaning of this depends on the type of the range:
+    %% * For inflight range, this is the iterator pointing to the first message in
+    %%   the range.
+    %% * For checkpoint range, this is the iterator pointing right past the last
+    %%   message in the range.
+    iterator :: emqx_ds:iterator(),
+    %% Reserved for future use.
+    misc = #{} :: map()
+}).
+-type ds_pubrange() :: #ds_pubrange{}.
+
+-record(ds_committed_offset, {
+    id :: {
+        %% What session this marker belongs to.
+        _Session :: emqx_persistent_session_ds:id(),
+        %% Marker name.
+        _CommitType
+    },
+    %% Where this marker is pointing to: the first seqno that is not marked.
+    until :: emqx_persistent_message_ds_replayer:seqno()
 }).
 
 -record(session, {
@@ -51,8 +92,8 @@
     id :: emqx_persistent_session_ds:id(),
     %% creation time
     created_at :: _Millisecond :: non_neg_integer(),
-    expires_at = never :: _Millisecond :: non_neg_integer() | never,
+    last_alive_at :: _Millisecond :: non_neg_integer(),
-    inflight :: emqx_persistent_message_ds_replayer:inflight(),
+    conninfo :: emqx_types:conninfo(),
     %% for future usage
     props = #{} :: map()
 }).

apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl

@@ -0,0 +1,161 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_persistent_session_ds_gc_worker).
+
+-behaviour(gen_server).
+
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+
+-include("emqx_persistent_session_ds.hrl").
+
+%% API
+-export([
+    start_link/0
+]).
+
+%% `gen_server' API
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%% call/cast/info records
+-record(gc, {}).
+
+%%--------------------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------------------
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+%%--------------------------------------------------------------------------------
+%% `gen_server' API
+%%--------------------------------------------------------------------------------
+
+init(_Opts) ->
+    ensure_gc_timer(),
+    State = #{},
+    {ok, State}.
+
+handle_call(_Call, _From, State) ->
+    {reply, error, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(#gc{}, State) ->
+    try_gc(),
+    ensure_gc_timer(),
+    {noreply, State};
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+%%--------------------------------------------------------------------------------
+%% Internal fns
+%%--------------------------------------------------------------------------------
+
+ensure_gc_timer() ->
+    Timeout = emqx_config:get([session_persistence, session_gc_interval]),
+    _ = erlang:send_after(Timeout, self(), #gc{}),
+    ok.
+
+try_gc() ->
+    %% Only cores should run GC.
+    CoreNodes = mria_membership:running_core_nodelist(),
+    Res = global:trans(
+        {?MODULE, self()},
+        fun() -> ?tp_span(ds_session_gc, #{}, start_gc()) end,
+        CoreNodes,
+        %% Note: we set retries to 1 here because, in rare occasions, GC might start at the
+        %% same time in more than one node, and each one will abort the other.  By allowing
+        %% one retry, at least one node will (hopefully) get to enter the transaction and
+        %% the other will abort.  If GC runs too fast, both nodes might run in sequence.
+        %% But, in that case, GC is clearly not too costly, and that shouldn't be a problem,
+        %% resource-wise.
+        _Retries = 1
+    ),
+    case Res of
+        aborted ->
+            ?tp(ds_session_gc_lock_taken, #{}),
+            ok;
+        ok ->
+            ok
+    end.
+
+now_ms() ->
+    erlang:system_time(millisecond).
+
+start_gc() ->
+    do_gc(more).
+
+zombie_session_ms() ->
+    NowMS = now_ms(),
+    GCInterval = emqx_config:get([session_persistence, session_gc_interval]),
+    BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
+    TimeThreshold = max(GCInterval, BumpInterval) * 3,
+    ets:fun2ms(
+        fun(
+            #session{
+                id = DSSessionId,
+                last_alive_at = LastAliveAt,
+                conninfo = #{expiry_interval := EI}
+            }
+        ) when
+            LastAliveAt + EI + TimeThreshold =< NowMS
+        ->
+            DSSessionId
+        end
+    ).
+
+do_gc(more) ->
+    GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]),
+    MS = zombie_session_ms(),
+    {atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() ->
+        Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write),
+        case Res of
+            '$end_of_table' ->
+                done;
+            {[], Cont} ->
+                %% since `GCBatchsize' is just a "recommendation" for `select', we try only
+                %% _once_ the continuation and then stop if it yields nothing, to avoid a
+                %% dead loop.
+                case mnesia:select(Cont) of
+                    '$end_of_table' ->
+                        done;
+                    {[], _Cont} ->
+                        done;
+                    {DSSessionIds0, _Cont} ->
+                        do_gc_(DSSessionIds0),
+                        more
+                end;
+            {DSSessionIds0, _Cont} ->
+                do_gc_(DSSessionIds0),
+                more
+        end
+    end),
+    do_gc(Next);
+do_gc(done) ->
+    ok.
+
+do_gc_(DSSessionIds) ->
+    lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds),
+    ?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}),
+    ok.

apps/emqx/src/emqx_persistent_session_ds_sup.erl

@@ -0,0 +1,78 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_persistent_session_ds_sup).
+
+-behaviour(supervisor).
+
+%% API
+-export([
+    start_link/0
+]).
+
+%% `supervisor' API
+-export([
+    init/1
+]).
+
+%%--------------------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------------------
+
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%%--------------------------------------------------------------------------------
+%% `supervisor' API
+%%--------------------------------------------------------------------------------
+
+init(Opts) ->
+    case emqx_persistent_message:is_persistence_enabled() of
+        true ->
+            do_init(Opts);
+        false ->
+            ignore
+    end.
+
+do_init(_Opts) ->
+    SupFlags = #{
+        strategy => rest_for_one,
+        intensity => 10,
+        period => 2,
+        auto_shutdown => never
+    },
+    CoreChildren = [
+        worker(gc_worker, emqx_persistent_session_ds_gc_worker, [])
+    ],
+    Children =
+        case mria_rlog:role() of
+            core -> CoreChildren;
+            replicant -> []
+        end,
+    {ok, {SupFlags, Children}}.
+
+%%--------------------------------------------------------------------------------
+%% Internal fns
+%%--------------------------------------------------------------------------------
+
+worker(Id, Mod, Args) ->
+    #{
+        id => Id,
+        start => {Mod, start_link, Args},
+        type => worker,
+        restart => permanent,
+        shutdown => 10_000,
+        significant => false
+    }.

apps/emqx/src/emqx_quic_stream.erl

@@ -184,7 +184,7 @@ peer_send_aborted(Stream, ErrorCode, S) ->
 
 -spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret().
 peer_send_shutdown(Stream, undefined, S) ->
-    ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
+    _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
     {ok, S}.
 
 -spec send_complete(stream_handle(), boolean(), cb_data()) -> cb_ret().

apps/emqx/src/emqx_reason_codes.erl

@@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER;
 compat(connack, 16#9F) -> ?CONNACK_SERVER;
 compat(suback, Code) when Code =< ?QOS_2 -> Code;
 compat(suback, Code) when Code >= 16#80 -> 16#80;
+%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1
 compat(unsuback, _Code) -> undefined;
 compat(_Other, _Code) -> undefined.
 

apps/emqx/src/emqx_router.erl

@@ -52,6 +52,9 @@
     lookup_routes/1
 ]).
 
+%% Topics API
+-export([select/3]).
+
 -export([print_routes/1]).
 
 -export([
@@ -59,7 +62,10 @@
     foldr_routes/2
 ]).
 
--export([topics/0]).
+-export([
+    topics/0,
+    stats/1
+]).
 
 %% Exported for tests
 -export([has_route/2]).
@@ -219,6 +225,19 @@ mria_delete_route(v2, Topic, Dest) ->
 mria_delete_route(v1, Topic, Dest) ->
     mria_delete_route_v1(Topic, Dest).
 
+-spec select(Spec, _Limit :: pos_integer(), Continuation) ->
+    {[emqx_types:route()], Continuation} | '$end_of_table'
+when
+    Spec :: {_TopicPat, _DestPat},
+    Continuation :: term() | '$end_of_table'.
+select(MatchSpec, Limit, Cont) ->
+    select(get_schema_vsn(), MatchSpec, Limit, Cont).
+
+select(v2, MatchSpec, Limit, Cont) ->
+    select_v2(MatchSpec, Limit, Cont);
+select(v1, MatchSpec, Limit, Cont) ->
+    select_v1(MatchSpec, Limit, Cont).
+
 -spec topics() -> list(emqx_types:topic()).
 topics() ->
     topics(get_schema_vsn()).
@@ -228,6 +247,15 @@ topics(v2) ->
 topics(v1) ->
     list_topics_v1().
 
+-spec stats(n_routes) -> non_neg_integer().
+stats(Item) ->
+    stats(get_schema_vsn(), Item).
+
+stats(v2, Item) ->
+    get_stats_v2(Item);
+stats(v1, Item) ->
+    get_stats_v1(Item).
+
 %% @doc Print routes to a topic
 -spec print_routes(emqx_types:topic()) -> ok.
 print_routes(Topic) ->
@@ -345,9 +373,17 @@ cleanup_routes_v1(Node) ->
         ]
     end).
 
+select_v1({MTopic, MDest}, Limit, undefined) ->
+    ets:match_object(?ROUTE_TAB, #route{topic = MTopic, dest = MDest}, Limit);
+select_v1(_Spec, _Limit, Cont) ->
+    ets:select(Cont).
+
 list_topics_v1() ->
     list_route_tab_topics().
 
+get_stats_v1(n_routes) ->
+    emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0).
+
 list_route_tab_topics() ->
     mnesia:dirty_all_keys(?ROUTE_TAB).
 
@@ -436,11 +472,52 @@ get_dest_node({_, Node}) ->
 get_dest_node(Node) ->
     Node.
 
+select_v2(Spec, Limit, undefined) ->
+    Stream = mk_route_stream(Spec),
+    select_next(Limit, Stream);
+select_v2(_Spec, Limit, Stream) ->
+    select_next(Limit, Stream).
+
+select_next(N, Stream) ->
+    case emqx_utils_stream:consume(N, Stream) of
+        {Routes, SRest} ->
+            {Routes, SRest};
+        Routes ->
+            {Routes, '$end_of_table'}
+    end.
+
+mk_route_stream(Spec) ->
+    emqx_utils_stream:chain(
+        mk_route_stream(route, Spec),
+        mk_route_stream(filter, Spec)
+    ).
+
+mk_route_stream(route, Spec) ->
+    emqx_utils_stream:ets(fun(Cont) -> select_v1(Spec, 1, Cont) end);
+mk_route_stream(filter, {MTopic, MDest}) ->
+    emqx_utils_stream:map(
+        fun routeidx_to_route/1,
+        emqx_utils_stream:ets(
+            fun
+                (undefined) ->
+                    MatchSpec = #routeidx{entry = emqx_trie_search:make_pat(MTopic, MDest)},
+                    ets:match_object(?ROUTE_TAB_FILTERS, MatchSpec, 1);
+                (Cont) ->
+                    ets:match_object(Cont)
+            end
+        )
+    ).
+
 list_topics_v2() ->
     Pat = #routeidx{entry = '$1'},
     Filters = [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_FILTERS, Pat)],
     list_route_tab_topics() ++ Filters.
 
+get_stats_v2(n_routes) ->
+    NTopics = emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0),
+    NWildcards = emqx_maybe:define(ets:info(?ROUTE_TAB_FILTERS, size), 0),
+    NTopics + NWildcards.
+
 fold_routes_v2(FunName, FoldFun, AccIn) ->
     FilterFoldFun = mk_filtertab_fold_fun(FoldFun),
     Acc = ets:FunName(FoldFun, AccIn, ?ROUTE_TAB),
@@ -449,6 +526,9 @@ fold_routes_v2(FunName, FoldFun, AccIn) ->
 mk_filtertab_fold_fun(FoldFun) ->
     fun(#routeidx{entry = K}, Acc) -> FoldFun(match_to_route(K), Acc) end.
 
+routeidx_to_route(#routeidx{entry = M}) ->
+    match_to_route(M).
+
 match_to_route(M) ->
     #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}.
 

apps/emqx/src/emqx_router_helper.erl

@@ -190,12 +190,7 @@ code_change(_OldVsn, State, _Extra) ->
 %%--------------------------------------------------------------------
 
 stats_fun() ->
-    case ets:info(?ROUTE_TAB, size) of
+    emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)).
-        undefined ->
-            ok;
-        Size ->
-            emqx_stats:setstat('topics.count', 'topics.max', Size)
-    end.
 
 cleanup_routes(Node) ->
     emqx_router:cleanup_routes(Node).

apps/emqx/src/emqx_schema.erl

@@ -294,7 +294,19 @@ roots(low) ->
         {"persistent_session_store",
             sc(
                 ref("persistent_session_store"),
-                #{importance => ?IMPORTANCE_HIDDEN}
+                #{
+                    %% NOTE
+                    %% Due to some quirks in interaction between `emqx_config` and
+                    %% `hocon_tconf`, schema roots cannot currently be deprecated.
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {"session_persistence",
+            sc(
+                ref("session_persistence"),
+                #{
+                    importance => ?IMPORTANCE_HIDDEN
+                }
             )},
         {"trace",
             sc(
@@ -309,11 +321,12 @@ roots(low) ->
     ].
 
 fields("persistent_session_store") ->
+    Deprecated = #{deprecated => {since, "5.4.0"}},
     [
         {"enabled",
             sc(
                 boolean(),
-                #{
+                Deprecated#{
                     default => false,
                     %% TODO(5.2): change field name to 'enable' and keep 'enabled' as an alias
                     aliases => [enable],
@@ -323,7 +336,7 @@ fields("persistent_session_store") ->
         {"ds",
             sc(
                 boolean(),
-                #{
+                Deprecated#{
                     default => false,
                     importance => ?IMPORTANCE_HIDDEN
                 }
@@ -331,7 +344,7 @@ fields("persistent_session_store") ->
         {"on_disc",
             sc(
                 boolean(),
-                #{
+                Deprecated#{
                     default => true,
                     desc => ?DESC(persistent_store_on_disc)
                 }
@@ -339,7 +352,7 @@ fields("persistent_session_store") ->
         {"ram_cache",
             sc(
                 boolean(),
-                #{
+                Deprecated#{
                     default => false,
                     desc => ?DESC(persistent_store_ram_cache)
                 }
@@ -347,7 +360,7 @@ fields("persistent_session_store") ->
         {"backend",
             sc(
                 hoconsc:union([ref("persistent_session_builtin")]),
-                #{
+                Deprecated#{
                     default => #{
                         <<"type">> => <<"builtin">>,
                         <<"session">> =>
@@ -363,7 +376,7 @@ fields("persistent_session_store") ->
         {"max_retain_undelivered",
             sc(
                 duration(),
-                #{
+                Deprecated#{
                     default => <<"1h">>,
                     desc => ?DESC(persistent_session_store_max_retain_undelivered)
                 }
@@ -371,7 +384,7 @@ fields("persistent_session_store") ->
         {"message_gc_interval",
             sc(
                 duration(),
-                #{
+                Deprecated#{
                     default => <<"1h">>,
                     desc => ?DESC(persistent_session_store_message_gc_interval)
                 }
@@ -379,7 +392,7 @@ fields("persistent_session_store") ->
         {"session_message_gc_interval",
             sc(
                 duration(),
-                #{
+                Deprecated#{
                     default => <<"1m">>,
                     desc => ?DESC(persistent_session_store_session_message_gc_interval)
                 }
@@ -1382,7 +1395,7 @@ fields("broker_routing") ->
             sc(
                 hoconsc:enum([v1, v2]),
                 #{
-                    default => v1,
+                    default => v2,
                     'readOnly' => true,
                     desc => ?DESC(broker_routing_storage_schema)
                 }
@@ -1740,6 +1753,103 @@ fields("trace") ->
                 importance => ?IMPORTANCE_HIDDEN,
                 desc => ?DESC(fields_trace_payload_encode)
             })}
+    ];
+fields("session_persistence") ->
+    [
+        {"enable",
+            sc(
+                boolean(), #{
+                    desc => ?DESC(session_persistence_enable),
+                    default => false
+                }
+            )},
+        {"storage",
+            sc(
+                ref("session_storage_backend"), #{
+                    desc => ?DESC(session_persistence_storage),
+                    validator => fun validate_backend_enabled/1,
+                    default => #{
+                        <<"builtin">> => #{}
+                    }
+                }
+            )},
+        {"idle_poll_interval",
+            sc(
+                timeout_duration(),
+                #{
+                    default => <<"100ms">>,
+                    desc => ?DESC(session_ds_idle_poll_interval)
+                }
+            )},
+        {"last_alive_update_interval",
+            sc(
+                timeout_duration(),
+                #{
+                    default => <<"5000ms">>,
+                    desc => ?DESC(session_ds_last_alive_update_interval)
+                }
+            )},
+        {"session_gc_interval",
+            sc(
+                timeout_duration(),
+                #{
+                    default => <<"10m">>,
+                    desc => ?DESC(session_ds_session_gc_interval)
+                }
+            )},
+        {"session_gc_batch_size",
+            sc(
+                pos_integer(),
+                #{
+                    default => 100,
+                    importance => ?IMPORTANCE_LOW,
+                    desc => ?DESC(session_ds_session_gc_batch_size)
+                }
+            )},
+        {"force_persistence",
+            sc(
+                boolean(),
+                #{
+                    default => false,
+                    %% Only for testing, shall remain hidden
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )}
+    ];
+fields("session_storage_backend") ->
+    [
+        {"builtin",
+            sc(ref("session_storage_backend_builtin"), #{
+                desc => ?DESC(session_storage_backend_builtin),
+                required => {false, recursively}
+            })}
+    ] ++ emqx_schema_hooks:injection_point('session_persistence.storage_backends', []);
+fields("session_storage_backend_builtin") ->
+    [
+        {"enable",
+            sc(
+                boolean(),
+                #{
+                    desc => ?DESC(session_storage_backend_enable),
+                    default => true
+                }
+            )},
+        {"n_shards",
+            sc(
+                pos_integer(),
+                #{
+                    desc => ?DESC(session_builtin_n_shards),
+                    default => 16
+                }
+            )},
+        {"replication_factor",
+            sc(
+                pos_integer(),
+                #{
+                    default => 3,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )}
     ].
 
 mqtt_listener(Bind) ->
@@ -1992,6 +2102,8 @@ desc("ocsp") ->
     "Per listener OCSP Stapling configuration.";
 desc("crl_cache") ->
     "Global CRL cache options.";
+desc("session_persistence") ->
+    "Settings governing durable sessions persistence.";
 desc(_) ->
     undefined.
 
@@ -2014,6 +2126,17 @@ ensure_list(V) ->
 filter(Opts) ->
     [{K, V} || {K, V} <- Opts, V =/= undefined].
 
+validate_backend_enabled(Config) ->
+    Enabled = maps:filter(fun(_, #{<<"enable">> := E}) -> E end, Config),
+    case maps:to_list(Enabled) of
+        [{_Type, _BackendConfig}] ->
+            ok;
+        _Conflicts = [_ | _] ->
+            {error, multiple_enabled_backends};
+        _None = [] ->
+            {error, no_enabled_backend}
+    end.
+
 %% @private This function defines the SSL opts which are commonly used by
 %% SSL listener and client.
 -spec common_ssl_opts_schema(map(), server | client) -> hocon_schema:field_schema().

apps/emqx/src/emqx_schema_secret.erl

@@ -0,0 +1,85 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc HOCON schema that defines _secret_ concept.
+-module(emqx_schema_secret).
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("typerefl/include/types.hrl").
+
+-export([mk/1]).
+
+%% HOCON Schema API
+-export([convert_secret/2]).
+
+%% @doc Secret value.
+-type t() :: binary().
+
+%% @doc Source of the secret value.
+%% * "file://...": file path to a file containing secret value.
+%% * other binaries: secret value itself.
+-type source() :: iodata().
+
+-type secret() :: binary() | function().
+-reflect_type([secret/0]).
+
+-define(SCHEMA, #{
+    required => false,
+    format => <<"password">>,
+    sensitive => true,
+    converter => fun ?MODULE:convert_secret/2
+}).
+
+-dialyzer({nowarn_function, source/1}).
+
+%%
+
+-spec mk(#{atom() => _}) -> hocon_schema:field_schema().
+mk(Overrides = #{}) ->
+    hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)).
+
+convert_secret(undefined, #{}) ->
+    undefined;
+convert_secret(Secret, #{make_serializable := true}) ->
+    unicode:characters_to_binary(source(Secret));
+convert_secret(Secret, #{}) when is_function(Secret, 0) ->
+    Secret;
+convert_secret(Secret, #{}) when is_integer(Secret) ->
+    wrap(integer_to_binary(Secret));
+convert_secret(Secret, #{}) ->
+    try unicode:characters_to_binary(Secret) of
+        String when is_binary(String) ->
+            wrap(String);
+        {error, _, _} ->
+            throw(invalid_string)
+    catch
+        error:_ ->
+            throw(invalid_type)
+    end.
+
+-spec wrap(source()) -> emqx_secret:t(t()).
+wrap(<<"file://", Filename/binary>>) ->
+    emqx_secret:wrap_load({file, Filename});
+wrap(Secret) ->
+    emqx_secret:wrap(Secret).
+
+-spec source(emqx_secret:t(t())) -> source().
+source(Secret) when is_function(Secret) ->
+    source(emqx_secret:term(Secret));
+source({file, Filename}) ->
+    <<"file://", Filename/binary>>;
+source(Secret) ->
+    Secret.

apps/emqx/src/emqx_secret.erl

@@ -19,23 +19,52 @@
 -module(emqx_secret).
 
 %% API:
--export([wrap/1, unwrap/1]).
+-export([wrap/1, wrap_load/1, unwrap/1, term/1]).
 
 -export_type([t/1]).
 
 -opaque t(T) :: T | fun(() -> t(T)).
 
+%% Secret loader module.
+%% Any changes related to processing of secrets should be made there.
+-define(LOADER, emqx_secret_loader).
+
 %%================================================================================
 %% API funcions
 %%================================================================================
 
+%% @doc Wrap a term in a secret closure.
+%% This effectively hides the term from any term formatting / printing code.
+-spec wrap(T) -> t(T).
 wrap(Term) ->
     fun() ->
         Term
     end.
 
+%% @doc Wrap a loader function call over a term in a secret closure.
+%% This is slightly more flexible form of `wrap/1` with the same basic purpose.
+-spec wrap_load(emqx_secret_loader:source()) -> t(_).
+wrap_load(Source) ->
+    fun() ->
+        apply(?LOADER, load, [Source])
+    end.
+
+%% @doc Unwrap a secret closure, revealing the secret.
+%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped.
+-spec unwrap(t(T)) -> T.
 unwrap(Term) when is_function(Term, 0) ->
     %% Handle potentially nested funs
     unwrap(Term());
 unwrap(Term) ->
     Term.
+
+%% @doc Inspect the term wrapped in a secret closure.
+-spec term(t(_)) -> _Term.
+term(Wrap) when is_function(Wrap, 0) ->
+    case erlang:fun_info(Wrap, module) of
+        {module, ?MODULE} ->
+            {env, Env} = erlang:fun_info(Wrap, env),
+            lists:last(Env);
+        _ ->
+            error(badarg, [Wrap])
+    end.

apps/emqx/src/emqx_secret_loader.erl

@@ -0,0 +1,42 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_secret_loader).
+
+%% API
+-export([load/1]).
+-export([file/1]).
+
+-export_type([source/0]).
+
+-type source() :: {file, file:filename_all()}.
+
+-spec load(source()) -> binary() | no_return().
+load({file, Filename}) ->
+    file(Filename).
+
+-spec file(file:filename_all()) -> binary() | no_return().
+file(Filename) ->
+    case file:read_file(Filename) of
+        {ok, Secret} ->
+            string:trim(Secret, trailing);
+        {error, Reason} ->
+            throw(#{
+                msg => failed_to_read_secret_file,
+                path => Filename,
+                reason => emqx_utils:explain_posix(Reason)
+            })
+    end.

apps/emqx/src/emqx_session.erl

@@ -84,7 +84,7 @@
 -export([
     deliver/3,
     handle_timeout/3,
-    disconnect/2,
+    disconnect/3,
     terminate/3
 ]).
 
@@ -267,7 +267,7 @@ destroy(Session) ->
 
 -spec subscribe(
     clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts(),
     t()
 ) ->
@@ -287,7 +287,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) ->
 
 -spec unsubscribe(
     clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts(),
     t()
 ) ->
@@ -418,7 +418,13 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
     end.
 
 enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
-    SubOpts = ?IMPL(Session):get_subscription(Topic, Session),
+    SubOpts =
+        case Msg of
+            #message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} ->
+                ?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session);
+            _ ->
+                ?IMPL(Session):get_subscription(Topic, Session)
+        end,
     enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS).
 
 enrich_message(
@@ -497,10 +503,10 @@ cancel_timer(Name, Timers) ->
 
 %%--------------------------------------------------------------------
 
--spec disconnect(clientinfo(), t()) ->
+-spec disconnect(clientinfo(), eqmx_types:conninfo(), t()) ->
     {idle | shutdown, t()}.
-disconnect(_ClientInfo, Session) ->
+disconnect(_ClientInfo, ConnInfo, Session) ->
-    ?IMPL(Session):disconnect(Session).
+    ?IMPL(Session):disconnect(Session, ConnInfo).
 
 -spec terminate(clientinfo(), Reason :: term(), t()) ->
     ok.
@@ -611,21 +617,27 @@ maybe_mock_impl_mod(_) ->
 
 -spec choose_impl_mod(conninfo()) -> module().
 choose_impl_mod(#{expiry_interval := EI}) ->
-    hd(choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled())).
+    hd(choose_impl_candidates(EI, emqx_persistent_message:is_persistence_enabled())).
 
 -spec choose_impl_candidates(conninfo()) -> [module()].
 choose_impl_candidates(#{expiry_interval := EI}) ->
-    choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled()).
+    choose_impl_candidates(EI, emqx_persistent_message:is_persistence_enabled()).
 
 choose_impl_candidates(_, _IsPSStoreEnabled = false) ->
     [emqx_session_mem];
 choose_impl_candidates(0, _IsPSStoreEnabled = true) ->
+    case emqx_persistent_message:force_ds() of
+        false ->
             %% NOTE
-    %% If ExpiryInterval is 0, the natural choice is `emqx_session_mem`. Yet we still
+            %% If ExpiryInterval is 0, the natural choice is
-    %% need to look the existing session up in the `emqx_persistent_session_ds` store
+            %% `emqx_session_mem'. Yet we still need to look the
-    %% first, because previous connection may have set ExpiryInterval to a non-zero
+            %% existing session up in the `emqx_persistent_session_ds'
-    %% value.
+            %% store first, because previous connection may have set
+            %% ExpiryInterval to a non-zero value.
             [emqx_session_mem, emqx_persistent_session_ds];
+        true ->
+            [emqx_persistent_session_ds]
+    end;
 choose_impl_candidates(EI, _IsPSStoreEnabled = true) when EI > 0 ->
     [emqx_persistent_session_ds].
 

apps/emqx/src/emqx_session_mem.erl

@@ -87,7 +87,7 @@
     deliver/3,
     replay/3,
     handle_timeout/3,
-    disconnect/1,
+    disconnect/2,
     terminate/2
 ]).
 
@@ -316,7 +316,7 @@ unsubscribe(
             {error, ?RC_NO_SUBSCRIPTION_EXISTED}
     end.
 
--spec get_subscription(emqx_types:topic(), session()) ->
+-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) ->
     emqx_types:subopts() | undefined.
 get_subscription(Topic, #session{subscriptions = Subs}) ->
     maps:get(Topic, Subs, undefined).
@@ -725,8 +725,8 @@ append(L1, L2) -> L1 ++ L2.
 
 %%--------------------------------------------------------------------
 
--spec disconnect(session()) -> {idle, session()}.
+-spec disconnect(session(), emqx_types:conninfo()) -> {idle, session()}.
-disconnect(Session = #session{}) ->
+disconnect(Session = #session{}, _ConnInfo) ->
     % TODO: isolate expiry timer / timeout handling here?
     {idle, Session}.
 

apps/emqx/src/emqx_shared_sub.erl

@@ -95,7 +95,6 @@
 -define(ACK, shared_sub_ack).
 -define(NACK(Reason), {shared_sub_nack, Reason}).
 -define(NO_ACK, no_ack).
--define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
 -define(SUBSCRIBER_DOWN, noproc).
 
 -type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
@@ -234,19 +233,18 @@ without_group_ack(Msg) ->
 get_group_ack(Msg) ->
     emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
 
-with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
+%% always add `redispatch_to` header to the message
-    Msg;
+%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1
 with_redispatch_to(Msg, Group, Topic) ->
     emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
 
-%% @hidden Redispatch is needed only for the messages with redispatch_to header added.
+%% @hidden Redispatch is needed only for the messages which not QOS_0
-is_redispatch_needed(#message{} = Msg) ->
+is_redispatch_needed(#message{qos = ?QOS_0}) ->
-    case get_redispatch_to(Msg) of
+    false;
-        ?REDISPATCH_TO(_, _) ->
+is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) ->
     true;
-        _ ->
+is_redispatch_needed(#message{}) ->
-            false
+    false.
-    end.
 
 %% @doc Redispatch shared deliveries to other members in the group.
 redispatch(Messages0) ->

apps/emqx/src/emqx_topic.erl

@@ -36,9 +36,16 @@
     parse/2
 ]).
 
+-export([
+    maybe_format_share/1,
+    get_shared_real_topic/1,
+    make_shared_record/2
+]).
+
 -type topic() :: emqx_types:topic().
 -type word() :: emqx_types:word().
 -type words() :: emqx_types:words().
+-type share() :: emqx_types:share().
 
 %% Guards
 -define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
@@ -50,7 +57,9 @@
 %%--------------------------------------------------------------------
 
 %% @doc Is wildcard topic?
--spec wildcard(topic() | words()) -> true | false.
+-spec wildcard(topic() | share() | words()) -> true | false.
+wildcard(#share{topic = Topic}) when is_binary(Topic) ->
+    wildcard(Topic);
 wildcard(Topic) when is_binary(Topic) ->
     wildcard(words(Topic));
 wildcard([]) ->
@@ -64,7 +73,7 @@ wildcard([_H | T]) ->
 
 %% @doc Match Topic name with filter.
 -spec match(Name, Filter) -> boolean() when
-    Name :: topic() | words(),
+    Name :: topic() | share() | words(),
     Filter :: topic() | words().
 match(<<$$, _/binary>>, <<$+, _/binary>>) ->
     false;
@@ -72,6 +81,10 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) ->
     false;
 match(Name, Filter) when is_binary(Name), is_binary(Filter) ->
     match(words(Name), words(Filter));
+match(#share{} = Name, Filter) ->
+    match_share(Name, Filter);
+match(Name, #share{} = Filter) ->
+    match_share(Name, Filter);
 match([], []) ->
     true;
 match([H | T1], [H | T2]) ->
@@ -87,12 +100,29 @@ match([_H1 | _], []) ->
 match([], [_H | _T2]) ->
     false.
 
+-spec match_share(Name, Filter) -> boolean() when
+    Name :: share(),
+    Filter :: topic() | share().
+match_share(#share{topic = Name}, Filter) when is_binary(Filter) ->
+    %% only match real topic filter for normal topic filter.
+    match(words(Name), words(Filter));
+match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) ->
+    %% Matching real topic filter When subed same share group.
+    match(words(Name), words(Filter));
+match_share(#share{}, _) ->
+    %% Otherwise, non-matched.
+    false;
+match_share(Name, #share{topic = Filter}) when is_binary(Name) ->
+    %% Only match real topic filter for normal topic_filter/topic_name.
+    match(Name, Filter).
+
 -spec match_any(Name, [Filter]) -> boolean() when
     Name :: topic() | words(),
     Filter :: topic() | words().
 match_any(Topic, Filters) ->
     lists:any(fun(Filter) -> match(Topic, Filter) end, Filters).
 
+%% TODO: validate share topic #share{} for emqx_trace.erl
 %% @doc Validate topic name or filter
 -spec validate(topic() | {name | filter, topic()}) -> true.
 validate(Topic) when is_binary(Topic) ->
@@ -107,7 +137,7 @@ validate(_, <<>>) ->
 validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
     %% MQTT-5.0 [MQTT-4.7.3-3]
     error(topic_too_long);
-validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
+validate(filter, SharedFilter = <<?SHARE, "/", _Rest/binary>>) ->
     validate_share(SharedFilter);
 validate(filter, Filter) when is_binary(Filter) ->
     validate2(words(Filter));
@@ -139,12 +169,12 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
 validate3(<<_/utf8, Rest/binary>>) ->
     validate3(Rest).
 
-validate_share(<<"$share/", Rest/binary>>) when
+validate_share(<<?SHARE, "/", Rest/binary>>) when
     Rest =:= <<>> orelse Rest =:= <<"/">>
 ->
     %% MQTT-5.0 [MQTT-4.8.2-1]
     error(?SHARE_EMPTY_FILTER);
-validate_share(<<"$share/", Rest/binary>>) ->
+validate_share(<<?SHARE, "/", Rest/binary>>) ->
     case binary:split(Rest, <<"/">>) of
         %% MQTT-5.0 [MQTT-4.8.2-1]
         [<<>>, _] ->
@@ -156,7 +186,7 @@ validate_share(<<"$share/", Rest/binary>>) ->
             validate_share(ShareName, Filter)
     end.
 
-validate_share(_, <<"$share/", _Rest/binary>>) ->
+validate_share(_, <<?SHARE, "/", _Rest/binary>>) ->
     error(?SHARE_RECURSIVELY);
 validate_share(ShareName, Filter) ->
     case binary:match(ShareName, [<<"+">>, <<"#">>]) of
@@ -185,7 +215,9 @@ bin('#') -> <<"#">>;
 bin(B) when is_binary(B) -> B;
 bin(L) when is_list(L) -> list_to_binary(L).
 
--spec levels(topic()) -> pos_integer().
+-spec levels(topic() | share()) -> pos_integer().
+levels(#share{topic = Topic}) when is_binary(Topic) ->
+    levels(Topic);
 levels(Topic) when is_binary(Topic) ->
     length(tokens(Topic)).
 
@@ -197,6 +229,8 @@ tokens(Topic) ->
 
 %% @doc Split Topic Path to Words
 -spec words(topic()) -> words().
+words(#share{topic = Topic}) when is_binary(Topic) ->
+    words(Topic);
 words(Topic) when is_binary(Topic) ->
     [word(W) || W <- tokens(Topic)].
 
@@ -237,26 +271,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
 do_join(TopicAcc, [Word | Words]) ->
     do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
 
--spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
+-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}.
 parse(TopicFilter) when is_binary(TopicFilter) ->
     parse(TopicFilter, #{});
 parse({TopicFilter, Options}) when is_binary(TopicFilter) ->
     parse(TopicFilter, Options).
 
--spec parse(topic(), map()) -> {topic(), map()}.
+-spec parse(topic() | share(), map()) -> {topic() | share(), map()}.
-parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) ->
+%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">>
-    error({invalid_topic_filter, TopicFilter});
+%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>`
-parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) ->
+parse(#share{topic = Topic = <<?QUEUE, "/", _/binary>>}, _Options) ->
-    error({invalid_topic_filter, TopicFilter});
+    error({invalid_topic_filter, Topic});
-parse(<<"$queue/", TopicFilter/binary>>, Options) ->
+parse(#share{topic = Topic = <<?SHARE, "/", _/binary>>}, _Options) ->
-    parse(TopicFilter, Options#{share => <<"$queue">>});
+    error({invalid_topic_filter, Topic});
-parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
+parse(<<?QUEUE, "/", Topic/binary>>, Options) ->
+    parse(#share{group = <<?QUEUE>>, topic = Topic}, Options);
+parse(TopicFilter = <<?SHARE, "/", Rest/binary>>, Options) ->
     case binary:split(Rest, <<"/">>) of
         [_Any] ->
             error({invalid_topic_filter, TopicFilter});
-        [ShareName, Filter] ->
+        %% `Group` could be `$share` or `$queue`
-            case binary:match(ShareName, [<<"+">>, <<"#">>]) of
+        [Group, Topic] ->
-                nomatch -> parse(Filter, Options#{share => ShareName});
+            case binary:match(Group, [<<"+">>, <<"#">>]) of
+                nomatch -> parse(#share{group = Group, topic = Topic}, Options);
                 _ -> error({invalid_topic_filter, TopicFilter})
             end
     end;
@@ -267,5 +304,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) ->
         _ ->
             {Topic, Options#{is_exclusive => true}}
     end;
-parse(TopicFilter, Options) ->
+parse(TopicFilter, Options) when
+    ?IS_TOPIC(TopicFilter)
+->
     {TopicFilter, Options}.
+
+get_shared_real_topic(#share{topic = TopicFilter}) ->
+    TopicFilter;
+get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) ->
+    TopicFilter.
+
+make_shared_record(Group, Topic) ->
+    #share{group = Group, topic = Topic}.
+
+maybe_format_share(#share{group = <<?QUEUE>>, topic = Topic}) ->
+    join([<<?QUEUE>>, Topic]);
+maybe_format_share(#share{group = Group, topic = Topic}) ->
+    join([<<?SHARE>>, Group, Topic]);
+maybe_format_share(Topic) ->
+    join([Topic]).

apps/emqx/src/emqx_trace/emqx_trace.erl

@@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
         ignore ->
             ignore;
         Log ->
-            case logger_config:get(ets:whereis(logger), Id) of
+            case logger_config:get(logger, Id) of
                 {ok, #{module := Module} = HandlerConfig0} ->
                     HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
                     try

apps/emqx/src/emqx_trie_search.erl

@@ -98,7 +98,7 @@
 
 -module(emqx_trie_search).
 
--export([make_key/2, filter/1]).
+-export([make_key/2, make_pat/2, filter/1]).
 -export([match/2, matches/3, get_id/1, get_topic/1]).
 -export_type([key/1, word/0, words/0, nextf/0, opts/0]).
 
@@ -127,6 +127,12 @@ make_key(Topic, ID) when is_binary(Topic) ->
 make_key(Words, ID) when is_list(Words) ->
     {Words, {ID}}.
 
+-spec make_pat(emqx_types:topic() | words() | '_', _ID | '_') -> _Pat.
+make_pat(Pattern = '_', ID) ->
+    {Pattern, {ID}};
+make_pat(Topic, ID) ->
+    make_key(Topic, ID).
+
 %% @doc Parse a topic filter into a list of words. Returns `false` if it's not a filter.
 -spec filter(emqx_types:topic()) -> words() | false.
 filter(Topic) ->

apps/emqx/src/emqx_types.erl

@@ -40,6 +40,10 @@
     words/0
 ]).
 
+-export_type([
+    share/0
+]).
+
 -export_type([
     socktype/0,
     sockstate/0,
@@ -136,11 +140,14 @@
 
 -type subid() :: binary() | atom().
 
--type group() :: binary() | undefined.
+%% '_' for match spec
+-type group() :: binary() | '_'.
 -type topic() :: binary().
 -type word() :: '' | '+' | '#' | binary().
 -type words() :: list(word()).
 
+-type share() :: #share{}.
+
 -type socktype() :: tcp | udp | ssl | proxy | atom().
 -type sockstate() :: idle | running | blocked | closed.
 -type conninfo() :: #{
@@ -207,7 +214,6 @@
     rap := 0 | 1,
     nl := 0 | 1,
     qos := qos(),
-    share => binary(),
     atom() => term()
 }.
 -type reason_code() :: 0..16#FF.

apps/emqx/src/emqx_vm.erl

@@ -418,6 +418,9 @@ get_otp_version() ->
     end.
 
 read_otp_version() ->
+    string:trim(do_read_otp_version()).
+
+do_read_otp_version() ->
     ReleasesDir = filename:join([code:root_dir(), "releases"]),
     Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILD_INFO"]),
     case file:read_file(Filename) of

apps/emqx/test/emqx_broker_SUITE.erl

@@ -299,14 +299,19 @@ t_nosub_pub(Config) when is_list(Config) ->
     ?assertEqual(1, emqx_metrics:val('messages.dropped')).
 
 t_shared_subscribe({init, Config}) ->
-    emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}),
+    emqx_broker:subscribe(
+        emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{}
+    ),
     ct:sleep(100),
     Config;
 t_shared_subscribe(Config) when is_list(Config) ->
     emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)),
     ?assert(
         receive
-            {deliver, <<"topic">>, #message{payload = <<"hello">>}} ->
+            {deliver, <<"topic">>, #message{
+                headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)},
+                payload = <<"hello">>
+            }} ->
                 true;
             Msg ->
                 ct:pal("Msg: ~p", [Msg]),
@@ -316,7 +321,7 @@ t_shared_subscribe(Config) when is_list(Config) ->
         end
     );
 t_shared_subscribe({'end', _Config}) ->
-    emqx_broker:unsubscribe(<<"$share/group/topic">>).
+    emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)).
 
 t_shared_subscribe_2({init, Config}) ->
     Config;
@@ -723,24 +728,6 @@ t_connack_auth_error(Config) when is_list(Config) ->
     ?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')),
     ok.
 
-t_handle_in_empty_client_subscribe_hook({init, Config}) ->
-    Hook = {?MODULE, client_subscribe_delete_all_hook, []},
-    ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100),
-    Config;
-t_handle_in_empty_client_subscribe_hook({'end', _Config}) ->
-    emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}),
-    ok;
-t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) ->
-    {ok, C} = emqtt:start_link(),
-    {ok, _} = emqtt:connect(C),
-    try
-        {ok, _, RCs} = emqtt:subscribe(C, <<"t">>),
-        ?assertEqual([?RC_UNSPECIFIED_ERROR], RCs),
-        ok
-    after
-        emqtt:disconnect(C)
-    end.
-
 authenticate_deny(_Credentials, _Default) ->
     {stop, {error, bad_username_or_password}}.
 
@@ -800,7 +787,3 @@ recv_msgs(Count, Msgs) ->
     after 100 ->
         Msgs
     end.
-
-client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) ->
-    EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter],
-    {stop, EmptyFilters}.

apps/emqx/test/emqx_channel_SUITE.erl

@@ -456,7 +456,7 @@ t_process_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
     TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
     {[{TopicFilter, ?RC_SUCCESS}], _Channel} =
-        emqx_channel:process_subscribe(TopicFilters, #{}, channel()).
+        emqx_channel:process_subscribe(TopicFilters, channel()).
 
 t_process_unsubscribe(_) ->
     ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end),
@@ -914,7 +914,13 @@ t_check_pub_alias(_) ->
 t_check_sub_authzs(_) ->
     emqx_config:put_zone_conf(default, [authorization, enable], true),
     TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
-    [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()).
+    SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]),
+    CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]),
+    Channel = channel(),
+    ?assertEqual(
+        {ok, CheckedSubPkt, Channel},
+        emqx_channel:check_sub_authzs(SubPkt, Channel)
+    ).
 
 t_enrich_connack_caps(_) ->
     ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),
@@ -1061,6 +1067,7 @@ clientinfo(InitProps) ->
             clientid => <<"clientid">>,
             username => <<"username">>,
             is_superuser => false,
+            is_bridge => false,
             mountpoint => undefined
         },
         InitProps

apps/emqx/test/emqx_common_test_helpers.erl

@@ -753,24 +753,15 @@ start_slave(Name, Opts) when is_map(Opts) ->
             case SlaveMod of
                 ct_slave ->
                     ct:pal("~p: node data dir: ~s", [Node, NodeDataDir]),
-                    ct_slave:start(
+                    Envs = [
-                        Node,
-                        [
-                            {kill_if_fail, true},
-                            {monitor_master, true},
-                            {init_timeout, 20_000},
-                            {startup_timeout, 20_000},
-                            {erl_flags, erl_flags()},
-                            {env, [
                         {"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"},
                         {"EMQX_NODE__COOKIE", Cookie},
                         {"EMQX_NODE__DATA_DIR", NodeDataDir}
-                            ]}
+                    ],
-                        ]
+                    emqx_cth_peer:start(Node, erl_flags(), Envs);
-                    );
                 slave ->
-                    Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_",
+                    Envs = [{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}],
-                    slave:start_link(host(), Name, ebin_path() ++ Env)
+                    emqx_cth_peer:start(Node, ebin_path(), Envs)
             end
         end,
     case DoStart() of
@@ -789,13 +780,7 @@ start_slave(Name, Opts) when is_map(Opts) ->
 %% Node stopping
 stop_slave(Node0) ->
     Node = node_name(Node0),
-    SlaveMod = get_peer_mod(Node),
+    emqx_cth_peer:stop(Node).
-    erase_peer_mod(Node),
-    case SlaveMod:stop(Node) of
-        ok -> ok;
-        {ok, _} -> ok;
-        {error, not_started, _} -> ok
-    end.
 
 %% EPMD starting
 start_epmd() ->
@@ -1022,11 +1007,11 @@ set_envs(Node, Env) ->
     ).
 
 erl_flags() ->
-    %% One core and redirecting logs to master
+    %% One core
-    "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
+    ["+S", "1:1"] ++ ebin_path().
 
 ebin_path() ->
-    string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
+    ["-pa" | lists:filter(fun is_lib/1, code:get_path())].
 
 is_lib(Path) ->
     string:prefix(Path, code:lib_dir()) =:= nomatch andalso

apps/emqx/test/emqx_common_test_http.erl

@@ -34,6 +34,9 @@
 -define(DEFAULT_APP_KEY, <<"default_app_key">>).
 -define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
 
+%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
+-define(ROLE_API_SUPERUSER, <<"administrator">>).
+
 request_api(Method, Url, Auth) ->
     request_api(Method, Url, [], Auth, []).
 
@@ -96,7 +99,8 @@ create_default_app() ->
         ?DEFAULT_APP_SECRET,
         true,
         ExpiredAt,
-        <<"default app key for test">>
+        <<"default app key for test">>,
+        ?ROLE_API_SUPERUSER
     ).
 
 delete_default_app() ->

apps/emqx/test/emqx_cth_cluster.erl

@@ -38,19 +38,19 @@
 %%    in `end_per_suite/1` or `end_per_group/2`) with the result from step 2.
 -module(emqx_cth_cluster).
 
--export([start/2]).
+-export([start/1, start/2, restart/2]).
 -export([stop/1, stop_node/1]).
 
--export([start_bare_node/2]).
+-export([start_bare_nodes/1, start_bare_nodes/2]).
 
 -export([share_load_module/2]).
 -export([node_name/1, mk_nodespecs/2]).
--export([start_apps/2, set_node_opts/2]).
+-export([start_apps/2]).
 
 -define(APPS_CLUSTERING, [gen_rpc, mria, ekka]).
 
 -define(TIMEOUT_NODE_START_MS, 15000).
--define(TIMEOUT_APPS_START_MS, 60000).
+-define(TIMEOUT_APPS_START_MS, 30000).
 -define(TIMEOUT_NODE_STOP_S, 15).
 
 %%
@@ -109,9 +109,12 @@ when
     }.
 start(Nodes, ClusterOpts) ->
     NodeSpecs = mk_nodespecs(Nodes, ClusterOpts),
-    ct:pal("Starting cluster:\n  ~p", [NodeSpecs]),
+    start(NodeSpecs).
+
+start(NodeSpecs) ->
+    ct:pal("(Re)starting nodes:\n  ~p", [NodeSpecs]),
     % 1. Start bare nodes with only basic applications running
-    _ = emqx_utils:pmap(fun start_node_init/1, NodeSpecs, ?TIMEOUT_NODE_START_MS),
+    ok = start_nodes_init(NodeSpecs, ?TIMEOUT_NODE_START_MS),
     % 2. Start applications needed to enable clustering
     % Generally, this causes some applications to restart, but we deliberately don't
     % start them yet.
@@ -121,6 +124,11 @@ start(Nodes, ClusterOpts) ->
     _ = emqx_utils:pmap(fun run_node_phase_apps/1, NodeSpecs, ?TIMEOUT_APPS_START_MS),
     [Node || #{name := Node} <- NodeSpecs].
 
+restart(Node, Spec) ->
+    ct:pal("Stopping peer node ~p", [Node]),
+    ok = emqx_cth_peer:stop(Node),
+    start([Spec#{boot_type => restart}]).
+
 mk_nodespecs(Nodes, ClusterOpts) ->
     NodeSpecs = lists:zipwith(
         fun(N, {Name, Opts}) -> mk_init_nodespec(N, Name, Opts, ClusterOpts) end,
@@ -282,8 +290,50 @@ allocate_listener_port(Type, #{base_port := BasePort}) ->
 allocate_listener_ports(Types, Spec) ->
     lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]).
 
-start_node_init(Spec = #{name := Node}) ->
+start_nodes_init(Specs, Timeout) ->
-    Node = start_bare_node(Node, Spec),
+    Names = lists:map(fun(#{name := Name}) -> Name end, Specs),
+    Nodes = start_bare_nodes(Names, Timeout),
+    lists:foreach(fun node_init/1, Nodes).
+
+start_bare_nodes(Names) ->
+    start_bare_nodes(Names, ?TIMEOUT_NODE_START_MS).
+start_bare_nodes(Names, Timeout) ->
+    Args = erl_flags(),
+    Envs = [],
+    Waits = lists:map(
+        fun(Name) ->
+            WaitTag = {boot_complete, Name},
+            WaitBoot = {self(), WaitTag},
+            {ok, _} = emqx_cth_peer:start(Name, Args, Envs, WaitBoot),
+            WaitTag
+        end,
+        Names
+    ),
+    Deadline = erlang:monotonic_time() + erlang:convert_time_unit(Timeout, millisecond, nanosecond),
+    Nodes = wait_boot_complete(Waits, Deadline),
+    lists:foreach(fun(Node) -> pong = net_adm:ping(Node) end, Nodes),
+    Nodes.
+
+wait_boot_complete([], _) ->
+    [];
+wait_boot_complete(Waits, Deadline) ->
+    case erlang:monotonic_time() > Deadline of
+        true ->
+            error({timeout, Waits});
+        false ->
+            ok
+    end,
+    receive
+        {{boot_complete, _Name} = Wait, {started, Node, _Pid}} ->
+            ct:pal("~p", [Wait]),
+            [Node | wait_boot_complete(Waits -- [Wait], Deadline)];
+        {{boot_complete, _Name}, Otherwise} ->
+            error({unexpected, Otherwise})
+    after 100 ->
+        wait_boot_complete(Waits, Deadline)
+    end.
+
+node_init(Node) ->
     % Make it possible to call `ct:pal` and friends (if running under rebar3)
     _ = share_load_module(Node, cthr),
     % Enable snabbkaffe trace forwarding
@@ -300,12 +350,6 @@ run_node_phase_apps(Spec = #{name := Node}) ->
     ok = start_apps(Node, Spec),
     ok.
 
-set_node_opts(Node, Spec) ->
-    erpc:call(Node, persistent_term, put, [{?MODULE, opts}, Spec]).
-
-get_node_opts(Node) ->
-    erpc:call(Node, persistent_term, get, [{?MODULE, opts}]).
-
 load_apps(Node, #{apps := Apps}) ->
     erpc:call(Node, emqx_cth_suite, load_apps, [Apps]).
 
@@ -322,8 +366,12 @@ start_apps(Node, #{apps := Apps} = Spec) ->
     ok.
 
 suite_opts(Spec) ->
-    maps:with([work_dir], Spec).
+    maps:with([work_dir, boot_type], Spec).
 
+maybe_join_cluster(_Node, #{boot_type := restart}) ->
+    %% when restart, the node should already be in the cluster
+    %% hence no need to (re)join
+    ok;
 maybe_join_cluster(_Node, #{role := replicant}) ->
     ok;
 maybe_join_cluster(Node, Spec) ->
@@ -352,23 +400,7 @@ stop(Nodes) ->
 
 stop_node(Name) ->
     Node = node_name(Name),
-    try get_node_opts(Node) of
+    ok = emqx_cth_peer:stop(Node).
-        Opts ->
-            stop_node(Name, Opts)
-    catch
-        error:{erpc, _} ->
-            ok
-    end.
-
-stop_node(Node, #{driver := ct_slave}) ->
-    case ct_slave:stop(Node, [{stop_timeout, ?TIMEOUT_NODE_STOP_S}]) of
-        {ok, _} ->
-            ok;
-        {error, Reason, _} when Reason == not_connected; Reason == not_started ->
-            ok
-    end;
-stop_node(Node, #{driver := slave}) ->
-    slave:stop(Node).
 
 %% Ports
 
@@ -391,36 +423,12 @@ listener_port(BasePort, wss) ->
 
 %%
 
--spec start_bare_node(atom(), map()) -> node().
-start_bare_node(Name, Spec = #{driver := ct_slave}) ->
-    {ok, Node} = ct_slave:start(
-        node_name(Name),
-        [
-            {kill_if_fail, true},
-            {monitor_master, true},
-            {init_timeout, 20_000},
-            {startup_timeout, 20_000},
-            {erl_flags, erl_flags()},
-            {env, []}
-        ]
-    ),
-    init_bare_node(Node, Spec);
-start_bare_node(Name, Spec = #{driver := slave}) ->
-    {ok, Node} = slave:start_link(host(), Name, ebin_path()),
-    init_bare_node(Node, Spec).
-
-init_bare_node(Node, Spec) ->
-    pong = net_adm:ping(Node),
-    % Preserve node spec right on the remote node
-    ok = set_node_opts(Node, Spec),
-    Node.
-
 erl_flags() ->
-    %% One core and redirecting logs to master
+    %% One core
-    "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
+    ["+S", "1:1"] ++ ebin_path().
 
 ebin_path() ->
-    string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
+    ["-pa" | lists:filter(fun is_lib/1, code:get_path())].
 
 is_lib(Path) ->
     string:prefix(Path, code:lib_dir()) =:= nomatch andalso

apps/emqx/test/emqx_cth_peer.erl

@@ -0,0 +1,79 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc Common Test Helper proxy module for slave -> peer migration.
+%% OTP 26 has slave module deprecated, use peer instead.
+
+-module(emqx_cth_peer).
+
+-export([start/2, start/3, start/4]).
+-export([start_link/2, start_link/3, start_link/4]).
+-export([stop/1]).
+
+start(Name, Args) ->
+    start(Name, Args, []).
+
+start(Name, Args, Envs) ->
+    start(Name, Args, Envs, timer:seconds(20)).
+
+start(Name, Args, Envs, Timeout) when is_atom(Name) ->
+    do_start(Name, Args, Envs, Timeout, start).
+
+start_link(Name, Args) ->
+    start_link(Name, Args, []).
+
+start_link(Name, Args, Envs) ->
+    start_link(Name, Args, Envs, timer:seconds(20)).
+
+start_link(Name, Args, Envs, Timeout) when is_atom(Name) ->
+    do_start(Name, Args, Envs, Timeout, start_link).
+
+do_start(Name0, Args, Envs, Timeout, Func) when is_atom(Name0) ->
+    {Name, Host} = parse_node_name(Name0),
+    {ok, Pid, Node} = peer:Func(#{
+        name => Name,
+        host => Host,
+        args => Args,
+        env => Envs,
+        wait_boot => Timeout,
+        longnames => true,
+        shutdown => {halt, 1000}
+    }),
+    true = register(Node, Pid),
+    {ok, Node}.
+
+stop(Node) when is_atom(Node) ->
+    Pid = whereis(Node),
+    case is_pid(Pid) of
+        true ->
+            unlink(Pid),
+            ok = peer:stop(Pid);
+        false ->
+            ct:pal("The control process for node ~p is unexpetedly down", [Node]),
+            ok
+    end.
+
+parse_node_name(NodeName) ->
+    case string:tokens(atom_to_list(NodeName), "@") of
+        [Name, Host] ->
+            {list_to_atom(Name), Host};
+        [_] ->
+            {NodeName, host()}
+    end.
+
+host() ->
+    [_Name, Host] = string:tokens(atom_to_list(node()), "@"),
+    Host.

apps/emqx/test/emqx_cth_suite.erl

@@ -453,6 +453,9 @@ stop_apps(Apps) ->
 
 %%
 
+verify_clean_suite_state(#{boot_type := restart}) ->
+    %% when testing node restart, we do not need to verify clean state
+    ok;
 verify_clean_suite_state(#{work_dir := WorkDir}) ->
     {ok, []} = file:list_dir(WorkDir),
     false = emqx_schema_hooks:any_injections(),

apps/emqx/test/emqx_metrics_worker_SUITE.erl

@@ -53,9 +53,9 @@ t_get_metrics(_) ->
     ?assertMatch(
         #{
             rate := #{
-                a := #{current := 0.0, max := 0.0, last5m := 0.0},
+                a := #{current := +0.0, max := +0.0, last5m := +0.0},
-                b := #{current := 0.0, max := 0.0, last5m := 0.0},
+                b := #{current := +0.0, max := +0.0, last5m := +0.0},
-                c := #{current := 0.0, max := 0.0, last5m := 0.0}
+                c := #{current := +0.0, max := +0.0, last5m := +0.0}
             },
             gauges := #{},
             counters := #{
@@ -118,9 +118,9 @@ t_clear_metrics(_Config) ->
     ?assertMatch(
         #{
             rate := #{
-                a := #{current := 0.0, max := 0.0, last5m := 0.0},
+                a := #{current := +0.0, max := +0.0, last5m := +0.0},
-                b := #{current := 0.0, max := 0.0, last5m := 0.0},
+                b := #{current := +0.0, max := +0.0, last5m := +0.0},
-                c := #{current := 0.0, max := 0.0, last5m := 0.0}
+                c := #{current := +0.0, max := +0.0, last5m := +0.0}
             },
             gauges := #{},
             slides := #{},
@@ -145,7 +145,7 @@ t_clear_metrics(_Config) ->
         #{
             counters => #{},
             gauges => #{},
-            rate => #{current => 0.0, last5m => 0.0, max => 0.0},
+            rate => #{current => +0.0, last5m => +0.0, max => +0.0},
             slides => #{}
         },
         emqx_metrics_worker:get_metrics(?NAME, Id)
@@ -160,9 +160,9 @@ t_reset_metrics(_) ->
     ?assertMatch(
         #{
             rate := #{
-                a := #{current := 0.0, max := 0.0, last5m := 0.0},
+                a := #{current := +0.0, max := +0.0, last5m := +0.0},
-                b := #{current := 0.0, max := 0.0, last5m := 0.0},
+                b := #{current := +0.0, max := +0.0, last5m := +0.0},
-                c := #{current := 0.0, max := 0.0, last5m := 0.0}
+                c := #{current := +0.0, max := +0.0, last5m := +0.0}
             },
             gauges := #{},
             counters := #{

apps/emqx/test/emqx_mountpoint_SUITE.erl

@@ -29,6 +29,7 @@
 ).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
@@ -52,6 +53,24 @@ t_mount(_) ->
         mount(<<"device/1/">>, TopicFilters)
     ).
 
+t_mount_share(_) ->
+    T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>),
+    TopicFilters = [T],
+    ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
+
+    ?assertEqual(
+        TopicFilter,
+        mount(undefined, TopicFilter)
+    ),
+    ?assertEqual(
+        #share{group = <<"group">>, topic = <<"device/1/topic">>},
+        mount(<<"device/1/">>, TopicFilter)
+    ),
+    ?assertEqual(
+        [{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}],
+        mount(<<"device/1/">>, TopicFilters)
+    ).
+
 t_unmount(_) ->
     Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>),
     ?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)),
@@ -61,6 +80,21 @@ t_unmount(_) ->
     ?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)),
     ?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)).
 
+t_unmount_share(_) ->
+    {TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>),
+    MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>},
+
+    ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
+
+    ?assertEqual(
+        TopicFilter,
+        unmount(undefined, TopicFilter)
+    ),
+    ?assertEqual(
+        #share{group = <<"group">>, topic = <<"topic">>},
+        unmount(<<"device/1/">>, MountedTopicFilter)
+    ).
+
 t_replvar(_) ->
     ?assertEqual(undefined, replvar(undefined, #{})),
     ?assertEqual(

apps/emqx/test/emqx_mqtt_caps_SUITE.erl

@@ -76,6 +76,8 @@ t_check_sub(_) ->
     ),
     ?assertEqual(
         {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
-        emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})
+        emqx_mqtt_caps:check_sub(
+            ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts
+        )
     ),
     emqx_config:put([zones], OldConf).

apps/emqx/test/emqx_persistent_messages_SUITE.erl

@@ -233,6 +233,55 @@ t_session_subscription_iterators(Config) ->
     ),
     ok.
 
+t_qos0(_Config) ->
+    Sub = connect(<<?MODULE_STRING "1">>, true, 30),
+    Pub = connect(<<?MODULE_STRING "2">>, true, 0),
+    try
+        {ok, _, [1]} = emqtt:subscribe(Sub, <<"t/#">>, qos1),
+
+        Messages = [
+            {<<"t/1">>, <<"1">>, 0},
+            {<<"t/1">>, <<"2">>, 1},
+            {<<"t/1">>, <<"3">>, 0}
+        ],
+        [emqtt:publish(Pub, Topic, Payload, Qos) || {Topic, Payload, Qos} <- Messages],
+        ?assertMatch(
+            [
+                #{qos := 0, topic := <<"t/1">>, payload := <<"1">>},
+                #{qos := 1, topic := <<"t/1">>, payload := <<"2">>},
+                #{qos := 0, topic := <<"t/1">>, payload := <<"3">>}
+            ],
+            receive_messages(3)
+        )
+    after
+        emqtt:stop(Sub),
+        emqtt:stop(Pub)
+    end.
+
+t_publish_as_persistent(_Config) ->
+    Sub = connect(<<?MODULE_STRING "1">>, true, 30),
+    Pub = connect(<<?MODULE_STRING "2">>, true, 30),
+    try
+        {ok, _, [1]} = emqtt:subscribe(Sub, <<"t/#">>, qos1),
+        Messages = [
+            {<<"t/1">>, <<"1">>, 0},
+            {<<"t/1">>, <<"2">>, 1},
+            {<<"t/1">>, <<"3">>, 2}
+        ],
+        [emqtt:publish(Pub, Topic, Payload, Qos) || {Topic, Payload, Qos} <- Messages],
+        ?assertMatch(
+            [
+                #{qos := 0, topic := <<"t/1">>, payload := <<"1">>},
+                #{qos := 1, topic := <<"t/1">>, payload := <<"2">>},
+                #{qos := 2, topic := <<"t/1">>, payload := <<"3">>}
+            ],
+            receive_messages(3)
+        )
+    after
+        emqtt:stop(Sub),
+        emqtt:stop(Pub)
+    end.
+
 %%
 
 connect(ClientId, CleanStart, EI) ->
@@ -273,7 +322,7 @@ consume(It) ->
     end.
 
 receive_messages(Count) ->
-    receive_messages(Count, []).
+    lists:reverse(receive_messages(Count, [])).
 
 receive_messages(0, Msgs) ->
     Msgs;
@@ -291,7 +340,7 @@ publish(Node, Message) ->
 app_specs() ->
     [
         emqx_durable_storage,
-        {emqx, "persistent_session_store {ds = true}"}
+        {emqx, "session_persistence {enable = true}"}
     ].
 
 cluster() ->
@@ -307,4 +356,6 @@ get_mqtt_port(Node, Type) ->
 
 clear_db() ->
     ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB),
+    mria:stop(),
+    ok = mnesia:delete_schema([node()]),
     ok.

apps/emqx/test/emqx_persistent_session_SUITE.erl

@@ -17,6 +17,7 @@
 -module(emqx_persistent_session_SUITE).
 
 -include_lib("stdlib/include/assert.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
@@ -35,8 +36,8 @@ all() ->
         % NOTE
         % Tests are disabled while existing session persistence impl is being
         % phased out.
-        {group, persistent_store_disabled},
+        {group, persistence_disabled},
-        {group, persistent_store_ds}
+        {group, persistence_enabled}
     ].
 
 %% A persistent session can be resumed in two ways:
@@ -53,28 +54,28 @@ all() ->
 groups() ->
     TCs = emqx_common_test_helpers:all(?MODULE),
     TCsNonGeneric = [t_choose_impl],
+    TCGroups = [{group, tcp}, {group, quic}, {group, ws}],
     [
-        {persistent_store_disabled, [{group, no_kill_connection_process}]},
+        {persistence_disabled, TCGroups},
-        {persistent_store_ds, [{group, no_kill_connection_process}]},
+        {persistence_enabled, TCGroups},
-        {no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]},
         {tcp, [], TCs},
         {quic, [], TCs -- TCsNonGeneric},
         {ws, [], TCs -- TCsNonGeneric}
     ].
 
-init_per_group(persistent_store_disabled, Config) ->
+init_per_group(persistence_disabled, Config) ->
     [
-        {emqx_config, "persistent_session_store { enabled = false }"},
+        {emqx_config, "session_persistence { enable = false }"},
-        {persistent_store, false}
+        {persistence, false}
         | Config
     ];
-init_per_group(persistent_store_ds, Config) ->
+init_per_group(persistence_enabled, Config) ->
     [
-        {emqx_config, "persistent_session_store { ds = true }"},
+        {emqx_config, "session_persistence { enable = true }"},
-        {persistent_store, ds}
+        {persistence, ds}
         | Config
     ];
-init_per_group(Group, Config) when Group == tcp ->
+init_per_group(tcp, Config) ->
     Apps = emqx_cth_suite:start(
         [{emqx, ?config(emqx_config, Config)}],
         #{work_dir => emqx_cth_suite:work_dir(Config)}
@@ -85,7 +86,7 @@ init_per_group(Group, Config) when Group == tcp ->
         {group_apps, Apps}
         | Config
     ];
-init_per_group(Group, Config) when Group == ws ->
+init_per_group(ws, Config) ->
     Apps = emqx_cth_suite:start(
         [{emqx, ?config(emqx_config, Config)}],
         #{work_dir => emqx_cth_suite:work_dir(Config)}
@@ -99,7 +100,7 @@ init_per_group(Group, Config) when Group == ws ->
         {group_apps, Apps}
         | Config
     ];
-init_per_group(Group, Config) when Group == quic ->
+init_per_group(quic, Config) ->
     Apps = emqx_cth_suite:start(
         [
             {emqx,
@@ -118,11 +119,7 @@ init_per_group(Group, Config) when Group == quic ->
         {ssl, true},
         {group_apps, Apps}
         | Config
-    ];
+    ].
-init_per_group(no_kill_connection_process, Config) ->
-    [{kill_connection_process, false} | Config];
-init_per_group(kill_connection_process, Config) ->
-    [{kill_connection_process, true} | Config].
 
 get_listener_port(Type, Name) ->
     case emqx_config:get([listeners, Type, Name, bind]) of
@@ -181,64 +178,106 @@ client_info(Key, Client) ->
     maps:get(Key, maps:from_list(emqtt:info(Client)), undefined).
 
 receive_messages(Count) ->
-    receive_messages(Count, []).
+    receive_messages(Count, 15000).
 
-receive_messages(0, Msgs) ->
+receive_messages(Count, Timeout) ->
-    Msgs;
+    Deadline = erlang:monotonic_time(millisecond) + Timeout,
-receive_messages(Count, Msgs) ->
+    receive_message_loop(Count, Deadline).
+
+receive_message_loop(0, _Deadline) ->
+    [];
+receive_message_loop(Count, Deadline) ->
+    Timeout = max(0, Deadline - erlang:monotonic_time(millisecond)),
     receive
         {publish, Msg} ->
-            receive_messages(Count - 1, [Msg | Msgs]);
+            [Msg | receive_message_loop(Count - 1, Deadline)];
+        {pubrel, Msg} ->
+            [{pubrel, Msg} | receive_message_loop(Count - 1, Deadline)];
         _Other ->
-            receive_messages(Count, Msgs)
+            receive_message_loop(Count, Deadline)
-    after 15000 ->
+    after Timeout ->
-        Msgs
+        []
     end.
 
 maybe_kill_connection_process(ClientId, Config) ->
-    case ?config(kill_connection_process, Config) of
+    Persistence = ?config(persistence, Config),
-        true ->
+    case emqx_cm:lookup_channels(ClientId) of
+        [] ->
+            ok;
+        [ConnectionPid] when Persistence == ds ->
+            Ref = monitor(process, ConnectionPid),
+            ConnectionPid ! die_if_test,
+            ?assertReceive(
+                {'DOWN', Ref, process, ConnectionPid, Reason} when
+                    Reason == normal orelse Reason == noproc,
+                3000
+            ),
+            wait_connection_process_unregistered(ClientId);
+        _ ->
+            ok
+    end.
+
+wait_connection_process_dies(ClientId) ->
     case emqx_cm:lookup_channels(ClientId) of
         [] ->
             ok;
         [ConnectionPid] ->
-                    ?assert(is_pid(ConnectionPid)),
             Ref = monitor(process, ConnectionPid),
-                    ConnectionPid ! die_if_test,
+            ?assertReceive(
-                    receive
+                {'DOWN', Ref, process, ConnectionPid, Reason} when
-                        {'DOWN', Ref, process, ConnectionPid, normal} -> ok
+                    Reason == normal orelse Reason == noproc,
-                    after 3000 -> error(process_did_not_die)
+                3000
-                    end,
+            ),
-                    wait_for_cm_unregister(ClientId)
+            wait_connection_process_unregistered(ClientId)
-            end;
-        false ->
-            ok
     end.
 
-wait_for_cm_unregister(ClientId) ->
+wait_connection_process_unregistered(ClientId) ->
-    wait_for_cm_unregister(ClientId, 100).
+    ?retry(
+        _Timeout = 100,
+        _Retries = 20,
+        ?assertEqual([], emqx_cm:lookup_channels(ClientId))
+    ).
 
-wait_for_cm_unregister(_ClientId, 0) ->
+wait_channel_disconnected(ClientId) ->
-    error(cm_did_not_unregister);
+    ?retry(
-wait_for_cm_unregister(ClientId, N) ->
+        _Timeout = 100,
+        _Retries = 20,
         case emqx_cm:lookup_channels(ClientId) of
             [] ->
-            ok;
+                false;
-        [_] ->
+            [ChanPid] ->
-            timer:sleep(100),
+                false = emqx_cm:is_channel_connected(ChanPid)
-            wait_for_cm_unregister(ClientId, N - 1)
+        end
-    end.
+    ).
 
-publish(Topic, Payloads) ->
+disconnect_client(ClientPid) ->
-    publish(Topic, Payloads, false, 2).
+    ClientId = proplists:get_value(clientid, emqtt:info(ClientPid)),
+    ok = emqtt:disconnect(ClientPid),
+    false = wait_channel_disconnected(ClientId),
+    ok.
 
-publish(Topic, Payloads, WaitForUnregister, QoS) ->
+messages(Topic, Payloads) ->
-    Fun = fun(Client, Payload) ->
+    messages(Topic, Payloads, ?QOS_2).
-        {ok, _} = emqtt:publish(Client, Topic, Payload, QoS)
+
+messages(Topic, Payloads, QoS) ->
+    [#mqtt_msg{topic = Topic, payload = P, qos = QoS} || P <- Payloads].
+
+publish(Topic, Payload) ->
+    publish(Topic, Payload, ?QOS_2).
+
+publish(Topic, Payload, QoS) ->
+    publish_many(messages(Topic, [Payload], QoS)).
+
+publish_many(Messages) ->
+    publish_many(Messages, false).
+
+publish_many(Messages, WaitForUnregister) ->
+    Fun = fun(Client, Message) ->
+        {ok, _} = emqtt:publish(Client, Message)
     end,
-    do_publish(Payloads, Fun, WaitForUnregister).
+    do_publish(Messages, Fun, WaitForUnregister).
 
-do_publish(Payloads = [_ | _], PublishFun, WaitForUnregister) ->
+do_publish(Messages = [_ | _], PublishFun, WaitForUnregister) ->
     %% Publish from another process to avoid connection confusion.
     {Pid, Ref} =
         spawn_monitor(
@@ -252,34 +291,16 @@ do_publish(Payloads = [_ | _], PublishFun, WaitForUnregister) ->
                     {port, 1883}
                 ]),
                 {ok, _} = emqtt:connect(Client),
-                lists:foreach(fun(Payload) -> PublishFun(Client, Payload) end, Payloads),
+                lists:foreach(fun(Message) -> PublishFun(Client, Message) end, Messages),
                 ok = emqtt:disconnect(Client),
                 %% Snabbkaffe sometimes fails unless all processes are gone.
-                case WaitForUnregister of
+                WaitForUnregister andalso wait_connection_process_dies(ClientID)
-                    false ->
-                        ok;
-                    true ->
-                        case emqx_cm:lookup_channels(ClientID) of
-                            [] ->
-                                ok;
-                            [ConnectionPid] ->
-                                ?assert(is_pid(ConnectionPid)),
-                                Ref1 = monitor(process, ConnectionPid),
-                                receive
-                                    {'DOWN', Ref1, process, ConnectionPid, _} -> ok
-                                after 3000 -> error(process_did_not_die)
-                                end,
-                                wait_for_cm_unregister(ClientID)
-                        end
-                end
             end
         ),
     receive
         {'DOWN', Ref, process, Pid, normal} -> ok;
         {'DOWN', Ref, process, Pid, What} -> error({failed_publish, What})
-    end;
+    end.
-do_publish(Payload, PublishFun, WaitForUnregister) ->
-    do_publish([Payload], PublishFun, WaitForUnregister).
 
 %%--------------------------------------------------------------------
 %% Test Cases
@@ -297,7 +318,7 @@ t_choose_impl(Config) ->
     {ok, _} = emqtt:ConnFun(Client),
     [ChanPid] = emqx_cm:lookup_channels(ClientId),
     ?assertEqual(
-        case ?config(persistent_store, Config) of
+        case ?config(persistence, Config) of
             false -> emqx_session_mem;
             ds -> emqx_persistent_session_ds
         end,
@@ -332,8 +353,6 @@ t_connect_discards_existing_client(Config) ->
     end.
 
 %% [MQTT-3.1.2-23]
-t_connect_session_expiry_interval(init, Config) -> skip_ds_tc(Config);
-t_connect_session_expiry_interval('end', _Config) -> ok.
 t_connect_session_expiry_interval(Config) ->
     ConnFun = ?config(conn_fun, Config),
     Topic = ?config(topic, Config),
@@ -341,6 +360,45 @@ t_connect_session_expiry_interval(Config) ->
     Payload = <<"test message">>,
     ClientId = ?config(client_id, Config),
 
+    {ok, Client1} = emqtt:start_link([
+        {clientid, ClientId},
+        {proto_ver, v5},
+        {properties, #{'Session-Expiry-Interval' => 30}}
+        | Config
+    ]),
+    {ok, _} = emqtt:ConnFun(Client1),
+    {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client1, STopic, ?QOS_1),
+    ok = emqtt:disconnect(Client1),
+
+    maybe_kill_connection_process(ClientId, Config),
+
+    publish(Topic, Payload, ?QOS_1),
+
+    {ok, Client2} = emqtt:start_link([
+        {clientid, ClientId},
+        {proto_ver, v5},
+        {properties, #{'Session-Expiry-Interval' => 30}},
+        {clean_start, false}
+        | Config
+    ]),
+    {ok, _} = emqtt:ConnFun(Client2),
+    [Msg | _] = receive_messages(1),
+    ?assertEqual({ok, iolist_to_binary(Topic)}, maps:find(topic, Msg)),
+    ?assertEqual({ok, iolist_to_binary(Payload)}, maps:find(payload, Msg)),
+    ?assertEqual({ok, ?QOS_1}, maps:find(qos, Msg)),
+    ok = emqtt:disconnect(Client2).
+
+%% [MQTT-3.1.2-23]
+%% TODO: un-skip after QoS 2 support is implemented in DS.
+t_connect_session_expiry_interval_qos2(init, Config) -> skip_ds_tc(Config);
+t_connect_session_expiry_interval_qos2('end', _Config) -> ok.
+t_connect_session_expiry_interval_qos2(Config) ->
+    ConnFun = ?config(conn_fun, Config),
+    Topic = ?config(topic, Config),
+    STopic = ?config(stopic, Config),
+    Payload = <<"test message">>,
+    ClientId = ?config(client_id, Config),
+
     {ok, Client1} = emqtt:start_link([
         {clientid, ClientId},
         {proto_ver, v5},
@@ -423,7 +481,7 @@ t_cancel_on_disconnect(Config) ->
     {ok, _} = emqtt:ConnFun(Client1),
     ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 0}),
 
-    wait_for_cm_unregister(ClientId),
+    wait_connection_process_unregistered(ClientId),
 
     {ok, Client2} = emqtt:start_link([
         {clientid, ClientId},
@@ -455,7 +513,7 @@ t_persist_on_disconnect(Config) ->
     %% Strangely enough, the disconnect is reported as successful by emqtt.
     ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 30}),
 
-    wait_for_cm_unregister(ClientId),
+    wait_connection_process_unregistered(ClientId),
 
     {ok, Client2} = emqtt:start_link([
         {clientid, ClientId},
@@ -470,8 +528,6 @@ t_persist_on_disconnect(Config) ->
     ?assertEqual(0, client_info(session_present, Client2)),
     ok = emqtt:disconnect(Client2).
 
-t_process_dies_session_expires(init, Config) -> skip_ds_tc(Config);
-t_process_dies_session_expires('end', _Config) -> ok.
 t_process_dies_session_expires(Config) ->
     %% Emulate an error in the connect process,
     %% or that the node of the process goes down.
@@ -494,7 +550,7 @@ t_process_dies_session_expires(Config) ->
 
     maybe_kill_connection_process(ClientId, Config),
 
-    ok = publish(Topic, [Payload]),
+    ok = publish(Topic, Payload),
 
     timer:sleep(1100),
 
@@ -535,7 +591,7 @@ t_publish_while_client_is_gone_qos1(Config) ->
     ok = emqtt:disconnect(Client1),
     maybe_kill_connection_process(ClientId, Config),
 
-    ok = publish(Topic, [Payload1, Payload2], false, 1),
+    ok = publish_many(messages(Topic, [Payload1, Payload2], ?QOS_1)),
 
     {ok, Client2} = emqtt:start_link([
         {proto_ver, v5},
@@ -547,7 +603,7 @@ t_publish_while_client_is_gone_qos1(Config) ->
     {ok, _} = emqtt:ConnFun(Client2),
     Msgs = receive_messages(2),
     ?assertMatch([_, _], Msgs),
-    [Msg2, Msg1] = Msgs,
+    [Msg1, Msg2] = Msgs,
     ?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)),
     ?assertEqual({ok, 1}, maps:find(qos, Msg1)),
     ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg2)),
@@ -555,8 +611,123 @@ t_publish_while_client_is_gone_qos1(Config) ->
 
     ok = emqtt:disconnect(Client2).
 
-t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config);
+t_publish_many_while_client_is_gone_qos1(Config) ->
-t_publish_while_client_is_gone('end', _Config) -> ok.
+    %% A persistent session should receive all of the still unacked messages
+    %% for its subscriptions after the client dies or reconnects, in addition
+    %% to new messages that were published while the client was gone. The order
+    %% of the messages should be consistent across reconnects.
+    ClientId = ?config(client_id, Config),
+    ConnFun = ?config(conn_fun, Config),
+    {ok, Client1} = emqtt:start_link([
+        {proto_ver, v5},
+        {clientid, ClientId},
+        {properties, #{'Session-Expiry-Interval' => 30}},
+        {clean_start, true},
+        {auto_ack, never}
+        | Config
+    ]),
+    {ok, _} = emqtt:ConnFun(Client1),
+
+    STopics = [
+        <<"t/+/foo">>,
+        <<"msg/feed/#">>,
+        <<"loc/+/+/+">>
+    ],
+    [{ok, _, [?QOS_1]} = emqtt:subscribe(Client1, ST, ?QOS_1) || ST <- STopics],
+
+    Pubs1 = [
+        #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1},
+        #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 1},
+        #mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 1},
+        #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M5">>, qos = 1},
+        #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M6">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M7">>, qos = 1}
+    ],
+    ok = publish_many(Pubs1),
+    NPubs1 = length(Pubs1),
+
+    Msgs1 = receive_messages(NPubs1),
+    NMsgs1 = length(Msgs1),
+    ?assertEqual(NPubs1, NMsgs1),
+
+    ct:pal("Msgs1 = ~p", [Msgs1]),
+
+    %% TODO
+    %% This assertion doesn't currently hold because `emqx_ds` doesn't enforce
+    %% strict ordering reflecting client publishing order. Instead, per-topic
+    %% ordering is guaranteed per each client. In fact, this violates the MQTT
+    %% specification, but we deemed it acceptable for now.
+    %% ?assertMatch([
+    %%     #{payload := <<"M1">>},
+    %%     #{payload := <<"M2">>},
+    %%     #{payload := <<"M3">>},
+    %%     #{payload := <<"M4">>},
+    %%     #{payload := <<"M5">>},
+    %%     #{payload := <<"M6">>},
+    %%     #{payload := <<"M7">>}
+    %% ], Msgs1),
+
+    ?assertEqual(
+        get_topicwise_order(Pubs1),
+        get_topicwise_order(Msgs1)
+    ),
+
+    NAcked = 4,
+    [ok = emqtt:puback(Client1, PktId) || #{packet_id := PktId} <- lists:sublist(Msgs1, NAcked)],
+
+    %% Ensure that PUBACKs are propagated to the channel.
+    pong = emqtt:ping(Client1),
+
+    ok = disconnect_client(Client1),
+    maybe_kill_connection_process(ClientId, Config),
+
+    Pubs2 = [
+        #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M8">>, qos = 1},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M9">>, qos = 1},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M10">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M11">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M12">>, qos = 1}
+    ],
+    ok = publish_many(Pubs2),
+    NPubs2 = length(Pubs2),
+
+    {ok, Client2} = emqtt:start_link([
+        {proto_ver, v5},
+        {clientid, ClientId},
+        {properties, #{'Session-Expiry-Interval' => 30}},
+        {clean_start, false},
+        {auto_ack, false}
+        | Config
+    ]),
+    {ok, _} = emqtt:ConnFun(Client2),
+
+    %% Try to receive _at most_ `NPubs` messages.
+    %% There shouldn't be that much unacked messages in the replay anyway,
+    %% but it's an easy number to pick.
+    NPubs = NPubs1 + NPubs2,
+    Msgs2 = receive_messages(NPubs, _Timeout = 2000),
+    NMsgs2 = length(Msgs2),
+
+    ct:pal("Msgs2 = ~p", [Msgs2]),
+
+    ?assert(NMsgs2 < NPubs, Msgs2),
+    ?assert(NMsgs2 > NPubs2, Msgs2),
+    ?assert(NMsgs2 >= NPubs - NAcked, Msgs2),
+    NSame = NMsgs2 - NPubs2,
+    ?assert(
+        lists:all(fun(#{dup := Dup}) -> Dup end, lists:sublist(Msgs2, NSame))
+    ),
+    ?assertNot(
+        lists:all(fun(#{dup := Dup}) -> Dup end, lists:nthtail(NSame, Msgs2))
+    ),
+    ?assertEqual(
+        [maps:with([packet_id, topic, payload], M) || M <- lists:nthtail(NMsgs1 - NSame, Msgs1)],
+        [maps:with([packet_id, topic, payload], M) || M <- lists:sublist(Msgs2, NSame)]
+    ),
+
+    ok = disconnect_client(Client2).
+
 t_publish_while_client_is_gone(Config) ->
     %% A persistent session should receive messages in its
     %% subscription even if the process owning the session dies.
@@ -579,7 +750,7 @@ t_publish_while_client_is_gone(Config) ->
     ok = emqtt:disconnect(Client1),
     maybe_kill_connection_process(ClientId, Config),
 
-    ok = publish(Topic, [Payload1, Payload2]),
+    ok = publish_many(messages(Topic, [Payload1, Payload2])),
 
     {ok, Client2} = emqtt:start_link([
         {proto_ver, v5},
@@ -591,7 +762,7 @@ t_publish_while_client_is_gone(Config) ->
     {ok, _} = emqtt:ConnFun(Client2),
     Msgs = receive_messages(2),
     ?assertMatch([_, _], Msgs),
-    [Msg2, Msg1] = Msgs,
+    [Msg1, Msg2] = Msgs,
     ?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)),
     ?assertEqual({ok, 2}, maps:find(qos, Msg1)),
     ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg2)),
@@ -599,9 +770,157 @@ t_publish_while_client_is_gone(Config) ->
 
     ok = emqtt:disconnect(Client2).
 
-%% TODO: don't skip after QoS2 support is added to DS.
+t_publish_many_while_client_is_gone(Config) ->
-t_clean_start_drops_subscriptions(init, Config) -> skip_ds_tc(Config);
+    %% A persistent session should receive all of the still unacked messages
-t_clean_start_drops_subscriptions('end', _Config) -> ok.
+    %% for its subscriptions after the client dies or reconnects, in addition
+    %% to PUBRELs for the messages it has PUBRECed. While client must send
+    %% PUBACKs and PUBRECs in order, those orders are independent of each other.
+    ClientId = ?config(client_id, Config),
+    ConnFun = ?config(conn_fun, Config),
+    ClientOpts = [
+        {proto_ver, v5},
+        {clientid, ClientId},
+        {properties, #{'Session-Expiry-Interval' => 30}},
+        {auto_ack, never}
+        | Config
+    ],
+
+    {ok, Client1} = emqtt:start_link([{clean_start, true} | ClientOpts]),
+    {ok, _} = emqtt:ConnFun(Client1),
+    {ok, _, [?QOS_1]} = emqtt:subscribe(Client1, <<"t/+/foo">>, ?QOS_1),
+    {ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"msg/feed/#">>, ?QOS_2),
+    {ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"loc/+/+/+">>, ?QOS_2),
+
+    Pubs1 = [
+        #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1},
+        #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 2},
+        #mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 2},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M5">>, qos = 2},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M6">>, qos = 1},
+        #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M7">>, qos = 2},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M8">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M9">>, qos = 2}
+    ],
+    ok = publish_many(Pubs1),
+    NPubs1 = length(Pubs1),
+
+    Msgs1 = receive_messages(NPubs1),
+    ct:pal("Msgs1 = ~p", [Msgs1]),
+    NMsgs1 = length(Msgs1),
+    ?assertEqual(NPubs1, NMsgs1),
+
+    ?assertEqual(
+        get_topicwise_order(Pubs1),
+        get_topicwise_order(Msgs1)
+    ),
+
+    %% PUBACK every QoS 1 message.
+    lists:foreach(
+        fun(PktId) -> ok = emqtt:puback(Client1, PktId) end,
+        [PktId || #{qos := 1, packet_id := PktId} <- Msgs1]
+    ),
+
+    %% PUBREC first `NRecs` QoS 2 messages.
+    NRecs = 3,
+    PubRecs1 = lists:sublist([PktId || #{qos := 2, packet_id := PktId} <- Msgs1], NRecs),
+    lists:foreach(
+        fun(PktId) -> ok = emqtt:pubrec(Client1, PktId) end,
+        PubRecs1
+    ),
+
+    %% Ensure that PUBACKs / PUBRECs are propagated to the channel.
+    pong = emqtt:ping(Client1),
+
+    %% Receive PUBRELs for the sent PUBRECs.
+    PubRels1 = receive_messages(NRecs),
+    ct:pal("PubRels1 = ~p", [PubRels1]),
+    ?assertEqual(
+        PubRecs1,
+        [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1],
+        PubRels1
+    ),
+
+    ok = disconnect_client(Client1),
+    maybe_kill_connection_process(ClientId, Config),
+
+    Pubs2 = [
+        #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M10">>, qos = 2},
+        #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M11">>, qos = 1},
+        #mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M12">>, qos = 2}
+    ],
+    ok = publish_many(Pubs2),
+    NPubs2 = length(Pubs2),
+
+    {ok, Client2} = emqtt:start_link([{clean_start, false} | ClientOpts]),
+    {ok, _} = emqtt:ConnFun(Client2),
+
+    %% Try to receive _at most_ `NPubs` messages.
+    %% There shouldn't be that much unacked messages in the replay anyway,
+    %% but it's an easy number to pick.
+    NPubs = NPubs1 + NPubs2,
+    Msgs2 = receive_messages(NPubs, _Timeout = 2000),
+    ct:pal("Msgs2 = ~p", [Msgs2]),
+
+    %% We should again receive PUBRELs for the PUBRECs we sent earlier.
+    ?assertEqual(
+        get_msgs_essentials(PubRels1),
+        [get_msg_essentials(PubRel) || PubRel = {pubrel, _} <- Msgs2]
+    ),
+
+    %% We should receive duplicates only for QoS 2 messages where PUBRELs were
+    %% not sent, in the same order as the original messages.
+    Msgs2Dups = [get_msg_essentials(M) || M = #{dup := true} <- Msgs2],
+    ?assertEqual(
+        Msgs2Dups,
+        [M || M = #{qos := 2} <- Msgs2Dups]
+    ),
+    ?assertEqual(
+        get_msgs_essentials(pick_respective_msgs(Msgs2Dups, Msgs1)),
+        Msgs2Dups
+    ),
+
+    %% Now complete all yet incomplete QoS 2 message flows instead.
+    PubRecs2 = [PktId || #{qos := 2, packet_id := PktId} <- Msgs2],
+    lists:foreach(
+        fun(PktId) -> ok = emqtt:pubrec(Client2, PktId) end,
+        PubRecs2
+    ),
+
+    PubRels2 = receive_messages(length(PubRecs2)),
+    ct:pal("PubRels2 = ~p", [PubRels2]),
+    ?assertEqual(
+        PubRecs2,
+        [PktId || {pubrel, #{packet_id := PktId}} <- PubRels2],
+        PubRels2
+    ),
+
+    %% PUBCOMP every PUBREL.
+    PubComps = [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1 ++ PubRels2],
+    lists:foreach(
+        fun(PktId) -> ok = emqtt:pubcomp(Client2, PktId) end,
+        PubComps
+    ),
+
+    %% Ensure that PUBCOMPs are propagated to the channel.
+    pong = emqtt:ping(Client2),
+
+    ok = disconnect_client(Client2),
+    maybe_kill_connection_process(ClientId, Config),
+
+    {ok, Client3} = emqtt:start_link([{clean_start, false} | ClientOpts]),
+    {ok, _} = emqtt:ConnFun(Client3),
+
+    %% Only the last unacked QoS 1 message should be retransmitted.
+    Msgs3 = receive_messages(NPubs, _Timeout = 2000),
+    ct:pal("Msgs3 = ~p", [Msgs3]),
+    ?assertMatch(
+        [#{topic := <<"t/100/foo">>, payload := <<"M11">>, qos := 1, dup := true}],
+        Msgs3
+    ),
+
+    ok = disconnect_client(Client3).
+
 t_clean_start_drops_subscriptions(Config) ->
     %% 1. A persistent session is started and disconnected.
     %% 2. While disconnected, a message is published and persisted.
@@ -627,13 +946,13 @@ t_clean_start_drops_subscriptions(Config) ->
         | Config
     ]),
     {ok, _} = emqtt:ConnFun(Client1),
-    {ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2),
+    {ok, _, [1]} = emqtt:subscribe(Client1, STopic, qos1),
 
     ok = emqtt:disconnect(Client1),
     maybe_kill_connection_process(ClientId, Config),
 
     %% 2.
-    ok = publish(Topic, Payload1),
+    ok = publish(Topic, Payload1, ?QOS_1),
 
     %% 3.
     {ok, Client2} = emqtt:start_link([
@@ -645,12 +964,14 @@ t_clean_start_drops_subscriptions(Config) ->
     ]),
     {ok, _} = emqtt:ConnFun(Client2),
     ?assertEqual(0, client_info(session_present, Client2)),
-    {ok, _, [2]} = emqtt:subscribe(Client2, STopic, qos2),
+    {ok, _, [1]} = emqtt:subscribe(Client2, STopic, qos1),
 
-    ok = publish(Topic, Payload2),
+    timer:sleep(100),
+    ok = publish(Topic, Payload2, ?QOS_1),
     [Msg1] = receive_messages(1),
     ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg1)),
 
+    pong = emqtt:ping(Client2),
     ok = emqtt:disconnect(Client2),
     maybe_kill_connection_process(ClientId, Config),
 
@@ -664,10 +985,11 @@ t_clean_start_drops_subscriptions(Config) ->
     ]),
     {ok, _} = emqtt:ConnFun(Client3),
 
-    ok = publish(Topic, Payload3),
+    ok = publish(Topic, Payload3, ?QOS_1),
     [Msg2] = receive_messages(1),
     ?assertEqual({ok, iolist_to_binary(Payload3)}, maps:find(payload, Msg2)),
 
+    pong = emqtt:ping(Client3),
     ok = emqtt:disconnect(Client3).
 
 t_unsubscribe(Config) ->
@@ -731,8 +1053,32 @@ t_multiple_subscription_matches(Config) ->
     ?assertEqual({ok, 2}, maps:find(qos, Msg2)),
     ok = emqtt:disconnect(Client2).
 
+get_topicwise_order(Msgs) ->
+    maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs).
+
+get_msgpub_topic(#mqtt_msg{topic = Topic}) ->
+    Topic;
+get_msgpub_topic(#{topic := Topic}) ->
+    Topic.
+
+get_msgpub_payload(#mqtt_msg{payload = Payload}) ->
+    Payload;
+get_msgpub_payload(#{payload := Payload}) ->
+    Payload.
+
+get_msg_essentials(Msg = #{}) ->
+    maps:with([packet_id, topic, payload, qos], Msg);
+get_msg_essentials({pubrel, Msg}) ->
+    {pubrel, maps:with([packet_id, reason_code], Msg)}.
+
+get_msgs_essentials(Msgs) ->
+    [get_msg_essentials(M) || M <- Msgs].
+
+pick_respective_msgs(MsgRefs, Msgs) ->
+    [M || M <- Msgs, Ref <- MsgRefs, maps:get(packet_id, M) =:= maps:get(packet_id, Ref)].
+
 skip_ds_tc(Config) ->
-    case ?config(persistent_store, Config) of
+    case ?config(persistence, Config) of
         ds ->
             {skip, "Testcase not yet supported under 'emqx_persistent_session_ds' implementation"};
         _ ->

apps/emqx/test/emqx_persistent_session_ds_router_SUITE.erl

@@ -38,7 +38,7 @@ init_per_suite(Config) ->
     AppSpecs = [
         emqx_durable_storage,
         {emqx, #{
-            config => #{persistent_session_store => #{ds => true}},
+            config => #{session_persistence => #{enable => true}},
             override_env => [{boot_modules, [broker]}]
         }}
     ],

apps/emqx/test/emqx_proper_types.erl

@@ -511,13 +511,7 @@ peercert() ->
 conn_mod() ->
     oneof([
         emqx_connection,
-        emqx_ws_connection,
+        emqx_ws_connection
-        emqx_coap_mqtt_adapter,
-        emqx_sn_gateway,
-        emqx_lwm2m_protocol,
-        emqx_gbt32960_conn,
-        emqx_jt808_connection,
-        emqx_tcp_connection
     ]).
 
 proto_name() ->

apps/emqx/test/emqx_quic_multistreams_SUITE.erl

@@ -669,22 +669,21 @@ t_multi_streams_packet_malform(Config) ->
     case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of
         {ok, 10} -> ok;
         {error, cancelled} -> ok;
-        {error, stm_send_error, aborted} -> ok
+        {error, stm_send_error, aborted} -> ok;
+        {error, closed} -> ok
     end,
 
     ?assert(is_list(emqtt:info(C))),
-
+    {error, closed} =
-    {error, stm_send_error, _} =
         snabbkaffe:retry(
             10000,
             10,
             fun() ->
-                {error, stm_send_error, _} = quicer:send(
+                {error, closed} = quicer:send(
                     MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>
                 )
             end
         ),
-
     ?assert(is_list(emqtt:info(C))),
 
     ok = emqtt:disconnect(C).
@@ -770,9 +769,9 @@ t_multi_streams_packet_too_large(Config) ->
     timeout = recv_pub(1),
     ?assert(is_list(emqtt:info(C))),
 
-    %% Connection could be kept
+    %% Connection could be kept but data stream are closed!
-    {error, stm_send_error, _} = quicer:send(via_stream(PubVia), <<1>>),
+    {error, closed} = quicer:send(via_stream(PubVia), <<1>>),
-    {error, stm_send_error, _} = quicer:send(via_stream(PubVia2), <<1>>),
+    {error, closed} = quicer:send(via_stream(PubVia2), <<1>>),
     %% We could send data over new stream
     {ok, PubVia3} = emqtt:start_data_stream(C, []),
     ok = emqtt:publish_async(

apps/emqx/test/emqx_router_helper_SUITE.erl

@@ -80,7 +80,7 @@ t_mnesia(_) ->
     ct:sleep(200).
 
 t_cleanup_membership_mnesia_down(_Config) ->
-    Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
+    Slave = emqx_cth_cluster:node_name(node2),
     emqx_router:add_route(<<"a/b/c">>, Slave),
     emqx_router:add_route(<<"d/e/f">>, node()),
     ?assertMatch([_, _], emqx_router:topics()),
@@ -92,7 +92,7 @@ t_cleanup_membership_mnesia_down(_Config) ->
     ?assertEqual([<<"d/e/f">>], emqx_router:topics()).
 
 t_cleanup_membership_node_down(_Config) ->
-    Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
+    Slave = emqx_cth_cluster:node_name(node3),
     emqx_router:add_route(<<"a/b/c">>, Slave),
     emqx_router:add_route(<<"d/e/f">>, node()),
     ?assertMatch([_, _], emqx_router:topics()),
@@ -104,7 +104,7 @@ t_cleanup_membership_node_down(_Config) ->
     ?assertEqual([<<"d/e/f">>], emqx_router:topics()).
 
 t_cleanup_monitor_node_down(_Config) ->
-    Slave = emqx_cth_cluster:start_bare_node(?FUNCTION_NAME, #{driver => ct_slave}),
+    [Slave] = emqx_cth_cluster:start_bare_nodes([node4]),
     emqx_router:add_route(<<"a/b/c">>, Slave),
     emqx_router:add_route(<<"d/e/f">>, node()),
     ?assertMatch([_, _], emqx_router:topics()),

apps/emqx/test/emqx_routing_SUITE.erl

@@ -218,8 +218,9 @@ t_routing_schema_switch(VFrom, VTo, Config) ->
         ],
         #{work_dir => WorkDir}
     ),
-    % Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
     Nodes = [Node1, Node2, Node3],
+    try
+        % Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
         ?assertEqual(
             [{ok, VTo}, {ok, VTo}, {ok, VTo}],
             erpc:multicall(Nodes, emqx_router, get_schema_vsn, [])
@@ -248,8 +249,10 @@ t_routing_schema_switch(VFrom, VTo, Config) ->
         ?assertNotReceive(_),
         ok = emqtt:stop(C1),
         ok = emqtt:stop(C2),
-    ok = emqtt:stop(C3),
+        ok = emqtt:stop(C3)
-    ok = emqx_cth_cluster:stop(Nodes).
+    after
+        ok = emqx_cth_cluster:stop(Nodes)
+    end.
 
 %%
 

apps/emqx/test/emqx_secret_tests.erl

@@ -0,0 +1,76 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_secret_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+wrap_unwrap_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:unwrap(emqx_secret:wrap(42))
+    ).
+
+unwrap_immediate_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:unwrap(42)
+    ).
+
+wrap_unwrap_load_test_() ->
+    Secret = <<"foobaz">>,
+    {
+        setup,
+        fun() -> write_temp_file(Secret) end,
+        fun(Filename) -> file:delete(Filename) end,
+        fun(Filename) ->
+            ?_assertEqual(
+                Secret,
+                emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename}))
+            )
+        end
+    }.
+
+wrap_load_term_test() ->
+    ?assertEqual(
+        {file, "no/such/file/i/swear"},
+        emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
+    ).
+
+wrap_unwrap_missing_file_test() ->
+    ?assertThrow(
+        #{msg := failed_to_read_secret_file, reason := "No such file or directory"},
+        emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
+    ).
+
+wrap_term_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:term(emqx_secret:wrap(42))
+    ).
+
+external_fun_term_error_test() ->
+    Term = {foo, bar},
+    ?assertError(
+        badarg,
+        emqx_secret:term(fun() -> Term end)
+    ).
+
+write_temp_file(Bytes) ->
+    Ts = erlang:system_time(millisecond),
+    Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)),
+    ok = file:write_file(Filename, Bytes),
+    Filename.

apps/emqx/test/emqx_shared_sub_SUITE.erl

@@ -63,6 +63,7 @@ init_per_suite(Config) ->
         end,
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
+    emqx_logger:set_log_level(debug),
     [{dist_pid, DistPid} | Config].
 
 end_per_suite(Config) ->
@@ -137,7 +138,8 @@ t_random_basic(Config) when is_list(Config) ->
     ClientId = <<"ClientId">>,
     Topic = <<"foo">>,
     Payload = <<"hello">>,
-    emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
+    Group = <<"group1">>,
+    emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
     MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload),
     %% wait for the subscription to show up
     ct:sleep(200),
@@ -402,7 +404,7 @@ t_hash(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
-t_hash_clinetid(Config) when is_list(Config) ->
+t_hash_clientid(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
@@ -528,14 +530,15 @@ last_message(ExpectedPayload, Pids, Timeout) ->
 t_dispatch(Config) when is_list(Config) ->
     ok = ensure_config(random),
     Topic = <<"foo">>,
+    Group = <<"group1">>,
     ?assertEqual(
         {error, no_subscribers},
-        emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
+        emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
     ),
-    emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
+    emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
     ?assertEqual(
         {ok, 1},
-        emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
+        emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
     ).
 
 t_uncovered_func(Config) when is_list(Config) ->
@@ -572,7 +575,7 @@ t_local(Config) when is_list(Config) ->
         <<"sticky_group">> => sticky
     },
 
-    Node = start_slave('local_shared_sub_testtesttest', 21999),
+    Node = start_slave('local_shared_sub_local_1', 21999),
     ok = ensure_group_config(GroupConfig),
     ok = ensure_group_config(Node, GroupConfig),
 
@@ -625,7 +628,7 @@ t_remote(Config) when is_list(Config) ->
         <<"sticky_group">> => sticky
     },
 
-    Node = start_slave('remote_shared_sub_testtesttest', 21999),
+    Node = start_slave('remote_shared_sub_remote_1', 21999),
     ok = ensure_group_config(GroupConfig),
     ok = ensure_group_config(Node, GroupConfig),
 
@@ -674,7 +677,7 @@ t_local_fallback(Config) when is_list(Config) ->
     Topic = <<"local_foo/bar">>,
     ClientId1 = <<"ClientId1">>,
     ClientId2 = <<"ClientId2">>,
-    Node = start_slave('local_fallback_shared_sub_test', 11888),
+    Node = start_slave('local_fallback_shared_sub_1', 11888),
 
     {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]),
     {ok, _} = emqtt:connect(ConnPid1),
@@ -991,37 +994,110 @@ t_session_kicked(Config) when is_list(Config) ->
     ?assertEqual([], collect_msgs(0)),
     ok.
 
-%% FIXME: currently doesn't work
+-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS),
-%% t_different_groups_same_topic({init, Config}) ->
+    ?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS}))
-%%     TestName = atom_to_binary(?FUNCTION_NAME),
+).
-%%     ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
-%%     {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
-%%     {ok, _} = emqtt:connect(C),
-%%     [{client, C}, {clientid, ClientId} | Config];
-%% t_different_groups_same_topic({'end', Config}) ->
-%%     C = ?config(client, Config),
-%%     emqtt:stop(C),
-%%     ok;
-%% t_different_groups_same_topic(Config) when is_list(Config) ->
-%%     C = ?config(client, Config),
-%%     ClientId = ?config(clientid, Config),
-%%     %% Subscribe and unsubscribe to both $queue and $shared topics
-%%     Topic = <<"t/1">>,
-%%     SharedTopic0 = <<"$share/aa/", Topic/binary>>,
-%%     SharedTopic1 = <<"$share/bb/", Topic/binary>>,
-%%     {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}),
-%%     {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}),
 
-%%     Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
+t_different_groups_same_topic({init, Config}) ->
-%%     emqx:publish(Message0),
+    TestName = atom_to_binary(?FUNCTION_NAME),
-%%     ?assertMatch([ {publish, #{payload := <<"hi">>}}
+    ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
-%%                  , {publish, #{payload := <<"hi">>}}
+    {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
-%%                  ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}),
+    {ok, _} = emqtt:connect(C),
+    [{client, C}, {clientid, ClientId} | Config];
+t_different_groups_same_topic({'end', Config}) ->
+    C = ?config(client, Config),
+    emqtt:stop(C),
+    ok;
+t_different_groups_same_topic(Config) when is_list(Config) ->
+    C = ?config(client, Config),
+    ClientId = ?config(clientid, Config),
+    %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
+    GroupA = <<"aa">>,
+    GroupB = <<"bb">>,
+    Topic = <<"t/1">>,
 
-%%     {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0),
+    SharedTopicGroupA = ?SHARE(GroupA, Topic),
-%%     {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1),
+    ?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2),
+    SharedTopicGroupB = ?SHARE(GroupB, Topic),
+    ?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2),
 
-%%     ok.
+    ?retry(
+        _Sleep0 = 100,
+        _Attempts0 = 50,
+        begin
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
+        end
+    ),
+
+    Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>),
+    emqx:publish(Message0),
+    ?assertMatch(
+        [
+            {publish, #{payload := <<"hi">>}},
+            {publish, #{payload := <<"hi">>}}
+        ],
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
+    ),
+
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
+
+    ok.
+
+t_different_groups_update_subopts({init, Config}) ->
+    TestName = atom_to_binary(?FUNCTION_NAME),
+    ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
+    {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C),
+    [{client, C}, {clientid, ClientId} | Config];
+t_different_groups_update_subopts({'end', Config}) ->
+    C = ?config(client, Config),
+    emqtt:stop(C),
+    ok;
+t_different_groups_update_subopts(Config) when is_list(Config) ->
+    C = ?config(client, Config),
+    ClientId = ?config(clientid, Config),
+    %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
+    Topic = <<"t/1">>,
+    GroupA = <<"aa">>,
+    GroupB = <<"bb">>,
+    SharedTopicGroupA = ?SHARE(GroupA, Topic),
+    SharedTopicGroupB = ?SHARE(GroupB, Topic),
+
+    Fun = fun(Group, QoS) ->
+        ?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS),
+        ?assertMatch(
+            #{qos := QoS},
+            emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic))
+        )
+    end,
+
+    [Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]],
+
+    ?retry(
+        _Sleep0 = 100,
+        _Attempts0 = 50,
+        begin
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
+        end
+    ),
+
+    Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
+    emqx:publish(Message0),
+    ?assertMatch(
+        [
+            {publish, #{payload := <<"hi">>}},
+            {publish, #{payload := <<"hi">>}}
+        ],
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
+    ),
+
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
+
+    ok.
 
 t_queue_subscription({init, Config}) ->
     TestName = atom_to_binary(?FUNCTION_NAME),
@@ -1038,23 +1114,19 @@ t_queue_subscription({'end', Config}) ->
 t_queue_subscription(Config) when is_list(Config) ->
     C = ?config(client, Config),
     ClientId = ?config(clientid, Config),
-    %% Subscribe and unsubscribe to both $queue and $shared topics
+    %% Subscribe and unsubscribe to both $queue share and $share/<group> with same topic
     Topic = <<"t/1">>,
     QueueTopic = <<"$queue/", Topic/binary>>,
     SharedTopic = <<"$share/aa/", Topic/binary>>,
-    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}),
-    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}),
 
-    %% FIXME: we should actually see 2 routes, one for each group
+    ?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2),
-    %% ($queue and aa), but currently the latest subscription
+    ?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2),
-    %% overwrites the existing one.
+
     ?retry(
         _Sleep0 = 100,
         _Attempts0 = 50,
         begin
-            ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
-            %% FIXME: should ensure we have 2 subscriptions
-            [_] = emqx_router:lookup_routes(Topic)
         end
     ),
 
@@ -1063,37 +1135,29 @@ t_queue_subscription(Config) when is_list(Config) ->
     emqx:publish(Message0),
     ?assertMatch(
         [
+            {publish, #{payload := <<"hi">>}},
             {publish, #{payload := <<"hi">>}}
-            %% FIXME: should receive one message from each group
-            %% , {publish, #{payload := <<"hi">>}}
         ],
-        collect_msgs(5_000)
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
     ),
 
     {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic),
-    %% FIXME: return code should be success instead of 17 ("no_subscription_existed")
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic),
-    {ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic),
 
-    %% FIXME: this should eventually be true, but currently we leak
+    ?retry(
-    %% the previous group subscription...
+        _Sleep0 = 100,
-    %% ?retry(
+        _Attempts0 = 50,
-    %%     _Sleep0 = 100,
+        begin
-    %%     _Attempts0 = 50,
+            ?assertEqual(0, length(emqx_router:match_routes(Topic)))
-    %%    begin
+        end
-    %%     ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
+    ),
-    %%     [] = emqx_router:lookup_routes(Topic)
-    %%    end
-    %%   ),
     ct:sleep(500),
 
     Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>),
     emqx:publish(Message1),
-    %% FIXME: we should *not* receive any messages...
+    %% we should *not* receive any messages.
-    %% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
+    ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
-    %% This is from the leaked group...
-    ?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{
-        routes => ets:tab2list(emqx_route)
-    }),
 
     ok.
 
@@ -1190,34 +1254,24 @@ recv_msgs(Count, Msgs) ->
     end.
 
 start_slave(Name, Port) ->
-    {ok, Node} = ct_slave:start(
+    {ok, Node} = emqx_cth_peer:start_link(
-        list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
+        Name,
-        [
+        ebin_path()
-            {kill_if_fail, true},
-            {monitor_master, true},
-            {init_timeout, 10000},
-            {startup_timeout, 10000},
-            {erl_flags, ebin_path()}
-        ]
     ),
-
     pong = net_adm:ping(Node),
     setup_node(Node, Port),
     Node.
 
 stop_slave(Node) ->
     rpc:call(Node, mria, leave, []),
-    ct_slave:stop(Node).
+    emqx_cth_peer:stop(Node).
 
 host() ->
     [_, Host] = string:tokens(atom_to_list(node()), "@"),
     Host.
 
 ebin_path() ->
-    string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
+    ["-pa" | code:get_path()].
-
-is_lib(Path) ->
-    string:prefix(Path, code:lib_dir()) =:= nomatch.
 
 setup_node(Node, Port) ->
     EnvHandler =

apps/emqx/test/emqx_topic_SUITE.erl

@@ -238,11 +238,11 @@ long_topic() ->
 t_parse(_) ->
     ?assertError(
         {invalid_topic_filter, <<"$queue/t">>},
-        parse(<<"$queue/t">>, #{share => <<"g">>})
+        parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{})
     ),
     ?assertError(
         {invalid_topic_filter, <<"$share/g/t">>},
-        parse(<<"$share/g/t">>, #{share => <<"g">>})
+        parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{})
     ),
     ?assertError(
         {invalid_topic_filter, <<"$share/t">>},
@@ -254,8 +254,12 @@ t_parse(_) ->
     ),
     ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)),
     ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})),
-    ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)),
+    ?assertEqual(
-    ?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)),
+        {#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>)
+    ),
+    ?assertEqual(
+        {#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>)
+    ),
     %% The '$local' and '$fastlane' topics have been deprecated.
     ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)),
     ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)),

apps/emqx/test/emqx_topic_index_SUITE.erl

@@ -209,9 +209,6 @@ t_match_fast_forward(Config) ->
     M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab),
     M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab),
     M:insert(<<"a/b/c/+">>, id3, <<>>, Tab),
-    % dbg:tracer(),
-    % dbg:p(all, c),
-    % dbg:tpl({ets, next, '_'}, x),
     ?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))),
     ?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]).
 

apps/emqx_audit/BSL.txt

@@ -0,0 +1,94 @@
+Business Source License 1.1
+
+Licensor:             Hangzhou EMQ Technologies Co., Ltd.
+Licensed Work:        EMQX Enterprise Edition
+                      The Licensed Work is (c) 2023
+                      Hangzhou EMQ Technologies Co., Ltd.
+Additional Use Grant: Students and educators are granted right to copy,
+                      modify, and create derivative work for research
+                      or education.
+Change Date:          2027-02-01
+Change License:       Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Software,
+please contact Licensor: https://www.emqx.com/en/contact
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

apps/emqx_audit/README.md

@@ -0,0 +1,5 @@
+emqx_audit
+=====
+
+Audit log for EMQX, empowers users to efficiently access the desired audit trail data
+and facilitates auditing, compliance, troubleshooting, and security analysis.

apps/emqx_audit/include/emqx_audit.hrl

@@ -0,0 +1,26 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-define(AUDIT, emqx_audit).
+
+-record(?AUDIT, {
+    %% basic info
+    created_at,
+    node,
+    from,
+    source,
+    source_ip,
+    %% operation info
+    operation_id,
+    operation_type,
+    args,
+    operation_result,
+    failure,
+    %% request detail
+    http_method,
+    http_request,
+    http_status_code,
+    duration_ms,
+    extra
+}).

apps/emqx_audit/rebar.config

@@ -0,0 +1,5 @@
+{erl_opts, [debug_info]}.
+{deps, [
+  {emqx, {path, "../emqx"}},
+  {emqx_utils, {path, "../emqx_utils"}}
+]}.

apps/emqx_audit/src/emqx_audit.app.src

@@ -0,0 +1,10 @@
+{application, emqx_audit, [
+    {description, "Audit log for EMQX"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_audit_app, []}},
+    {applications, [kernel, stdlib, emqx]},
+    {env, []},
+    {modules, []},
+    {links, []}
+]}.

apps/emqx_audit/src/emqx_audit.erl

@@ -0,0 +1,245 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit).
+
+-behaviour(gen_server).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include("emqx_audit.hrl").
+
+%% API
+-export([start_link/0]).
+-export([log/3]).
+
+-export([trans_clean_expired/2]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
+
+-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]).
+
+-ifdef(TEST).
+-define(INTERVAL, 100).
+-else.
+-define(INTERVAL, 10000).
+-endif.
+
+to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) ->
+    #?AUDIT{
+        operation_id = <<"">>,
+        operation_type = atom_to_binary(Cmd),
+        args = Args,
+        operation_result = <<"">>,
+        failure = <<"">>,
+        duration_ms = DurationMs,
+        from = cli,
+        source = <<"">>,
+        source_ip = <<"">>,
+        http_status_code = <<"">>,
+        http_method = <<"">>,
+        http_request = <<"">>
+    };
+to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api ->
+    #{
+        source := Source,
+        source_ip := SourceIp,
+        %% operation info
+        operation_id := OperationId,
+        operation_type := OperationType,
+        operation_result := OperationResult,
+        %% request detail
+        http_status_code := StatusCode,
+        http_method := Method,
+        http_request := Request,
+        duration_ms := DurationMs
+    } = Log,
+    #?AUDIT{
+        from = From,
+        source = Source,
+        source_ip = SourceIp,
+        %% operation info
+        operation_id = OperationId,
+        operation_type = OperationType,
+        operation_result = OperationResult,
+        failure = maps:get(failure, Log, <<"">>),
+        %% request detail
+        http_status_code = StatusCode,
+        http_method = Method,
+        http_request = Request,
+        duration_ms = DurationMs,
+        args = <<"">>
+    };
+to_audit(#{from := erlang_console, function := F, args := Args}) ->
+    #?AUDIT{
+        from = erlang_console,
+        source = <<"">>,
+        source_ip = <<"">>,
+        %% operation info
+        operation_id = <<"">>,
+        operation_type = <<"">>,
+        operation_result = <<"">>,
+        failure = <<"">>,
+        %% request detail
+        http_status_code = <<"">>,
+        http_method = <<"">>,
+        http_request = <<"">>,
+        duration_ms = 0,
+        args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args]))
+    }.
+
+log(_Level, undefined, _Handler) ->
+    ok;
+log(Level, Meta1, Handler) ->
+    Meta2 = Meta1#{time => logger:timestamp(), level => Level},
+    log_to_file(Level, Meta2, Handler),
+    log_to_db(Meta2),
+    remove_handler_when_disabled().
+
+remove_handler_when_disabled() ->
+    case emqx_config:get([log, audit, enable], false) of
+        true ->
+            ok;
+        false ->
+            _ = logger:remove_handler(?AUDIT_HANDLER),
+            ok
+    end.
+
+log_to_db(Log) ->
+    Audit0 = to_audit(Log),
+    Audit = Audit0#?AUDIT{
+        node = node(),
+        created_at = erlang:system_time(microsecond)
+    },
+    mria:dirty_write(?AUDIT, Audit).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init([]) ->
+    ok = mria:create_table(?AUDIT, [
+        {type, ordered_set},
+        {rlog_shard, ?COMMON_SHARD},
+        {storage, disc_copies},
+        {record_name, ?AUDIT},
+        {attributes, record_info(fields, ?AUDIT)}
+    ]),
+    {ok, #{}, {continue, setup}}.
+
+handle_continue(setup, State) ->
+    ok = mria:wait_for_tables([?AUDIT]),
+    NewState = State#{role => mria_rlog:role()},
+    ?AUDIT(alert, #{
+        cmd => emqx,
+        args => ["start"],
+        version => emqx_release:version(),
+        from => cli,
+        duration_ms => 0
+    }),
+    {noreply, NewState, interval(NewState)}.
+
+handle_call(_Request, _From, State) ->
+    {reply, ignore, State, interval(State)}.
+
+handle_cast(_Request, State) ->
+    {noreply, State, interval(State)}.
+
+handle_info(timeout, State) ->
+    ExtraWait = clean_expired_logs(),
+    {noreply, State, interval(State) + ExtraWait};
+handle_info(_Info, State) ->
+    {noreply, State, interval(State)}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+
+%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds.
+clean_expired_logs() ->
+    MaxSize = max_size(),
+    Oldest = mnesia:dirty_first(?AUDIT),
+    CurSize = mnesia:table_info(?AUDIT, size),
+    case CurSize - MaxSize of
+        DelSize when DelSize > 0 ->
+            case
+                mria:transaction(
+                    ?COMMON_SHARD,
+                    fun ?MODULE:trans_clean_expired/2,
+                    [Oldest, DelSize]
+                )
+            of
+                {atomic, ok} ->
+                    0;
+                {aborted, Reason} ->
+                    ?SLOG(error, #{
+                        msg => "clean_expired_audit_aborted",
+                        reason => Reason,
+                        delete_size => DelSize,
+                        current_size => CurSize,
+                        max_count => MaxSize
+                    }),
+                    60000
+            end;
+        _ ->
+            0
+    end.
+
+trans_clean_expired(Oldest, DelCount) ->
+    First = mnesia:first(?AUDIT),
+    %% Other node already clean from the oldest record.
+    %% ensure not delete twice, otherwise records that should not be deleted will be deleted.
+    case First =:= Oldest of
+        true -> do_clean_expired(First, DelCount);
+        false -> ok
+    end.
+
+do_clean_expired(_, DelSize) when DelSize =< 0 -> ok;
+do_clean_expired('$end_of_table', _DelSize) ->
+    ok;
+do_clean_expired(CurKey, DeleteSize) ->
+    mnesia:delete(?AUDIT, CurKey, sticky_write),
+    do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1).
+
+max_size() ->
+    emqx_conf:get([log, audit, max_filter_size], 5000).
+
+interval(#{role := replicant}) -> hibernate;
+interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL).
+
+log_to_file(Level, Meta, #{module := Module} = Handler) ->
+    Log = #{level => Level, meta => Meta, msg => undefined},
+    Handler1 = maps:without(?OWN_KEYS, Handler),
+    try
+        erlang:apply(Module, log, [Log, Handler1])
+    catch
+        C:R:S ->
+            case logger:remove_handler(?AUDIT_HANDLER) of
+                ok ->
+                    logger:internal_log(
+                        error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S}
+                    );
+                {error, {not_found, _}} ->
+                    ok;
+                {error, Reason} ->
+                    logger:internal_log(
+                        error,
+                        {removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S}
+                    )
+            end
+    end.

apps/emqx_audit/src/emqx_audit_api.erl

@@ -0,0 +1,398 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_api).
+
+-behaviour(minirest_api).
+
+%% API
+-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]).
+-export([audit/2]).
+-export([qs2ms/2, format/1]).
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include("emqx_audit.hrl").
+
+-import(hoconsc, [mk/2, ref/2, array/1]).
+
+-define(TAGS, ["Audit"]).
+
+-define(AUDIT_QS_SCHEMA, [
+    {<<"node">>, atom},
+    {<<"from">>, atom},
+    {<<"source">>, binary},
+    {<<"source_ip">>, binary},
+    {<<"operation_id">>, binary},
+    {<<"operation_type">>, binary},
+    {<<"operation_result">>, atom},
+    {<<"http_status_code">>, integer},
+    {<<"http_method">>, atom},
+    {<<"gte_created_at">>, timestamp},
+    {<<"lte_created_at">>, timestamp},
+    {<<"gte_duration_ms">>, timestamp},
+    {<<"lte_duration_ms">>, timestamp}
+]).
+-define(DISABLE_MSG, <<"Audit is disabled">>).
+
+namespace() -> "audit".
+
+api_spec() ->
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
+
+paths() ->
+    ["/audit"].
+
+schema("/audit") ->
+    #{
+        'operationId' => audit,
+        get => #{
+            tags => ?TAGS,
+            description => ?DESC(audit_get),
+            parameters => [
+                {node,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"emqx@127.0.0.1">>,
+                        desc => ?DESC(filter_node)
+                    })},
+                {from,
+                    ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
+                        in => query,
+                        required => false,
+                        example => <<"dashboard">>,
+                        desc => ?DESC(filter_from)
+                    })},
+                {source,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"admin">>,
+                        desc => ?DESC(filter_source)
+                    })},
+                {source_ip,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"127.0.0.1">>,
+                        desc => ?DESC(filter_source_ip)
+                    })},
+                {operation_id,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"/rules/{id}">>,
+                        desc => ?DESC(filter_operation_id)
+                    })},
+                {operation_type,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        example => <<"rules">>,
+                        required => false,
+                        desc => ?DESC(filter_operation_type)
+                    })},
+                {operation_result,
+                    ?HOCON(?ENUM([success, failure]), #{
+                        in => query,
+                        example => failure,
+                        required => false,
+                        desc => ?DESC(filter_operation_result)
+                    })},
+                {http_status_code,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 200,
+                        required => false,
+                        desc => ?DESC(filter_http_status_code)
+                    })},
+                {http_method,
+                    ?HOCON(?ENUM([post, put, delete]), #{
+                        in => query,
+                        example => post,
+                        required => false,
+                        desc => ?DESC(filter_http_method)
+                    })},
+                {gte_duration_ms,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 0,
+                        required => false,
+                        desc => ?DESC(filter_gte_duration_ms)
+                    })},
+                {lte_duration_ms,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 1000,
+                        required => false,
+                        desc => ?DESC(filter_lte_duration_ms)
+                    })},
+                {gte_created_at,
+                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                        in => query,
+                        required => false,
+                        example => <<"2023-10-15T00:00:00.820384+08:00">>,
+                        desc => ?DESC(filter_gte_created_at)
+                    })},
+                {lte_created_at,
+                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                        in => query,
+                        example => <<"2023-10-16T00:00:00.820384+08:00">>,
+                        required => false,
+                        desc => ?DESC(filter_lte_created_at)
+                    })},
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit)
+            ],
+            summary => <<"List audit logs">>,
+            responses => #{
+                200 =>
+                    emqx_dashboard_swagger:schema_with_example(
+                        array(?REF(audit_list)),
+                        audit_log_list_example()
+                    ),
+                400 => emqx_dashboard_swagger:error_codes(
+                    ['BAD_REQUEST'],
+                    ?DISABLE_MSG
+                )
+            }
+        }
+    }.
+
+fields(audit_list) ->
+    [
+        {data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})},
+        {meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
+    ];
+fields(audit) ->
+    [
+        {created_at,
+            ?HOCON(
+                emqx_utils_calendar:epoch_millisecond(),
+                #{
+                    desc => "The time when the log is created"
+                }
+            )},
+        {node,
+            ?HOCON(binary(), #{
+                desc => "The node name to which the log is created"
+            })},
+        {from,
+            ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
+                desc => "The source type of the log"
+            })},
+        {source,
+            ?HOCON(binary(), #{
+                desc => "The source of the log"
+            })},
+        {source_ip,
+            ?HOCON(binary(), #{
+                desc => "The source ip of the log"
+            })},
+        {operation_id,
+            ?HOCON(binary(), #{
+                desc => "The operation id of the log"
+            })},
+        {operation_type,
+            ?HOCON(binary(), #{
+                desc => "The operation type of the log"
+            })},
+        {operation_result,
+            ?HOCON(?ENUM([success, failure]), #{
+                desc => "The operation result of the log"
+            })},
+        {http_status_code,
+            ?HOCON(integer(), #{
+                desc => "The http status code of the log"
+            })},
+        {http_method,
+            ?HOCON(?ENUM([post, put, delete]), #{
+                desc => "The http method of the log"
+            })},
+        {duration_ms,
+            ?HOCON(integer(), #{
+                desc => "The duration of the log"
+            })},
+        {args,
+            ?HOCON(?ARRAY(binary()), #{
+                desc => "The args of the log"
+            })},
+        {failure,
+            ?HOCON(?ARRAY(binary()), #{
+                desc => "The failure of the log"
+            })},
+        {http_request,
+            ?HOCON(?REF(http_request), #{
+                desc => "The http request of the log"
+            })}
+    ];
+fields(http_request) ->
+    [
+        {bindings, ?HOCON(map(), #{})},
+        {body, ?HOCON(map(), #{})},
+        {headers, ?HOCON(map(), #{})},
+        {method, ?HOCON(?ENUM([post, put, delete]), #{})}
+    ].
+
+audit(get, #{query_string := QueryString}) ->
+    case emqx_config:get([log, audit, enable], false) of
+        false ->
+            {400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}};
+        true ->
+            case
+                emqx_mgmt_api:node_query(
+                    node(),
+                    ?AUDIT,
+                    QueryString,
+                    ?AUDIT_QS_SCHEMA,
+                    fun ?MODULE:qs2ms/2,
+                    fun ?MODULE:format/1
+                )
+            of
+                {error, page_limit_invalid} ->
+                    {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
+                {error, Node, Error} ->
+                    Message = list_to_binary(
+                        io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])
+                    ),
+                    {500, #{code => <<"NODE_DOWN">>, message => Message}};
+                Result ->
+                    {200, Result}
+            end
+    end.
+
+qs2ms(_Tab, {Qs, _}) ->
+    #{
+        match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []),
+        fuzzy_fun => undefined
+    }.
+
+gen_match_spec([], Audit, Conn) ->
+    [{Audit, Conn, ['$_']}];
+gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn);
+gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn);
+gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn);
+gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn);
+gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn);
+gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn);
+gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn);
+gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn);
+gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn);
+gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]);
+gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [
+        {'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn
+    ]);
+gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]);
+gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [
+        {'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn
+    ]).
+
+format(Audit) ->
+    #?AUDIT{
+        created_at = CreatedAt,
+        node = Node,
+        from = From,
+        source = Source,
+        source_ip = SourceIp,
+        operation_id = OperationId,
+        operation_type = OperationType,
+        operation_result = OperationResult,
+        http_status_code = HttpStatusCode,
+        http_method = HttpMethod,
+        duration_ms = DurationMs,
+        args = Args,
+        failure = Failure,
+        http_request = HttpRequest
+    } = Audit,
+    #{
+        created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond),
+        node => Node,
+        from => From,
+        source => Source,
+        source_ip => SourceIp,
+        operation_id => OperationId,
+        operation_type => OperationType,
+        operation_result => OperationResult,
+        http_status_code => HttpStatusCode,
+        http_method => HttpMethod,
+        duration_ms => DurationMs,
+        args => Args,
+        failure => Failure,
+        http_request => HttpRequest
+    }.
+
+audit_log_list_example() ->
+    #{
+        data => [api_example(), cli_example()],
+        meta => #{
+            <<"count">> => 2,
+            <<"hasnext">> => false,
+            <<"limit">> => 50,
+            <<"page">> => 1
+        }
+    }.
+
+api_example() ->
+    #{
+        <<"args">> => "",
+        <<"created_at">> => "2023-10-17T10:41:20.383993+08:00",
+        <<"duration_ms">> => 0,
+        <<"failure">> => "",
+        <<"from">> => "dashboard",
+        <<"http_method">> => "post",
+        <<"http_request">> => #{
+            <<"bindings">> => #{},
+            <<"body">> => #{
+                <<"password">> => "******",
+                <<"username">> => "admin"
+            },
+            <<"headers">> => #{
+                <<"accept">> => "*/*",
+                <<"authorization">> => "******",
+                <<"connection">> => "keep-alive",
+                <<"content-length">> => "45",
+                <<"content-type">> => "application/json"
+            },
+            <<"method">> => "post"
+        },
+        <<"http_status_code">> => 200,
+        <<"node">> => "emqx@127.0.0.1",
+        <<"operation_id">> => "/login",
+        <<"operation_result">> => "success",
+        <<"operation_type">> => "login",
+        <<"source">> => "admin",
+        <<"source_ip">> => "127.0.0.1"
+    }.
+
+cli_example() ->
+    #{
+        <<"args">> => [<<"show">>, <<"log">>],
+        <<"created_at">> => "2023-10-17T10:45:13.100426+08:00",
+        <<"duration_ms">> => 7,
+        <<"failure">> => "",
+        <<"from">> => "cli",
+        <<"http_method">> => "",
+        <<"http_request">> => "",
+        <<"http_status_code">> => "",
+        <<"node">> => "emqx@127.0.0.1",
+        <<"operation_id">> => "",
+        <<"operation_result">> => "",
+        <<"operation_type">> => "conf",
+        <<"source">> => "",
+        <<"source_ip">> => ""
+    }.

apps/emqx_audit/src/emqx_audit_app.erl

@@ -0,0 +1,15 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_audit_sup:start_link().
+
+stop(_State) ->
+    ok.

apps/emqx_audit/src/emqx_audit_sup.erl

@@ -0,0 +1,33 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_all,
+        intensity => 10,
+        period => 10
+    },
+    ChildSpecs = [
+        #{
+            id => emqx_audit,
+            start => {emqx_audit, start_link, []},
+            type => worker,
+            restart => transient,
+            shutdown => 1000
+        }
+    ],
+    {ok, {SupFlags, ChildSpecs}}.

apps/emqx_audit/test/emqx_audit_api_SUITE.erl

@@ -0,0 +1,248 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_audit_api_SUITE).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() ->
+    [
+        {group, audit, [sequence]}
+    ].
+
+groups() ->
+    [
+        {audit, [sequence], common_tests()}
+    ].
+
+common_tests() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+-define(CONF_DEFAULT, #{
+    node =>
+        #{
+            name => "emqx1@127.0.0.1",
+            cookie => "emqxsecretcookie",
+            data_dir => "data"
+        },
+    log => #{
+        audit =>
+            #{
+                enable => true,
+                ignore_high_frequency_request => true,
+                level => info,
+                max_filter_size => 15,
+                rotation_count => 2,
+                rotation_size => "10MB",
+                time_offset => "system"
+            }
+    }
+}).
+
+init_per_suite(Config) ->
+    _ = application:load(emqx_conf),
+    emqx_config:erase_all(),
+    emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
+    ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
+    emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
+    ok = emqx_config_logger:refresh_config(),
+    application:set_env(emqx, boot_modules, []),
+    emqx_conf_cli:load(),
+    Config.
+
+end_per_suite(_) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
+
+t_http_api(_) ->
+    process_flag(trap_exit, true),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
+    NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
+    {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
+    ?assertMatch(#{<<"max_qos_allowed">> := 1}, Res),
+    {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    ?assertMatch(
+        #{
+            <<"data">> := [
+                #{
+                    <<"from">> := <<"rest_api">>,
+                    <<"operation_id">> := <<"/configs/global_zone">>,
+                    <<"source_ip">> := <<"127.0.0.1">>,
+                    <<"source">> := _,
+                    <<"http_request">> := #{
+                        <<"method">> := <<"put">>,
+                        <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}},
+                        <<"bindings">> := _,
+                        <<"headers">> := #{<<"authorization">> := <<"******">>}
+                    },
+                    <<"http_status_code">> := 200,
+                    <<"operation_result">> := <<"success">>,
+                    <<"operation_type">> := <<"configs">>
+                }
+            ]
+        },
+        emqx_utils_json:decode(Res1, [return_maps])
+    ),
+    ok.
+
+t_disabled(_) ->
+    Enable = [log, audit, enable],
+    ?assertEqual(true, emqx:get_config(Enable)),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    Size1 = mnesia:table_info(emqx_audit, size),
+
+    {ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100),
+    NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false),
+    {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs),
+    {ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    ?assertEqual(NewLogs, GetLog1),
+    ?assertMatch(
+        {error, _},
+        emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader)
+    ),
+
+    Size2 = mnesia:table_info(emqx_audit, size),
+    %% Record the audit disable action, so the size + 1
+    ?assertEqual(Size1 + 1, Size2),
+
+    {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
+    NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111),
+    {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
+    ?assertMatch(#{<<"max_topic_levels">> := 111}, Res),
+    Size3 = mnesia:table_info(emqx_audit, size),
+    %% Don't record mqtt update request.
+    ?assertEqual(Size2, Size3),
+    %% enabled again
+    {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1),
+    {ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    ?assertEqual(Logs1, GetLog2),
+    Size4 = mnesia:table_info(emqx_audit, size),
+    ?assertEqual(Size3 + 1, Size4),
+    ok.
+
+t_cli(_Config) ->
+    ok = emqx_ctl:run_command(["conf", "show", "log"]),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
+    ?assertMatch(
+        [
+            #{
+                <<"from">> := <<"cli">>,
+                <<"operation_id">> := <<"">>,
+                <<"source_ip">> := <<"">>,
+                <<"operation_type">> := <<"conf">>,
+                <<"args">> := [<<"show">>, <<"log">>],
+                <<"node">> := _,
+                <<"source">> := <<"">>,
+                <<"http_request">> := <<"">>
+            }
+        ],
+        Data
+    ),
+
+    %% check filter
+    {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
+    #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
+    ?assertEqual(Data, Data1),
+    {ok, Res2} = emqx_mgmt_api_test_util:request_api(
+        get, AuditPath, "from=erlang_console", AuthHeader
+    ),
+    ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])),
+    ok.
+
+t_max_size(_Config) ->
+    {ok, _} = emqx:update_config([log, audit, max_filter_size], 1000),
+    SizeFun =
+        fun() ->
+            AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+            AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+            Limit = "limit=1000",
+            {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader),
+            #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
+            erlang:length(Data)
+        end,
+    InitSize = SizeFun(),
+    lists:foreach(
+        fun(_) ->
+            ok = emqx_ctl:run_command(["conf", "show", "log"])
+        end,
+        lists:duplicate(100, 1)
+    ),
+    timer:sleep(110),
+    Size1 = SizeFun(),
+    ?assert(Size1 - InitSize >= 100, {Size1, InitSize}),
+    {ok, _} = emqx:update_config([log, audit, max_filter_size], 10),
+    %% wait for clean_expired
+    timer:sleep(250),
+    ExpectSize = emqx:get_config([log, audit, max_filter_size]),
+    Size2 = SizeFun(),
+    ?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}),
+    ok.
+
+t_kickout_clients_without_log(_) ->
+    process_flag(trap_exit, true),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    {ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
+    kickout_clients(),
+    {ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
+    ?assertEqual(AuditLogs1, AuditLogs2),
+    ok.
+
+kickout_clients() ->
+    ClientId1 = <<"client1">>,
+    ClientId2 = <<"client2">>,
+    ClientId3 = <<"client3">>,
+
+    {ok, C1} = emqtt:start_link(#{
+        clientid => ClientId1,
+        proto_ver => v5,
+        properties => #{'Session-Expiry-Interval' => 120}
+    }),
+    {ok, _} = emqtt:connect(C1),
+    {ok, C2} = emqtt:start_link(#{clientid => ClientId2}),
+    {ok, _} = emqtt:connect(C2),
+    {ok, C3} = emqtt:start_link(#{clientid => ClientId3}),
+    {ok, _} = emqtt:connect(C3),
+
+    timer:sleep(300),
+
+    %% get /clients
+    ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
+    {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
+    ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]),
+    ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
+    ClientsPage = maps:get(<<"page">>, ClientsMeta),
+    ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
+    ClientsCount = maps:get(<<"count">>, ClientsMeta),
+    ?assertEqual(ClientsPage, 1),
+    ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()),
+    ?assertEqual(ClientsCount, 3),
+
+    %% kickout clients
+    KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]),
+    KickoutBody = [ClientId1, ClientId2, ClientId3],
+    {ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody),
+
+    {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
+    ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]),
+    ?assertMatch(#{<<"data">> := []}, ClientsResponse2).

apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl

@@ -26,10 +26,6 @@
 -define(AUTHN_BACKEND, ldap).
 -define(AUTHN_BACKEND_BIN, <<"ldap">>).
 
--define(AUTHN_BACKEND_BIND, ldap_bind).
--define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>).
-
 -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}).
--define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}).
 
 -endif.

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_ldap, [
     {description, "EMQX LDAP Authentication and Authorization"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_auth_ldap_app, []}},
     {applications, [

apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl

@@ -25,12 +25,10 @@
 start(_StartType, _StartArgs) ->
     ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap),
     ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap),
-    ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind),
     {ok, Sup} = emqx_auth_ldap_sup:start_link(),
     {ok, Sup}.
 
 stop(_State) ->
     ok = emqx_authn:deregister_provider(?AUTHN_TYPE),
-    ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND),
     ok = emqx_authz:unregister_source(?AUTHZ_TYPE),
     ok.

apps/emqx_auth_ldap/src/emqx_authn_ldap.erl

@@ -16,19 +16,10 @@
 
 -module(emqx_authn_ldap).
 
--include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("emqx/include/logger.hrl").
--include_lib("eldap/include/eldap.hrl").
 
 -behaviour(emqx_authn_provider).
 
-%% a compatible attribute for version 4.x
--define(ISENABLED_ATTR, "isEnabled").
--define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
-%% TODO
-%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
-%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-
 -export([
     create/2,
     update/2,
@@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) ->
     ignore;
 authenticate(#{password := undefined}, _) ->
     {error, bad_username_or_password};
-authenticate(
+authenticate(Credential, #{method := #{type := Type}} = State) ->
-    #{password := Password} = Credential,
+    case Type of
-    #{
+        hash ->
-        password_attribute := PasswordAttr,
+            emqx_authn_ldap_hash:authenticate(Credential, State);
-        is_superuser_attribute := IsSuperuserAttr,
+        bind ->
-        query_timeout := Timeout,
+            emqx_authn_ldap_bind:authenticate(Credential, State)
-        resource_id := ResourceId
-    } = State
-) ->
-    case
-        emqx_resource:simple_sync_query(
-            ResourceId,
-            {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
-        )
-    of
-        {ok, []} ->
-            ignore;
-        {ok, [Entry]} ->
-            is_enabled(Password, Entry, State);
-        {error, Reason} ->
-            ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
-                resource => ResourceId,
-                timeout => Timeout,
-                reason => Reason
-            }),
-            ignore
     end.
 
+%% it used the deprecated config form
+parse_config(
+    #{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0
+) ->
+    Config = maps:without([password_attribute, is_superuser_attribute], Config0),
+    parse_config(Config#{
+        method => #{
+            type => hash,
+            password_attribute => PasswordAttr,
+            is_superuser_attribute => IsSuperuserAttr
+        }
+    });
 parse_config(Config) ->
-    maps:with([query_timeout, password_attribute, is_superuser_attribute], Config).
+    maps:with([query_timeout, method], Config).
-
-%% To compatible v4.x
-is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
-    IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
-    case emqx_authn_utils:to_bool(IsEnabled) of
-        true ->
-            ensure_password(Password, Entry, State);
-        _ ->
-            {error, user_disabled}
-    end.
-
-ensure_password(
-    Password,
-    #eldap_entry{attributes = Attributes} = Entry,
-    #{password_attribute := PasswordAttr} = State
-) ->
-    case get_value(PasswordAttr, Attributes) of
-        undefined ->
-            {error, no_password};
-        [LDAPPassword | _] ->
-            extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
-    end.
-
-%% RFC 2307 format password
-%% https://datatracker.ietf.org/doc/html/rfc2307
-extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
-    case
-        re:run(
-            LDAPPassword,
-            "{([^{}]+)}(.+)",
-            [{capture, all_but_first, list}, global]
-        )
-    of
-        {match, [[HashTypeStr, PasswordHashStr]]} ->
-            case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
-                {ok, HashType} ->
-                    PasswordHash = to_binary(PasswordHashStr),
-                    is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
-                _Error ->
-                    {error, invalid_hash_type}
-            end;
-        _ ->
-            OnFail(LDAPPassword, Password, Entry, State)
-    end.
-
-is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
-    case lists:member(HashType, ?VALID_ALGORITHMS) of
-        true ->
-            verify_password(HashType, PasswordHash, Password, Entry, State);
-        _ ->
-            {error, {invalid_hash_type, HashType}}
-    end.
-
-%% this password is in LDIF format which is base64 encoding
-try_decode_password(LDAPPassword, Password, Entry, State) ->
-    case safe_base64_decode(LDAPPassword) of
-        {ok, Decode} ->
-            extract_hash_algorithm(
-                Decode,
-                Password,
-                fun(_, _, _, _) ->
-                    {error, invalid_password}
-                end,
-                Entry,
-                State
-            );
-        {error, Reason} ->
-            {error, {invalid_password, Reason}}
-    end.
-
-%% sha with salt
-%% https://www.openldap.org/faq/data/cache/347.html
-verify_password(ssha, PasswordData, Password, Entry, State) ->
-    case safe_base64_decode(PasswordData) of
-        {ok, <<PasswordHash:20/binary, Salt/binary>>} ->
-            verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
-        {ok, _} ->
-            {error, invalid_ssha_password};
-        {error, Reason} ->
-            {error, {invalid_password, Reason}}
-    end;
-verify_password(
-    Algorithm,
-    Base64HashData,
-    Password,
-    Entry,
-    State
-) ->
-    verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
-
-verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
-    PasswordHash = hash_password(Algorithm, Salt, Position, Password),
-    case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
-        true ->
-            {ok, is_superuser(Entry, State)};
-        _ ->
-            {error, bad_username_or_password}
-    end.
-
-is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) ->
-    Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
-    #{is_superuser => emqx_authn_utils:to_bool(Value)}.
-
-safe_base64_decode(Data) ->
-    try
-        {ok, base64:decode(Data)}
-    catch
-        _:Reason ->
-            {error, {invalid_base64_data, Reason}}
-    end.
-
-get_lower_bin_value(Key, Proplists, Default) ->
-    [Value | _] = get_value(Key, Proplists, [Default]),
-    to_binary(string:to_lower(Value)).
-
-to_binary(Value) ->
-    erlang:list_to_binary(Value).
-
-hash_password(Algorithm, _Salt, disable, Password) ->
-    hash_password(Algorithm, Password);
-hash_password(Algorithm, Salt, suffix, Password) ->
-    hash_password(Algorithm, <<Password/binary, Salt/binary>>).
-
-hash_password(Algorithm, Data) ->
-    crypto:hash(Algorithm, Data).
-
-compare_password(hash, LDAPPasswordHash, PasswordHash) ->
-    emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
-compare_password(base64, Base64HashData, PasswordHash) ->
-    emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl

@@ -20,32 +20,13 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("eldap/include/eldap.hrl").
 
--behaviour(emqx_authn_provider).
-
 -export([
-    create/2,
+    authenticate/2
-    update/2,
-    authenticate/2,
-    destroy/1
 ]).
 
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------
-
-create(_AuthenticatorID, Config) ->
-    emqx_authn_ldap:do_create(?MODULE, Config).
-
-update(Config, State) ->
-    emqx_authn_ldap:update(Config, State).
-
-destroy(State) ->
-    emqx_authn_ldap:destroy(State).
-
-authenticate(#{auth_method := _}, _) ->
-    ignore;
-authenticate(#{password := undefined}, _) ->
-    {error, bad_username_or_password};
 authenticate(
     #{password := _Password} = Credential,
     #{

apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl

@@ -1,66 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_authn_ldap_bind_schema).
-
--behaviour(emqx_authn_schema).
-
--export([
-    fields/1,
-    desc/1,
-    refs/0,
-    select_union_member/1,
-    namespace/0
-]).
-
--include("emqx_auth_ldap.hrl").
--include_lib("hocon/include/hoconsc.hrl").
-
-namespace() -> "authn".
-
-refs() ->
-    [?R_REF(ldap_bind)].
-
-select_union_member(#{
-    <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN
-}) ->
-    refs();
-select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) ->
-    throw(#{
-        reason => "unknown_mechanism",
-        expected => ?AUTHN_MECHANISM
-    });
-select_union_member(_) ->
-    undefined.
-
-fields(ldap_bind) ->
-    [
-        {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
-        {backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)},
-        {query_timeout, fun query_timeout/1}
-    ] ++
-        emqx_authn_schema:common_fields() ++
-        emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts).
-
-desc(ldap_bind) ->
-    ?DESC(ldap_bind);
-desc(_) ->
-    undefined.
-
-query_timeout(type) -> emqx_schema:timeout_duration_ms();
-query_timeout(desc) -> ?DESC(?FUNCTION_NAME);
-query_timeout(default) -> <<"5s">>;
-query_timeout(_) -> undefined.

apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl

@@ -0,0 +1,197 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_ldap_hash).
+
+-include_lib("emqx_auth/include/emqx_authn.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("eldap/include/eldap.hrl").
+
+%% a compatible attribute for version 4.x
+-define(ISENABLED_ATTR, "isEnabled").
+-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
+%% TODO
+%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
+%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
+
+-export([
+    authenticate/2
+]).
+
+-import(proplists, [get_value/2, get_value/3]).
+
+%%------------------------------------------------------------------------------
+%% APIs
+%%------------------------------------------------------------------------------
+authenticate(
+    #{password := Password} = Credential,
+    #{
+        method := #{
+            password_attribute := PasswordAttr,
+            is_superuser_attribute := IsSuperuserAttr
+        },
+        query_timeout := Timeout,
+        resource_id := ResourceId
+    } = State
+) ->
+    case
+        emqx_resource:simple_sync_query(
+            ResourceId,
+            {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
+        )
+    of
+        {ok, []} ->
+            ignore;
+        {ok, [Entry]} ->
+            is_enabled(Password, Entry, State);
+        {error, Reason} ->
+            ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
+                resource => ResourceId,
+                timeout => Timeout,
+                reason => Reason
+            }),
+            ignore
+    end.
+
+%% To compatible v4.x
+is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
+    IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
+    case emqx_authn_utils:to_bool(IsEnabled) of
+        true ->
+            ensure_password(Password, Entry, State);
+        _ ->
+            {error, user_disabled}
+    end.
+
+ensure_password(
+    Password,
+    #eldap_entry{attributes = Attributes} = Entry,
+    #{method := #{password_attribute := PasswordAttr}} = State
+) ->
+    case get_value(PasswordAttr, Attributes) of
+        undefined ->
+            {error, no_password};
+        [LDAPPassword | _] ->
+            extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
+    end.
+
+%% RFC 2307 format password
+%% https://datatracker.ietf.org/doc/html/rfc2307
+extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
+    case
+        re:run(
+            LDAPPassword,
+            "{([^{}]+)}(.+)",
+            [{capture, all_but_first, list}, global]
+        )
+    of
+        {match, [[HashTypeStr, PasswordHashStr]]} ->
+            case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
+                {ok, HashType} ->
+                    PasswordHash = to_binary(PasswordHashStr),
+                    is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
+                _Error ->
+                    {error, invalid_hash_type}
+            end;
+        _ ->
+            OnFail(LDAPPassword, Password, Entry, State)
+    end.
+
+is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
+    case lists:member(HashType, ?VALID_ALGORITHMS) of
+        true ->
+            verify_password(HashType, PasswordHash, Password, Entry, State);
+        _ ->
+            {error, {invalid_hash_type, HashType}}
+    end.
+
+%% this password is in LDIF format which is base64 encoding
+try_decode_password(LDAPPassword, Password, Entry, State) ->
+    case safe_base64_decode(LDAPPassword) of
+        {ok, Decode} ->
+            extract_hash_algorithm(
+                Decode,
+                Password,
+                fun(_, _, _, _) ->
+                    {error, invalid_password}
+                end,
+                Entry,
+                State
+            );
+        {error, Reason} ->
+            {error, {invalid_password, Reason}}
+    end.
+
+%% sha with salt
+%% https://www.openldap.org/faq/data/cache/347.html
+verify_password(ssha, PasswordData, Password, Entry, State) ->
+    case safe_base64_decode(PasswordData) of
+        {ok, <<PasswordHash:20/binary, Salt/binary>>} ->
+            verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
+        {ok, _} ->
+            {error, invalid_ssha_password};
+        {error, Reason} ->
+            {error, {invalid_password, Reason}}
+    end;
+verify_password(
+    Algorithm,
+    Base64HashData,
+    Password,
+    Entry,
+    State
+) ->
+    verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
+
+verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
+    PasswordHash = hash_password(Algorithm, Salt, Position, Password),
+    case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
+        true ->
+            {ok, is_superuser(Entry, State)};
+        _ ->
+            {error, bad_username_or_password}
+    end.
+
+is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) ->
+    Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
+    #{is_superuser => emqx_authn_utils:to_bool(Value)}.
+
+safe_base64_decode(Data) ->
+    try
+        {ok, base64:decode(Data)}
+    catch
+        _:Reason ->
+            {error, {invalid_base64_data, Reason}}
+    end.
+
+get_lower_bin_value(Key, Proplists, Default) ->
+    [Value | _] = get_value(Key, Proplists, [Default]),
+    to_binary(string:to_lower(Value)).
+
+to_binary(Value) ->
+    erlang:list_to_binary(Value).
+
+hash_password(Algorithm, _Salt, disable, Password) ->
+    hash_password(Algorithm, Password);
+hash_password(Algorithm, Salt, suffix, Password) ->
+    hash_password(Algorithm, <<Password/binary, Salt/binary>>).
+
+hash_password(Algorithm, Data) ->
+    crypto:hash(Algorithm, Data).
+
+compare_password(hash, LDAPPasswordHash, PasswordHash) ->
+    emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
+compare_password(base64, Base64HashData, PasswordHash) ->
+    emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl

@@ -32,7 +32,7 @@
 namespace() -> "authn".
 
 refs() ->
-    [?R_REF(ldap)].
+    [?R_REF(ldap), ?R_REF(ldap_deprecated)].
 
 select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) ->
     refs();
@@ -44,12 +44,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) ->
 select_union_member(_) ->
     undefined.
 
+fields(ldap_deprecated) ->
+    common_fields() ++
+        [
+            {password_attribute, password_attribute()},
+            {is_superuser_attribute, is_superuser_attribute()}
+        ];
 fields(ldap) ->
+    common_fields() ++
+        [
+            {method,
+                ?HOCON(
+                    hoconsc:union([?R_REF(hash_method), ?R_REF(bind_method)]),
+                    #{desc => ?DESC(method)}
+                )}
+        ];
+fields(hash_method) ->
+    [
+        {type, method_type(hash)},
+        {password_attribute, password_attribute()},
+        {is_superuser_attribute, is_superuser_attribute()}
+    ];
+fields(bind_method) ->
+    [{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts).
+
+common_fields() ->
     [
         {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
         {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
-        {password_attribute, fun password_attribute/1},
-        {is_superuser_attribute, fun is_superuser_attribute/1},
         {query_timeout, fun query_timeout/1}
     ] ++
         emqx_authn_schema:common_fields() ++
@@ -57,18 +79,35 @@ fields(ldap) ->
 
 desc(ldap) ->
     ?DESC(ldap);
+desc(ldap_deprecated) ->
+    ?DESC(ldap_deprecated);
+desc(hash_method) ->
+    ?DESC(hash_method);
+desc(bind_method) ->
+    ?DESC(bind_method);
 desc(_) ->
     undefined.
 
-password_attribute(type) -> string();
+method_type(Type) ->
-password_attribute(desc) -> ?DESC(?FUNCTION_NAME);
+    ?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}).
-password_attribute(default) -> <<"userPassword">>;
-password_attribute(_) -> undefined.
 
-is_superuser_attribute(type) -> string();
+password_attribute() ->
-is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME);
+    ?HOCON(
-is_superuser_attribute(default) -> <<"isSuperuser">>;
+        string(),
-is_superuser_attribute(_) -> undefined.
+        #{
+            desc => ?DESC(?FUNCTION_NAME),
+            default => <<"userPassword">>
+        }
+    ).
+
+is_superuser_attribute() ->
+    ?HOCON(
+        string(),
+        #{
+            desc => ?DESC(?FUNCTION_NAME),
+            default => <<"isSuperuser">>
+        }
+    ).
 
 query_timeout(type) -> emqx_schema:timeout_duration_ms();
 query_timeout(desc) -> ?DESC(?FUNCTION_NAME);

apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl

@@ -70,6 +70,29 @@ end_per_suite(Config) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
+t_create_with_deprecated_cfg(_Config) ->
+    AuthConfig = deprecated_raw_ldap_auth_config(),
+
+    {ok, _} = emqx:update_config(
+        ?PATH,
+        {create_authenticator, ?GLOBAL, AuthConfig}
+    ),
+
+    {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(
+        ?GLOBAL
+    ),
+    ?assertMatch(
+        #{
+            method := #{
+                type := hash,
+                is_superuser_attribute := _,
+                password_attribute := "not_the_default_value"
+            }
+        },
+        State
+    ),
+    emqx_authn_test_lib:delete_config(?ResourceID).
+
 t_create(_Config) ->
     AuthConfig = raw_ldap_auth_config(),
 
@@ -225,6 +248,19 @@ raw_ldap_auth_config() ->
         <<"pool_size">> => 8
     }.
 
+deprecated_raw_ldap_auth_config() ->
+    #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"ldap">>,
+        <<"server">> => ldap_server(),
+        <<"is_superuser_attribute">> => <<"isSuperuser">>,
+        <<"password_attribute">> => <<"not_the_default_value">>,
+        <<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>,
+        <<"username">> => <<"cn=root,dc=emqx,dc=io">>,
+        <<"password">> => <<"public">>,
+        <<"pool_size">> => 8
+    }.
+
 user_seeds() ->
     New = fun(Username, Password, Result) ->
         #{

apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl

@@ -27,7 +27,7 @@
 -define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>).
 
 -define(PATH, [authentication]).
--define(ResourceID, <<"password_based:ldap_bind">>).
+-define(ResourceID, <<"password_based:ldap">>).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -78,7 +78,7 @@ t_create(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
+    {ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
     emqx_authn_test_lib:delete_config(?ResourceID).
 
 t_create_invalid(_Config) ->
@@ -146,10 +146,10 @@ t_destroy(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_authn_ldap_bind, state := State}]} =
+    {ok, [#{provider := emqx_authn_ldap, state := State}]} =
         emqx_authn_chains:list_authenticators(?GLOBAL),
 
-    {ok, _} = emqx_authn_ldap_bind:authenticate(
+    {ok, _} = emqx_authn_ldap:authenticate(
         #{
             username => <<"mqttuser0001">>,
             password => <<"mqttuser0001">>
@@ -165,7 +165,7 @@ t_destroy(_Config) ->
     % Authenticator should not be usable anymore
     ?assertMatch(
         ignore,
-        emqx_authn_ldap_bind:authenticate(
+        emqx_authn_ldap:authenticate(
             #{
                 username => <<"mqttuser0001">>,
                 password => <<"mqttuser0001">>
@@ -199,7 +199,7 @@ t_update(_Config) ->
     % We update with config with correct query, provider should update and work properly
     {ok, _} = emqx:update_config(
         ?PATH,
-        {update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig}
+        {update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig}
     ),
 
     {ok, _} = emqx_access_control:authenticate(
@@ -218,14 +218,17 @@ t_update(_Config) ->
 raw_ldap_auth_config() ->
     #{
         <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"ldap_bind">>,
+        <<"backend">> => <<"ldap">>,
         <<"server">> => ldap_server(),
         <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
         <<"filter">> => <<"(uid=${username})">>,
         <<"username">> => <<"cn=root,dc=emqx,dc=io">>,
         <<"password">> => <<"public">>,
         <<"pool_size">> => 8,
+        <<"method">> => #{
+            <<"type">> => <<"bind">>,
             <<"bind_password">> => <<"${password}">>
+        }
     }.
 
 user_seeds() ->

apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl

@@ -278,6 +278,10 @@ raw_mongo_auth_config() ->
         <<"server">> => mongo_server(),
         <<"w_mode">> => <<"unsafe">>,
 
+        <<"auth_source">> => mongo_authsource(),
+        <<"username">> => mongo_username(),
+        <<"password">> => mongo_password(),
+
         <<"filter">> => #{<<"username">> => <<"${username}">>},
         <<"password_hash_field">> => <<"password_hash">>,
         <<"salt_field">> => <<"salt">>,
@@ -464,9 +468,21 @@ mongo_config() ->
         {database, <<"mqtt">>},
         {host, ?MONGO_HOST},
         {port, ?MONGO_DEFAULT_PORT},
+        {auth_source, mongo_authsource()},
+        {login, mongo_username()},
+        {password, mongo_password()},
         {register, ?MONGO_CLIENT}
     ].
 
+mongo_authsource() ->
+    iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
+
+mongo_username() ->
+    iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
+
+mongo_password() ->
+    iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
+
 start_apps(Apps) ->
     lists:foreach(fun application:ensure_all_started/1, Apps).
 

apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl

@@ -397,6 +397,10 @@ raw_mongo_authz_config() ->
         <<"collection">> => <<"acl">>,
         <<"server">> => mongo_server(),
 
+        <<"auth_source">> => mongo_authsource(),
+        <<"username">> => mongo_username(),
+        <<"password">> => mongo_password(),
+
         <<"filter">> => #{<<"username">> => <<"${username}">>}
     }.
 
@@ -408,9 +412,21 @@ mongo_config() ->
         {database, <<"mqtt">>},
         {host, ?MONGO_HOST},
         {port, ?MONGO_DEFAULT_PORT},
+        {auth_source, mongo_authsource()},
+        {login, mongo_username()},
+        {password, mongo_password()},
         {register, ?MONGO_CLIENT}
     ].
 
+mongo_authsource() ->
+    iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
+
+mongo_username() ->
+    iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
+
+mongo_password() ->
+    iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
+
 start_apps(Apps) ->
     lists:foreach(fun application:ensure_all_started/1, Apps).
 

apps/emqx_auth_redis/src/emqx_auth_redis.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_redis, [
     {description, "EMQX Redis Authentication and Authorization"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_auth_redis_app, []}},
     {applications, [

apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl

@@ -64,12 +64,8 @@ refs(_) ->
         expected => "single | cluster | sentinel"
     }).
 
-fields(redis_single) ->
+fields(Type) ->
-    common_fields() ++ emqx_redis:fields(single);
+    common_fields() ++ emqx_redis:fields(Type).
-fields(redis_cluster) ->
-    common_fields() ++ emqx_redis:fields(cluster);
-fields(redis_sentinel) ->
-    common_fields() ++ emqx_redis:fields(sentinel).
 
 desc(redis_single) ->
     ?DESC(single);

apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl

@@ -34,17 +34,9 @@ namespace() -> "authz".
 
 type() -> ?AUTHZ_TYPE.
 
-fields(redis_single) ->
+fields(Type) ->
     emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
-        emqx_redis:fields(single) ++
+        emqx_redis:fields(Type) ++
-        [{cmd, cmd()}];
-fields(redis_sentinel) ->
-    emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
-        emqx_redis:fields(sentinel) ++
-        [{cmd, cmd()}];
-fields(redis_cluster) ->
-    emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++
-        emqx_redis:fields(cluster) ++
         [{cmd, cmd()}].
 
 desc(redis_single) ->

apps/emqx_bridge/src/emqx_action_info.erl

@@ -22,7 +22,7 @@
 
 -export([
     action_type_to_connector_type/1,
-    action_type_to_bridge_v1_type/1,
+    action_type_to_bridge_v1_type/2,
     bridge_v1_type_to_action_type/1,
     is_action_type/1,
     registered_schema_modules/0,
@@ -35,7 +35,12 @@
     transform_bridge_v1_config_to_action_config/4
 ]).
 
--callback bridge_v1_type_name() -> atom().
+-callback bridge_v1_type_name() ->
+    atom()
+    | {
+        fun(({ActionConfig :: map(), ConnectorConfig :: map()}) -> Type :: atom()),
+        TypeList :: [atom()]
+    }.
 -callback action_type_name() -> atom().
 -callback connector_type_name() -> atom().
 -callback schema_module() -> atom().
@@ -68,8 +73,16 @@
 -if(?EMQX_RELEASE_EDITION == ee).
 hard_coded_action_info_modules_ee() ->
     [
+        emqx_bridge_azure_event_hub_action_info,
+        emqx_bridge_confluent_producer_action_info,
+        emqx_bridge_gcp_pubsub_producer_action_info,
         emqx_bridge_kafka_action_info,
-        emqx_bridge_azure_event_hub_action_info
+        emqx_bridge_matrix_action_info,
+        emqx_bridge_mongodb_action_info,
+        emqx_bridge_pgsql_action_info,
+        emqx_bridge_syskeeper_action_info,
+        emqx_bridge_timescale_action_info,
+        emqx_bridge_redis_action_info
     ].
 -else.
 hard_coded_action_info_modules_ee() ->
@@ -106,16 +119,30 @@ bridge_v1_type_to_action_type(Type) ->
         ActionType -> ActionType
     end.
 
-action_type_to_bridge_v1_type(Bin) when is_binary(Bin) ->
+action_type_to_bridge_v1_type(Bin, Conf) when is_binary(Bin) ->
-    action_type_to_bridge_v1_type(binary_to_existing_atom(Bin));
+    action_type_to_bridge_v1_type(binary_to_existing_atom(Bin), Conf);
-action_type_to_bridge_v1_type(Type) ->
+action_type_to_bridge_v1_type(ActionType, ActionConf) ->
     ActionInfoMap = info_map(),
     ActionTypeToBridgeV1Type = maps:get(action_type_to_bridge_v1_type, ActionInfoMap),
-    case maps:get(Type, ActionTypeToBridgeV1Type, undefined) of
+    case maps:get(ActionType, ActionTypeToBridgeV1Type, undefined) of
-        undefined -> Type;
+        undefined ->
-        BridgeV1Type -> BridgeV1Type
+            ActionType;
+        BridgeV1TypeFun when is_function(BridgeV1TypeFun) ->
+            case get_confs(ActionType, ActionConf) of
+                {ConnectorConfig, ActionConfig} -> BridgeV1TypeFun({ConnectorConfig, ActionConfig});
+                undefined -> ActionType
+            end;
+        BridgeV1Type ->
+            BridgeV1Type
     end.
 
+get_confs(ActionType, #{<<"connector">> := ConnectorName} = ActionConfig) ->
+    ConnectorType = action_type_to_connector_type(ActionType),
+    ConnectorConfig = emqx_conf:get_raw([connectors, ConnectorType, ConnectorName]),
+    {ConnectorConfig, ActionConfig};
+get_confs(_, _) ->
+    undefined.
+
 %% This function should return true for all inputs that are bridge V1 types for
 %% bridges that have been refactored to bridge V2s, and for all all bridge V2
 %% types. For everything else the function should return false.
@@ -232,37 +259,56 @@ get_info_map(Module) ->
     %% Force the module to get loaded
     _ = code:ensure_loaded(Module),
     ActionType = Module:action_type_name(),
-    BridgeV1Type =
+    {BridgeV1TypeOrFun, BridgeV1Types} =
         case erlang:function_exported(Module, bridge_v1_type_name, 0) of
             true ->
-                Module:bridge_v1_type_name();
+                case Module:bridge_v1_type_name() of
+                    {_BridgeV1TypeFun, _BridgeV1Types} = BridgeV1TypeTuple ->
+                        BridgeV1TypeTuple;
+                    BridgeV1Type0 ->
+                        {BridgeV1Type0, [BridgeV1Type0]}
+                end;
             false ->
-                Module:action_type_name()
+                {ActionType, [ActionType]}
         end,
     #{
-        action_type_names => #{
+        action_type_names =>
-            ActionType => true,
+            lists:foldl(
-            BridgeV1Type => true
+                fun(BridgeV1Type, M) ->
-        },
+                    M#{BridgeV1Type => true}
-        bridge_v1_type_to_action_type => #{
+                end,
-            BridgeV1Type => ActionType,
+                #{ActionType => true},
+                BridgeV1Types
+            ),
+        bridge_v1_type_to_action_type =>
+            lists:foldl(
+                fun(BridgeV1Type, M) ->
                     %% Alias the bridge V1 type to the action type
-            ActionType => ActionType
+                    M#{BridgeV1Type => ActionType}
-        },
+                end,
+                #{ActionType => ActionType},
+                BridgeV1Types
+            ),
         action_type_to_bridge_v1_type => #{
-            ActionType => BridgeV1Type
+            ActionType => BridgeV1TypeOrFun
-        },
-        action_type_to_connector_type => #{
-            ActionType => Module:connector_type_name(),
-            %% Alias the bridge V1 type to the action type
-            BridgeV1Type => Module:connector_type_name()
         },
+        action_type_to_connector_type =>
+            lists:foldl(
+                fun(BridgeV1Type, M) ->
+                    M#{BridgeV1Type => Module:connector_type_name()}
+                end,
+                #{ActionType => Module:connector_type_name()},
+                BridgeV1Types
+            ),
         action_type_to_schema_module => #{
             ActionType => Module:schema_module()
         },
-        action_type_to_info_module => #{
+        action_type_to_info_module =>
-            ActionType => Module,
+            lists:foldl(
-            %% Alias the bridge V1 type to the action type
+                fun(BridgeV1Type, M) ->
-            BridgeV1Type => Module
+                    M#{BridgeV1Type => Module}
-        }
+                end,
+                #{ActionType => Module},
+                BridgeV1Types
+            )
     }.

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.30"},
+    {vsn, "0.1.31"},
     {registered, [emqx_bridge_sup]},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge.erl

@@ -92,7 +92,8 @@
     T == iotdb;
     T == kinesis_producer;
     T == greptimedb;
-    T == azure_event_hub_producer
+    T == azure_event_hub_producer;
+    T == syskeeper_forwarder
 ).
 
 -define(ROOT_KEY, bridges).
@@ -236,9 +237,15 @@ send_to_matched_egress_bridges_loop(Topic, Msg, [Id | Ids]) ->
     send_to_matched_egress_bridges_loop(Topic, Msg, Ids).
 
 send_message(BridgeId, Message) ->
-    {BridgeType, BridgeName} = emqx_bridge_resource:parse_bridge_id(BridgeId),
+    {BridgeV1Type, BridgeName} = emqx_bridge_resource:parse_bridge_id(BridgeId),
-    ResId = emqx_bridge_resource:resource_id(BridgeType, BridgeName),
+    case emqx_bridge_v2:is_bridge_v2_type(BridgeV1Type) of
-    send_message(BridgeType, BridgeName, ResId, Message, #{}).
+        true ->
+            ActionType = emqx_action_info:bridge_v1_type_to_action_type(BridgeV1Type),
+            emqx_bridge_v2:send_message(ActionType, BridgeName, Message, #{});
+        false ->
+            ResId = emqx_bridge_resource:resource_id(BridgeV1Type, BridgeName),
+            send_message(BridgeV1Type, BridgeName, ResId, Message, #{})
+    end.
 
 send_message(BridgeType, BridgeName, ResId, Message, QueryOpts0) ->
     case emqx:get_config([?ROOT_KEY, BridgeType, BridgeName], not_found) of
@@ -376,8 +383,8 @@ disable_enable(Action, BridgeType0, BridgeName) when
             )
     end.
 
-create(BridgeType0, BridgeName, RawConf) ->
+create(BridgeV1Type, BridgeName, RawConf) ->
-    BridgeType = upgrade_type(BridgeType0),
+    BridgeType = upgrade_type(BridgeV1Type),
     ?SLOG(debug, #{
         bridge_action => create,
         bridge_type => BridgeType,
@@ -386,7 +393,7 @@ create(BridgeType0, BridgeName, RawConf) ->
     }),
     case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of
         true ->
-            emqx_bridge_v2:bridge_v1_split_config_and_create(BridgeType, BridgeName, RawConf);
+            emqx_bridge_v2:bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf);
         false ->
             emqx_conf:update(
                 emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
@@ -407,7 +414,7 @@ remove(BridgeType0, BridgeName) ->
     }),
     case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of
         true ->
-            emqx_bridge_v2:remove(BridgeType, BridgeName);
+            emqx_bridge_v2:bridge_v1_remove(BridgeType0, BridgeName);
         false ->
             remove_v1(BridgeType, BridgeName)
     end.