From 08e1566490c8e8c1bfa7d7753a55df3339f0031f Mon Sep 17 00:00:00 2001 From: Mayer Maximilian Date: Mon, 24 Oct 2022 09:51:49 +0200 Subject: [PATCH 001/135] feat: Add pod disruption budget to helm chart Now it is possible to define the pod disruption budget. Closes: #8222 --- deploy/charts/emqx/templates/pdb.yaml | 18 ++++++++++++++++++ deploy/charts/emqx/values.yaml | 7 +++++++ 2 files changed, 25 insertions(+) create mode 100644 deploy/charts/emqx/templates/pdb.yaml diff --git a/deploy/charts/emqx/templates/pdb.yaml b/deploy/charts/emqx/templates/pdb.yaml new file mode 100644 index 000000000..a3f233064 --- /dev/null +++ b/deploy/charts/emqx/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "emqx.fullname" . }}-pdb + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index b648f070f..7af0eff71 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -211,3 +211,10 @@ ssl: issuer: name: letsencrypt-dns kind: ClusterIssuer + +## Setting PodDisruptionBudget. +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb +## +pdb: + enabled: false + maxUnavailable: 1 From cf77dcf25eded3b3058b8192ce470f28edd11a38 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 2 Feb 2023 14:53:17 +0100 Subject: [PATCH 002/135] test(emqx_ee_connector): add basic tests for influxdb incl. SSL opts This adds a test suite for the emqx_ee_connector_influxdb. We add it so that SSL transport options are properly tested. --- lib-ee/emqx_ee_connector/docker-ct | 2 + .../src/emqx_ee_connector_influxdb.erl | 14 ++ .../test/emqx_ee_connector_influxdb_SUITE.erl | 196 ++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 lib-ee/emqx_ee_connector/docker-ct create mode 100644 lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct new file mode 100644 index 000000000..ef579c036 --- /dev/null +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +influxdb diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index d689f4bf3..988c19156 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -29,6 +29,7 @@ -export([reply_callback/2]). -export([ + roots/0, namespace/0, fields/1, desc/1 @@ -139,6 +140,19 @@ on_get_status(_InstId, #{client := Client}) -> %% schema namespace() -> connector_influxdb. +roots() -> + [ + {config, #{ + type => hoconsc:union( + [ + hoconsc:ref(?MODULE, influxdb_udp), + hoconsc:ref(?MODULE, influxdb_api_v1), + hoconsc:ref(?MODULE, influxdb_api_v2) + ] + ) + }} + ]. + fields(common) -> [ {server, server()}, diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl new file mode 100644 index 000000000..01bb8a08e --- /dev/null +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_influxdb_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(INFLUXDB_RESOURCE_MOD, emqx_ee_connector_influxdb). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + InfluxDBTCPHost = os:getenv("INFLUXDB_APIV2_TCP_HOST", "toxiproxy"), + InfluxDBTCPPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")), + InfluxDBTLSHost = os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"), + InfluxDBTLSPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")), + case emqx_common_test_helpers:is_tcp_server_available(InfluxDBTCPHost, InfluxDBTCPPort) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + [ + {influxdb_tcp_host, InfluxDBTCPHost}, + {influxdb_tcp_port, InfluxDBTCPPort}, + {influxdb_tls_host, InfluxDBTLSHost}, + {influxdb_tls_port, InfluxDBTLSPort} + | Config + ]; + false -> + {skip, no_influxdb} + end. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_lifecycle(Config) -> + Host = ?config(influxdb_tcp_host, Config), + Port = ?config(influxdb_tcp_port, Config), + perform_lifecycle_check( + <<"emqx_ee_connector_influxdb_SUITE">>, + influxdb_config(Host, Port, false, "verify_none") + ). + +perform_lifecycle_check(PoolName, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig), + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, + {ok, #{ + state := #{client := #{pool := ReturnedPoolName}} = State, + status := InitialStatus + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?INFLUXDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + % Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + % % Perform query as further check that the resource is working as expected + ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Resource will be listed still, but state will be changed and healthcheck will fail + % as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(PoolName)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + +t_tls_opts(Config) -> + PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + Host = ?config(influxdb_tls_host, Config), + Port = ?config(influxdb_tls_port, Config), + VerifyNoneStatus = perform_tls_opts_check( + PoolName, influxdb_config(Host, Port, true, "verify_none") + ), + ?assertEqual(connected, VerifyNoneStatus), + VerifyPeerStatus = perform_tls_opts_check( + PoolName, influxdb_config(Host, Port, true, "verify_peer") + ), + ?assertEqual(disconnected, VerifyPeerStatus), + ok. + +perform_tls_opts_check(PoolName, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig), + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, + {ok, #{ + config := #{ssl := #{enable := SslEnabled}}, + status := Status + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?INFLUXDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assert(SslEnabled), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + Status. + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ + +influxdb_config(Host, Port, SslEnabled, Verify) -> + RawConfig = list_to_binary( + io_lib:format( + "" + "\n" + " bucket = mqtt\n" + " org = emqx\n" + " token = abcdefg\n" + " server = \"~s:~b\"\n" + " ssl {\n" + " enable = ~s\n" + " verify = ~s\n" + " }\n" + " " + "", + [Host, Port, SslEnabled, Verify] + ) + ), + + {ok, ResourceConfig} = hocon:binary(RawConfig), + #{<<"config">> => ResourceConfig}. + +influxdb_write_syntax() -> + [ + #{ + measurement => "${topic}", + tags => [{"clientid", "${clientid}"}], + fields => [{"payload", "${payload}"}], + timestamp => undefined + } + ]. + +test_query() -> + {send_message, #{ + <<"clientid">> => <<"something">>, + <<"payload">> => #{bool => true}, + <<"topic">> => <<"connector_test">> + }}. From cfd0e9ebdd9dbf3fef3d9b0c4d8ced08008ea7ba Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 9 Feb 2023 17:08:36 +0100 Subject: [PATCH 003/135] refactor(emqx_ee_connector): use emqx_tls_lib for influx ssl opts We used to simply pass on SSL options given to the influxdb EE connector, but we now pass them to emqx_tls_lib instead. This ensures a proper handling of SSL options and also allow us to use meck to inject custom options in tests. --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 988c19156..f056f4af2 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -341,7 +341,7 @@ ssl_config(SSL = #{enable := true}) -> [ {https_enabled, true}, {transport, ssl}, - {transport_opts, maps:to_list(maps:remove(enable, SSL))} + {transport_opts, emqx_tls_lib:to_client_opts(SSL)} ]. username(#{username := Username}) -> From bc6c653c9fd83dbd446de42b9465fc8289820cc0 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 9 Feb 2023 17:12:01 +0100 Subject: [PATCH 004/135] test(emqx_ee_connector): rework and improve influxdb tests --- .../test/emqx_ee_connector_influxdb_SUITE.erl | 88 +++++++++++++------ 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl index 01bb8a08e..a2e6c7c8f 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl @@ -117,40 +117,64 @@ perform_lifecycle_check(PoolName, InitialConfig) -> % Should not even be able to get the resource data out of ets now unlike just stopping. ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). -t_tls_opts(Config) -> +t_tls_verify_none(Config) -> PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, Host = ?config(influxdb_tls_host, Config), Port = ?config(influxdb_tls_port, Config), - VerifyNoneStatus = perform_tls_opts_check( - PoolName, influxdb_config(Host, Port, true, "verify_none") - ), - ?assertEqual(connected, VerifyNoneStatus), - VerifyPeerStatus = perform_tls_opts_check( - PoolName, influxdb_config(Host, Port, true, "verify_peer") - ), - ?assertEqual(disconnected, VerifyPeerStatus), + InitialConfig = influxdb_config(Host, Port, true, "verify_none"), + ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), + ?assertEqual(connected, ValidStatus), + InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), + ?assertEqual(disconnected, InvalidStatus), ok. -perform_tls_opts_check(PoolName, InitialConfig) -> +t_tls_verify_peer(Config) -> + PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + Host = ?config(influxdb_tls_host, Config), + Port = ?config(influxdb_tls_port, Config), + InitialConfig = influxdb_config(Host, Port, true, "verify_peer"), + ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), + ?assertEqual(connected, ValidStatus), + InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), + ?assertEqual(disconnected, InvalidStatus), + ok. + +perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig), - % We need to add a write_syntax to the config since the connector - % expects this - FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, - {ok, #{ - config := #{ssl := #{enable := SslEnabled}}, - status := Status - }} = emqx_resource:create_local( - PoolName, - ?CONNECTOR_RESOURCE_GROUP, - ?INFLUXDB_RESOURCE_MOD, - FullConfig, - #{} + % Meck handling of TLS opt handling so that we can inject custom + % verification returns + meck:new(emqx_tls_lib, [passthrough, no_link]), + meck:expect( + emqx_tls_lib, + to_client_opts, + fun(Opts) -> + Verify = {verify_fun, {custom_verify(), {return, VerifyReturn}}}, + [Verify | meck:passthrough([Opts])] + end ), - ?assert(SslEnabled), - % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - Status. + try + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, + {ok, #{ + config := #{ssl := #{enable := SslEnabled}}, + status := Status + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?INFLUXDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assert(SslEnabled), + ?assert(meck:validate(emqx_tls_lib)), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + Status + after + meck:unload(emqx_tls_lib) + end. % %%------------------------------------------------------------------------------ % %% Helpers @@ -178,6 +202,18 @@ influxdb_config(Host, Port, SslEnabled, Verify) -> {ok, ResourceConfig} = hocon:binary(RawConfig), #{<<"config">> => ResourceConfig}. +custom_verify() -> + fun + (_, {bad_cert, unknown_ca} = Event, {return, Return} = UserState) -> + ct:pal("Call to custom verify fun. Event: ~p UserState: ~p", [Event, UserState]), + {Return, UserState}; + (_, Event, UserState) -> + ct:pal("Unexpected call to custom verify fun. Event: ~p UserState: ~p", [ + Event, UserState + ]), + {fail, unexpected_call_to_verify_fun} + end. + influxdb_write_syntax() -> [ #{ From 4eb4430fe22e7f024ae8fadc5047993dab856271 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Thu, 9 Feb 2023 17:22:19 +0100 Subject: [PATCH 005/135] chore: bump VSN --- lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 56d128601..1d9927d3f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, From ba65cf48c3d62b11dfee40dc4bca5fb401b1c9b1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 13 Feb 2023 16:45:48 +0100 Subject: [PATCH 006/135] feat: add new module emqx_cover.erl --- apps/emqx_machine/src/emqx_cover.erl | 214 +++++++++++++++++++++ apps/emqx_machine/src/emqx_machine.app.src | 2 +- changes/v5.0.18/fix-9966.en.md | 2 + changes/v5.0.18/fix-9966.zh.md | 2 + mix.exs | 3 + rebar.config | 1 + rebar.config.erl | 2 + 7 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_machine/src/emqx_cover.erl create mode 100644 changes/v5.0.18/fix-9966.en.md create mode 100644 changes/v5.0.18/fix-9966.zh.md diff --git a/apps/emqx_machine/src/emqx_cover.erl b/apps/emqx_machine/src/emqx_cover.erl new file mode 100644 index 000000000..c6f610746 --- /dev/null +++ b/apps/emqx_machine/src/emqx_cover.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This module is NOT used in production. +%% It is used to collect coverage data when running blackbox test +-module(emqx_cover). + +-include_lib("covertool/include/covertool.hrl"). + +-ifdef(EMQX_ENTERPRISE). +-define(OUTPUT_APPNAME, 'EMQX Enterprise'). +-else. +-define(OUTPUT_APPNAME, 'EMQX'). +-endif. + +-export([ + start/0, + start/1, + abort/0, + export_and_stop/1, + lookup_source/1 +]). + +%% This is a ETS table to keep a mapping of module name (atom) to +%% .erl file path (relative path from project root) +%% We needed this ETS table because the source file information +%% is missing from the .beam metadata sicne we are using 'deterministic' +%% compile flag. +-define(SRC, emqx_cover_module_src). + +%% @doc Start cover. +%% All emqx_ modules will be cover-compiled, this may cause +%% some excessive RAM consumption and result in warning logs. +start() -> + start(#{}). + +%% @doc Start cover. +%% All emqx_ modules will be cover-compiled, this may cause +%% some excessive RAM consumption and result in warning logs. +%% Supported options: +%% - project_root: the directory to search for .erl source code +%% - debug_secret_file: only applicable to EMQX Enterprise +start(Opts) -> + ok = abort(), + DefaultDir = os_env("EMQX_PROJECT_ROOT"), + ProjRoot = maps:get(project_root, Opts, DefaultDir), + case ProjRoot =:= "" of + true -> + io:format("Project source code root dir is not provided.~n"), + io:format( + "You may either start EMQX node with environment variable EMQX_PROJECT_ROOT set~n" + ), + io:format("Or provide #{project_root => \"/path/to/emqx/\"} as emqx_cover:start arg~n"), + exit(project_root_is_not_set); + false -> + ok + end, + %% spawn a ets table owner + %% this implementation is kept dead-simple + %% because there is no concurrency requirement + Parent = self(), + {Pid, Ref} = + erlang:spawn_monitor( + fun() -> + true = register(?SRC, self()), + _ = ets:new(?SRC, [named_table, public]), + _ = Parent ! {started, self()}, + receive + stop -> + ok + end + end + ), + receive + {started, Pid} -> + ok; + {'DOWN', Ref, process, Pid, Reason} -> + throw({failed_to_start, Reason}) + after 1000 -> + throw({failed_to_start, timeout}) + end, + Modules = modules(Opts), + case cover:start() of + {ok, _Pid} -> + ok; + {error, {already_started, _Pid}} -> + ok; + Other -> + throw(Other) + end, + ok = cover_compile(Modules), + io:format("cover-compiled ~p modules~n", [length(Modules)]), + ok = put_project_root(ProjRoot), + ok = do_build_source_mapping(ProjRoot, Modules), + CachedModulesCount = ets:info(?SRC, size), + io:format("source-cached ~p modules~n", [CachedModulesCount]), + ok. + +%% @doc Abort cover data collection without exporting. +abort() -> + _ = cover:stop(), + case whereis(?SRC) of + undefined -> + ok; + Pid -> + Ref = monitor(process, Pid), + exit(Pid, kill), + receive + {'DOWN', Ref, process, Pid, _} -> + ok + end + end, + ok. + +%% @doc Export coverage report (xml) format. +%% e.g. `emqx_cover:export_and_stop("/tmp/cover.xml").' +export_and_stop(Path) when is_list(Path) -> + ProjectRoot = get_project_root(), + Config = #config{ + appname = ?OUTPUT_APPNAME, + sources = [ProjectRoot], + output = Path, + lookup_source = fun ?MODULE:lookup_source/1 + }, + covertool:generate_report(Config, cover:modules()). + +get_project_root() -> + [{_, Dir}] = ets:lookup(?SRC, {root, ?OUTPUT_APPNAME}), + Dir. + +put_project_root(Dir) -> + _ = ets:insert(?SRC, {{root, ?OUTPUT_APPNAME}, Dir}), + ok. + +do_build_source_mapping(Dir, Modules0) -> + Modules = sets:from_list(Modules0, [{version, 2}]), + All = filelib:wildcard("**/*.erl", Dir), + lists:foreach( + fun(Path) -> + ModuleNameStr = filename:basename(Path, ".erl"), + Module = list_to_atom(ModuleNameStr), + case sets:is_element(Module, Modules) of + true -> + ets:insert(?SRC, {Module, Path}); + false -> + ok + end + end, + All + ), + ok. + +lookup_source(Module) -> + case ets:lookup(?SRC, Module) of + [{_, Path}] -> + Path; + [] -> + false + end. + +modules(_Opts) -> + %% TODO better filter based on Opts, + %% e.g. we may want to see coverage info for ehttpc + Filter = fun is_emqx_module/1, + find_modules(Filter). + +cover_compile(Modules) -> + Results = cover:compile_beam(Modules), + Errors = lists:filter( + fun + ({ok, _}) -> false; + (_) -> true + end, + Results + ), + case Errors of + [] -> + ok; + _ -> + io:format("failed_to_cover_compile:~n~p~n", [Errors]), + throw(failed_to_cover_compile) + end. + +find_modules(Filter) -> + All = code:all_loaded(), + F = fun({M, _BeamPath}) -> Filter(M) andalso {true, M} end, + lists:filtermap(F, All). + +is_emqx_module(?MODULE) -> + %% do not cover-compile self + false; +is_emqx_module(Module) -> + case erlang:atom_to_binary(Module, utf8) of + <<"emqx", _/binary>> -> + true; + _ -> + false + end. + +os_env(Name) -> + os:getenv(Name, ""). diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index c805fdd25..4acb3cb84 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.1.3"}, + {vsn, "0.2.0"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib]}, diff --git a/changes/v5.0.18/fix-9966.en.md b/changes/v5.0.18/fix-9966.en.md new file mode 100644 index 000000000..cc3a0bb8f --- /dev/null +++ b/changes/v5.0.18/fix-9966.en.md @@ -0,0 +1,2 @@ +Add two new Erlang apps 'tools' and 'covertool' to the release. +So we can run profiling and test coverage analysis on release packages. diff --git a/changes/v5.0.18/fix-9966.zh.md b/changes/v5.0.18/fix-9966.zh.md new file mode 100644 index 000000000..df5b7cff7 --- /dev/null +++ b/changes/v5.0.18/fix-9966.zh.md @@ -0,0 +1,2 @@ +在发布包中增加了2个新的 Erlang app，分别是 ‘tools’ 和 ‘covertool’。 +这两个 app 可以用于性能和测试覆盖率的分析。 diff --git a/mix.exs b/mix.exs index a2df76701..baa5750f0 100644 --- a/mix.exs +++ b/mix.exs @@ -46,6 +46,7 @@ defmodule EMQXUmbrella.MixProject do [ {:lc, github: "emqx/lc", tag: "0.3.2", override: true}, {:redbug, "2.0.8"}, + {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.6", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, @@ -222,6 +223,8 @@ defmodule EMQXUmbrella.MixProject do emqx_plugin_libs: :load, esasl: :load, observer_cli: :permanent, + tools: :load, + covertool: :load, system_monitor: :load, emqx_http_lib: :permanent, emqx_resource: :permanent, diff --git a/rebar.config b/rebar.config index ffdb7407a..7997f2c4b 100644 --- a/rebar.config +++ b/rebar.config @@ -46,6 +46,7 @@ {deps, [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} , {redbug, "2.0.8"} + , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 4ff94bd78..14b84213b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -378,6 +378,8 @@ relx_apps(ReleaseType, Edition) -> {emqx_plugin_libs, load}, {esasl, load}, observer_cli, + {tools, load}, + {covertool, load}, % started by emqx_machine {system_monitor, load}, emqx_http_lib, From f738415b631f1272cba6deb33c036c0df0a714c4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 13 Feb 2023 19:18:32 +0100 Subject: [PATCH 007/135] ci: allow minor version bump in app vsns --- scripts/apps-version-check.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 3432c757c..797204cc8 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -51,6 +51,10 @@ for app in ${APPS}; do [ "${old_app_version_semver[1]}" = "${now_app_version_semver[1]}" ] && \ [ "$(( old_app_version_semver[2] + 1 ))" = "${now_app_version_semver[2]}" ]; then true + elif [ "${old_app_version_semver[0]}" = "${now_app_version_semver[0]}" ] && \ + [ "$(( old_app_version_semver[1] + 1 ))" = "${now_app_version_semver[1]}" ] && \ + [ "${now_app_version_semver[2]}" = "0" ]; then + true else echo "$src_file: non-strict semver version bump from $old_app_version to $now_app_version" bad_app_count=$(( bad_app_count + 1)) From 1cf88ea0b4c141a21a37372925b7178bac088f54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Feb 2023 15:24:10 +0100 Subject: [PATCH 008/135] docs: improve bridge_mode description --- apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf index f9f79beb8..0de97d84b 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf @@ -114,9 +114,13 @@ topic filters for remote.topic of ingress connections.""" desc { en: """If enable bridge mode. NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT -broker MUST support this feature.""" +broker MUST support this feature. +If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. +This means that loop detection will be more effective and that retained messages will be propagated correctly.""" zh: """是否启用 Bridge Mode。 -注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。""" +注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 +如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 +这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" } label { en: "Bridge Mode" From 24825da2a813a5ecb8722874627feda9f62e15e6 Mon Sep 17 00:00:00 2001 From: Adrian Deaconu Date: Fri, 17 Feb 2023 01:10:52 +0000 Subject: [PATCH 009/135] feat: Add MQTT ingress and remove mgmt references --- deploy/charts/emqx/README.md | 24 ++++++----- deploy/charts/emqx/templates/ingress.yaml | 50 +++++++++++++++++++++++ deploy/charts/emqx/values.yaml | 14 +++++++ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 6ee3617ce..352de3740 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | | `service.loadBalancerIP` | loadBalancerIP for Service | nil | | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | | `service.externalIPs` | ExternalIPs for the service | [] | -`service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` +| `service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` | `service.annotations` | Service annotations | {}(evaluated as a template) | | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.mqtt.enabled` | Enable ingress for MQTT | false | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | +| `ingress.mqtt.path` | Ingress path for MQTT | / | +| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` | +| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local | +| `ingress.mqtt.tls` | Ingress tls for MQTT | [] | +| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | | `metrics.type` | Now we only supported "prometheus" | "prometheus" | | `ssl.enabled` | Enable SSL support | false | diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index b6f496d88..29bac213d 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -48,3 +48,53 @@ spec: {{- end }} --- {{- end }} +{{- if .Values.ingress.mqtt.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.ingress.mqtt.annotations }} + annotations: + {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }} + {{- end }} +spec: +{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }} +{{- end }} + rules: + {{- range $host := .Values.ingress.mqtt.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ $.Values.ingress.mqtt.path | default "/" }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mqtt }} + {{- else }} + serviceName: {{ include "emqx.fullname" $ }} + servicePort: {{ $.Values.service.mqtt }} + {{- end }} + {{- end -}} + {{- if .Values.ingress.mqtt.tls }} + tls: + {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index c737c8808..e3cb4fc25 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -189,6 +189,20 @@ ingress: hosts: - dashboard.emqx.local tls: [] + ## ingress for MQTT + mqtt: + enabled: false + # ingressClassName: haproxy + annotations: {} + # kubernetes.io/ingress.class: haproxy + # kubernetes.io/tls-acme: "true" + # haproxy-ingress.github.io/tcp-service-port: "8883" + # haproxy-ingress.github.io/proxy-protocol: "v2" + path: / + pathType: ImplementationSpecific + hosts: + - mqtt.emqx.local + tls: [] podSecurityContext: enabled: true From 843ed464d57878f1871ef4b1c0069bb124e67c53 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Fri, 17 Feb 2023 10:47:57 +0100 Subject: [PATCH 010/135] fix: remove all mentions of UDP support for InfluxDB EE bridge The InfluxDB EE bridge doesn't support the UDP protocol, but it is defined in the schema. This commit removes all such traces in the schema. --- .../src/emqx_ee_bridge_influxdb.erl | 22 +------------------ .../i18n/emqx_ee_connector_influxdb.conf | 14 ++---------- .../src/emqx_ee_connector_influxdb.erl | 16 +------------- 3 files changed, 4 insertions(+), 48 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 6d96e3883..14f53b5e7 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -31,12 +31,6 @@ conn_bridge_examples(Method) -> [ - #{ - <<"influxdb_udp">> => #{ - summary => <<"InfluxDB UDP Bridge">>, - value => values("influxdb_udp", Method) - } - }, #{ <<"influxdb_api_v1">> => #{ summary => <<"InfluxDB HTTP API V1 Bridge">>, @@ -71,12 +65,6 @@ values("influxdb_api_v1", post) -> server => <<"127.0.0.1:8086">> }, values(common, "influxdb_api_v1", SupportUint, TypeOpts); -values("influxdb_udp", post) -> - SupportUint = <<>>, - TypeOpts = #{ - server => <<"127.0.0.1:8089">> - }, - values(common, "influxdb_udp", SupportUint, TypeOpts); values(Protocol, put) -> values(Protocol, post). @@ -106,26 +94,20 @@ namespace() -> "bridge_influxdb". roots() -> []. -fields("post_udp") -> - method_fileds(post, influxdb_udp); fields("post_api_v1") -> method_fileds(post, influxdb_api_v1); fields("post_api_v2") -> method_fileds(post, influxdb_api_v2); -fields("put_udp") -> - method_fileds(put, influxdb_udp); fields("put_api_v1") -> method_fileds(put, influxdb_api_v1); fields("put_api_v2") -> method_fileds(put, influxdb_api_v2); -fields("get_udp") -> - method_fileds(get, influxdb_udp); fields("get_api_v1") -> method_fileds(get, influxdb_api_v1); fields("get_api_v2") -> method_fileds(get, influxdb_api_v2); fields(Type) when - Type == influxdb_udp orelse Type == influxdb_api_v1 orelse Type == influxdb_api_v2 + Type == influxdb_api_v1 orelse Type == influxdb_api_v2 -> influxdb_bridge_common_fields() ++ connector_fields(Type). @@ -164,8 +146,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."]; -desc(influxdb_udp) -> - ?DESC(emqx_ee_connector_influxdb, "influxdb_udp"); desc(influxdb_api_v1) -> ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1"); desc(influxdb_api_v2) -> diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf index 81ea39d49..03246c07d 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf @@ -28,24 +28,14 @@ The InfluxDB default port 8086 is used if `[:Port]` is not specified. } protocol { desc { - en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2.""" - zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。""" + en: """InfluxDB's protocol. HTTP API or HTTP API V2.""" + zh: """InfluxDB 协议。HTTP API 或 HTTP API V2。""" } label { en: """Protocol""" zh: """协议""" } } - influxdb_udp { - desc { - en: """InfluxDB's UDP protocol.""" - zh: """InfluxDB UDP 协议。""" - } - label { - en: """UDP Protocol""" - zh: """UDP 协议""" - } - } influxdb_api_v1 { desc { en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index f056f4af2..785ec5d07 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -145,7 +145,6 @@ roots() -> {config, #{ type => hoconsc:union( [ - hoconsc:ref(?MODULE, influxdb_udp), hoconsc:ref(?MODULE, influxdb_api_v1), hoconsc:ref(?MODULE, influxdb_api_v2) ] @@ -165,8 +164,6 @@ fields(common) -> required => false, default => ms, desc => ?DESC("precision") })} ]; -fields(influxdb_udp) -> - fields(common); fields(influxdb_api_v1) -> fields(common) ++ [ @@ -199,8 +196,6 @@ server() -> desc(common) -> ?DESC("common"); -desc(influxdb_udp) -> - ?DESC("influxdb_udp"); desc(influxdb_api_v1) -> ?DESC("influxdb_api_v1"); desc(influxdb_api_v2) -> @@ -326,12 +321,7 @@ protocol_config(#{ {bucket, str(Bucket)}, {org, str(Org)}, {token, Token} - ] ++ ssl_config(SSL); -%% udp config -protocol_config(_) -> - [ - {protocol, udp} - ]. + ] ++ ssl_config(SSL). ssl_config(#{enable := false}) -> [ @@ -659,10 +649,6 @@ desc_test_() -> {desc, _, _}, desc(common) ), - ?_assertMatch( - {desc, _, _}, - desc(influxdb_udp) - ), ?_assertMatch( {desc, _, _}, desc(influxdb_api_v1) From caacc08ff9a8a326a093b96604cdbfcc6f5c0113 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Fri, 17 Feb 2023 15:28:27 +0100 Subject: [PATCH 011/135] chore: bump app VSN --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index b2a3c80c6..c30c927f2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, From ca28be4387214f0eb86ff76e7b1e8230b72a57b9 Mon Sep 17 00:00:00 2001 From: Erik Timan Date: Fri, 17 Feb 2023 15:28:49 +0100 Subject: [PATCH 012/135] test(emqx_ee_connector): review fixes of influxdb tests --- .../test/emqx_ee_connector_influxdb_SUITE.erl | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl index a2e6c7c8f..f5e43c0bb 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl @@ -24,7 +24,8 @@ init_per_suite(Config) -> InfluxDBTCPPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")), InfluxDBTLSHost = os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"), InfluxDBTLSPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")), - case emqx_common_test_helpers:is_tcp_server_available(InfluxDBTCPHost, InfluxDBTCPPort) of + Servers = [{InfluxDBTCPHost, InfluxDBTCPPort}, {InfluxDBTLSHost, InfluxDBTLSPort}], + case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of true -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), @@ -37,7 +38,12 @@ init_per_suite(Config) -> | Config ]; false -> - {skip, no_influxdb} + case os:getenv("IS_CI") of + "yes" -> + throw(no_influxdb); + _ -> + {skip, no_influxdb} + end end. end_per_suite(_Config) -> @@ -60,7 +66,7 @@ t_lifecycle(Config) -> Port = ?config(influxdb_tcp_port, Config), perform_lifecycle_check( <<"emqx_ee_connector_influxdb_SUITE">>, - influxdb_config(Host, Port, false, "verify_none") + influxdb_config(Host, Port, false, <<"verify_none">>) ). perform_lifecycle_check(PoolName, InitialConfig) -> @@ -121,7 +127,7 @@ t_tls_verify_none(Config) -> PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, Host = ?config(influxdb_tls_host, Config), Port = ?config(influxdb_tls_port, Config), - InitialConfig = influxdb_config(Host, Port, true, "verify_none"), + InitialConfig = influxdb_config(Host, Port, true, <<"verify_none">>), ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), ?assertEqual(connected, ValidStatus), InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), @@ -132,7 +138,8 @@ t_tls_verify_peer(Config) -> PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, Host = ?config(influxdb_tls_host, Config), Port = ?config(influxdb_tls_port, Config), - InitialConfig = influxdb_config(Host, Port, true, "verify_peer"), + InitialConfig = influxdb_config(Host, Port, true, <<"verify_peer">>), + %% This works without a CA-cert & friends since we are using a mock ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), ?assertEqual(connected, ValidStatus), InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), @@ -181,25 +188,17 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> % %%------------------------------------------------------------------------------ influxdb_config(Host, Port, SslEnabled, Verify) -> - RawConfig = list_to_binary( - io_lib:format( - "" - "\n" - " bucket = mqtt\n" - " org = emqx\n" - " token = abcdefg\n" - " server = \"~s:~b\"\n" - " ssl {\n" - " enable = ~s\n" - " verify = ~s\n" - " }\n" - " " - "", - [Host, Port, SslEnabled, Verify] - ) - ), - - {ok, ResourceConfig} = hocon:binary(RawConfig), + Server = list_to_binary(io_lib:format("~s:~b", [Host, Port])), + ResourceConfig = #{ + <<"bucket">> => <<"mqtt">>, + <<"org">> => <<"emqx">>, + <<"token">> => <<"abcdefg">>, + <<"server">> => Server, + <<"ssl">> => #{ + <<"enable">> => SslEnabled, + <<"verify">> => Verify + } + }, #{<<"config">> => ResourceConfig}. custom_verify() -> From 00b59b493928daa001fe9300f19ee18325580cbb Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 1 Nov 2022 23:03:22 +0100 Subject: [PATCH 013/135] feat(quic): WIP multi-stream --- apps/emqx/src/emqx_connection.erl | 21 ++-- apps/emqx/src/emqx_listeners.erl | 3 +- apps/emqx/src/emqx_quic_connection.erl | 104 +++++++++++++++--- apps/emqx/src/emqx_quic_stream.erl | 2 +- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 8 ++ 5 files changed, 111 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 5b783f2fe..6c88b87cf 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -525,11 +525,10 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), when_bytes_in(Oct, Data, State); -handle_msg({quic, Data, _Sock, _, _, _}, State) -> - Oct = iolist_size(Data), - inc_counter(incoming_bytes, Oct), - ok = emqx_metrics:inc('bytes.received', Oct), - when_bytes_in(Oct, Data, State); +handle_msg({quic, Data, _Stream, #{len := Len}}, State) when is_binary(Data) -> + inc_counter(incoming_bytes, Len), + ok = emqx_metrics:inc('bytes.received', Len), + when_bytes_in(Len, Data, State); handle_msg(check_cache, #state{limiter_buffer = Cache} = State) -> case queue:peek(Cache) of empty -> @@ -893,12 +892,12 @@ handle_info({sock_error, Reason}, State) -> false -> ok end, handle_info({sock_closed, Reason}, close_socket(State)); -handle_info({quic, peer_send_shutdown, _Stream}, State) -> - handle_info({sock_closed, force}, close_socket(State)); -handle_info({quic, closed, _Channel, ReasonFlag}, State) -> - handle_info({sock_closed, ReasonFlag}, State); -handle_info({quic, closed, _Stream}, State) -> - handle_info({sock_closed, force}, State); +%% handle_info({quic, peer_send_shutdown, _Stream}, State) -> +%% handle_info({sock_closed, force}, close_socket(State)); +%% handle_info({quic, closed, _Channel, ReasonFlag}, State) -> +%% handle_info({sock_closed, ReasonFlag}, State); +%% handle_info({quic, closed, _Stream}, State) -> +%% handle_info({sock_closed, force}, State); handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 003c8785e..45f3b2cfd 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -375,7 +375,8 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, - {server_resumption_level, 2} + {server_resumption_level, 2}, + {verify, none} ], ConnectionOpts = #{ conn_callback => emqx_quic_connection, diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 9a2589a3a..6da9ec9a8 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -22,24 +22,42 @@ -define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0). -endif. -%% Callbacks +-behavior(quicer_connection). + -export([ init/1, - new_conn/2, - connected/2, - shutdown/2 + new_conn/3, + connected/3, + transport_shutdown/3, + shutdown/3, + closed/3, + local_address_changed/3, + peer_address_changed/3, + streams_available/3, + peer_needs_streams/3, + resumed/3, + nst_received/3, + new_stream/3 ]). -type cb_state() :: map() | proplists:proplist(). +-type cb_ret() :: ok. --spec init(cb_state()) -> cb_state(). init(ConnOpts) when is_list(ConnOpts) -> init(maps:from_list(ConnOpts)); +init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> + init(S#{stream_opts := maps:from_list(SOpts)}); init(ConnOpts) when is_map(ConnOpts) -> - ConnOpts. + {ok, ConnOpts}. --spec new_conn(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -new_conn(Conn, #{zone := Zone} = S) -> +closed(_Conn, #{is_peer_acked := true}, S) -> + {stop, normal, S}; +closed(_Conn, #{is_peer_acked := false}, S) -> + {stop, abnorml, S}. + +-spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) -> + {ok, cb_state()} | {error, any()}. +new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) -> process_flag(trap_exit, true), case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of false -> @@ -47,7 +65,7 @@ new_conn(Conn, #{zone := Zone} = S) -> receive {Pid, stream_acceptor_ready} -> ok = quicer:async_handshake(Conn), - {ok, S}; + {ok, S#{conn => Conn}}; {'EXIT', Pid, _Reason} -> {error, stream_accept_error} end; @@ -56,18 +74,76 @@ new_conn(Conn, #{zone := Zone} = S) -> {error, overloaded} end. --spec connected(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -connected(Conn, #{slow_start := false} = S) -> +-spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) -> + {ok, cb_state()} | {error, any()}. +connected(Conn, _Props, #{slow_start := false} = S) -> {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), {ok, S}; -connected(_Conn, S) -> +connected(_Conn, _Props, S) -> {ok, S}. --spec shutdown(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -shutdown(Conn, S) -> +-spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret(). +resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when + is_function(ResumeFun) +-> + ResumeFun(Conn, Data, S); +resumed(_Conn, _Data, S) -> + {ok, S}. + +-spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret(). +nst_received(_Conn, _Data, S) -> + {stop, no_nst_for_server, S}. + +-spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret(). +new_stream( + Stream, + #{is_orphan := true} = Props, + #{ + conn := Conn, + streams := Streams, + stream_opts := SOpts + } = CBState +) -> + %% Spawn new stream + case quicer_stream:start_link(emqx_quic_stream, Stream, Conn, SOpts, Props) of + {ok, StreamOwner} -> + quicer_connection:handoff_stream(Stream, StreamOwner), + {ok, CBState#{streams := [{StreamOwner, Stream} | Streams]}}; + Other -> + Other + end. +-spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). +shutdown(Conn, _ErrorCode, S) -> quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. +-spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) -> + cb_ret(). +transport_shutdown(_C, _DownInfo, S) -> + {ok, S}. + +-spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret(). +peer_address_changed(_C, _NewAddr, S) -> + {ok, S}. + +-spec local_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state()) -> + cb_ret(). +local_address_changed(_C, _NewAddr, S) -> + {ok, S}. + +-spec streams_available( + quicer:connection_handle(), + {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()}, + cb_state() +) -> cb_ret(). +streams_available(_C, {_BidirCnt, _UnidirCnt}, S) -> + {ok, S}. + +-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). +%% for https://github.com/microsoft/msquic/issues/3120 +peer_needs_streams(_C, undefined, S) -> + {ok, S}. + -spec is_zone_olp_enabled(emqx_types:zone()) -> boolean(). is_zone_olp_enabled(Zone) -> case emqx_config:get_zone_conf(Zone, [overload_protection]) of diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 567488862..fe6ff692c 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -37,7 +37,7 @@ wait({ConnOwner, Conn}) -> ConnOwner ! {self(), stream_acceptor_ready}, receive %% from msquic - {quic, new_stream, Stream} -> + {quic, new_stream, Stream, _Props} -> {ok, {quic, Conn, Stream}}; {'EXIT', ConnOwner, _Reason} -> {error, enotconn} diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 7e97c5bf4..07299bd42 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -78,6 +78,14 @@ end_per_group(_Group, _Config) -> init_per_suite(Config) -> %% Start Apps + %% dbg:tracer(process, {fun dbg:dhandler/2,group_leader()}), + %% dbg:p(all,c), + %% dbg:tp(emqx_quic_connection,cx), + %% dbg:tp(emqx_quic_stream,cx), + %% dbg:tp(emqtt_quic,cx), + %% dbg:tp(emqtt,cx), + %% dbg:tp(emqtt_quic_stream,cx), + %% dbg:tp(emqtt_quic_connection,cx), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), Config. From 2d09a054e328ff824b28354c80d917549341805e Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 2 Nov 2022 09:46:48 +0100 Subject: [PATCH 014/135] chore: add some typing --- apps/emqx/rebar.config.script | 2 +- apps/emqx/src/emqx_quic_connection.erl | 6 +++++- rebar.config | 2 +- rebar.config.erl | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 75f748017..0ecd21715 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. +Quicer = {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. %% @TODO revert ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 6da9ec9a8..22d068237 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -40,9 +40,11 @@ new_stream/3 ]). --type cb_state() :: map() | proplists:proplist(). +-type cb_state() :: map(). -type cb_ret() :: ok. +-spec init(map() | list()) -> cb_state(). + init(ConnOpts) when is_list(ConnOpts) -> init(maps:from_list(ConnOpts)); init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> @@ -50,6 +52,8 @@ init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> init(ConnOpts) when is_map(ConnOpts) -> {ok, ConnOpts}. +-spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) -> + {ok, cb_state()} | {error, any()}. closed(_Conn, #{is_peer_acked := true}, S) -> {stop, normal, S}; closed(_Conn, #{is_peer_acked := false}, S) -> diff --git a/rebar.config b/rebar.config index ffdb7407a..76402897b 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}} + , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/multi-streams"}}} %% @TODO revert , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 4ff94bd78..9da71355b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,8 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. + %% @TODO revert + {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From a51c8869086358e9e4387727bdc3b8f593448f46 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 2 Nov 2022 14:20:17 +0100 Subject: [PATCH 015/135] fix: prepare for multi stream --- apps/emqx/src/emqx_connection.erl | 27 ++-- apps/emqx/src/emqx_quic_connection.erl | 72 ++++++--- apps/emqx/src/emqx_quic_stream.erl | 201 +++++++++++++++++++++++-- 3 files changed, 263 insertions(+), 37 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 6c88b87cf..1c8b85808 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/TCP|TLS Connection +%% MQTT/TCP|TLS Connection|QUIC Stream -module(emqx_connection). -include("emqx.hrl"). @@ -189,12 +189,16 @@ ]} ). --spec start_link( - esockd:transport(), - esockd:socket() | {pid(), quicer:connection_handler()}, - emqx_channel:opts() -) -> - {ok, pid()}. +-spec start_link + (esockd:transport(), esockd:socket(), emqx_channel:opts()) -> + {ok, pid()}; + ( + emqx_quic_stream, + {ConnOwner :: pid(), quicer:connection_handler(), quicer:new_conn_props()}, + emqx_quic_connection:cb_state() + ) -> + {ok, pid()}. + start_link(Transport, Socket, Options) -> Args = [self(), Transport, Socket, Options], CPid = proc_lib:spawn_link(?MODULE, init, Args), @@ -324,6 +328,7 @@ init_state( Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg), FrameOpts = #{ + %% @TODO:q what is strict_mode? strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size]) }, @@ -476,7 +481,9 @@ process_msg([Msg | More], State) -> {ok, Msgs, NState} -> process_msg(append_msg(More, Msgs), NState); {stop, Reason, NState} -> - {stop, Reason, NState} + {stop, Reason, NState}; + {stop, Reason} -> + {stop, Reason, State} end catch exit:normal -> @@ -507,7 +514,6 @@ append_msg(Q, Msg) -> %%-------------------------------------------------------------------- %% Handle a Msg - handle_msg({'$gen_call', From, Req}, State) -> case handle_call(From, Req, State) of {reply, Reply, NState} -> @@ -747,6 +753,7 @@ when_bytes_in(Oct, Data, State) -> NState ). +%% @doc: return a reversed Msg list -compile({inline, [next_incoming_msgs/3]}). next_incoming_msgs([Packet], Msgs, State) -> {ok, [{incoming, Packet} | Msgs], State}; @@ -892,6 +899,8 @@ handle_info({sock_error, Reason}, State) -> false -> ok end, handle_info({sock_closed, Reason}, close_socket(State)); +handle_info({quic, Event, Handle, Prop}, State) -> + emqx_quic_stream:Event(Handle, Prop, State); %% handle_info({quic, peer_send_shutdown, _Stream}, State) -> %% handle_info({sock_closed, force}, close_socket(State)); %% handle_info({quic, closed, _Channel, ReasonFlag}, State) -> diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 22d068237..a5af3d4b3 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -16,6 +16,7 @@ -module(emqx_quic_connection). +-include("logger.hrl"). -ifndef(BUILD_WITHOUT_QUIC). -include_lib("quicer/include/quicer.hrl"). -else. @@ -40,37 +41,50 @@ new_stream/3 ]). --type cb_state() :: map(). --type cb_ret() :: ok. - --spec init(map() | list()) -> cb_state(). +-type cb_state() :: #{ + ctrl_pid := undefined | pid(), + conn := undefined | quicer:conneciton_hanlder(), + stream_opts := map(), + is_resumed => boolean(), + _ => _ +}. +-type cb_ret() :: quicer_lib:cb_ret(). +-spec init(map() | list()) -> {ok, cb_state()}. init(ConnOpts) when is_list(ConnOpts) -> init(maps:from_list(ConnOpts)); init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> init(S#{stream_opts := maps:from_list(SOpts)}); init(ConnOpts) when is_map(ConnOpts) -> - {ok, ConnOpts}. + {ok, init_cb_state(ConnOpts)}. -spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) -> - {ok, cb_state()} | {error, any()}. -closed(_Conn, #{is_peer_acked := true}, S) -> - {stop, normal, S}; -closed(_Conn, #{is_peer_acked := false}, S) -> - {stop, abnorml, S}. + {stop, normal, cb_state()}. +closed(_Conn, #{is_peer_acked := _} = Prop, S) -> + ?SLOG(debug, Prop), + {stop, normal, S}. -spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. -new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) -> +new_conn( + Conn, + #{version := _Vsn} = ConnInfo, + #{zone := Zone, conn := undefined, ctrl_pid := undefined} = S +) -> process_flag(trap_exit, true), + ?SLOG(debug, ConnInfo), case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of false -> - {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, {self(), Conn}, S), + {ok, Pid} = emqx_connection:start_link( + emqx_quic_stream, + {self(), Conn, maps:without([crypto_buffer], ConnInfo)}, + S + ), receive {Pid, stream_acceptor_ready} -> ok = quicer:async_handshake(Conn), - {ok, S#{conn => Conn}}; - {'EXIT', Pid, _Reason} -> + {ok, S#{conn := Conn, ctrl_pid := Pid}}; + {'EXIT', _Pid, _Reason} -> {error, stream_accept_error} end; true -> @@ -80,10 +94,12 @@ new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) -> -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. -connected(Conn, _Props, #{slow_start := false} = S) -> +connected(Conn, Props, #{slow_start := false} = S) -> + ?SLOG(debug, Props), {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), {ok, S}; -connected(_Conn, _Props, S) -> +connected(_Conn, Props, S) -> + ?SLOG(debug, Props), {ok, S}. -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret(). @@ -92,10 +108,11 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when -> ResumeFun(Conn, Data, S); resumed(_Conn, _Data, S) -> - {ok, S}. + {ok, S#{is_resumed := true}}. -spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret(). nst_received(_Conn, _Data, S) -> + %% As server we should not recv NST! {stop, no_nst_for_server, S}. -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret(). @@ -116,14 +133,17 @@ new_stream( Other -> Other end. + -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). shutdown(Conn, _ErrorCode, S) -> + %% @TODO check spec what to do with the ErrorCode? quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) -> cb_ret(). transport_shutdown(_C, _DownInfo, S) -> + %% @TODO some counter {ok, S}. -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret(). @@ -140,14 +160,21 @@ local_address_changed(_C, _NewAddr, S) -> {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()}, cb_state() ) -> cb_ret(). -streams_available(_C, {_BidirCnt, _UnidirCnt}, S) -> - {ok, S}. +streams_available(_C, {BidirCnt, UnidirCnt}, S) -> + {ok, S#{ + peer_bidi_stream_count => BidirCnt, + peer_unidi_stream_count => UnidirCnt + }}. -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). +%% @TODO this is not going to get triggered. %% for https://github.com/microsoft/msquic/issues/3120 peer_needs_streams(_C, undefined, S) -> {ok, S}. +%%% +%%% Internals +%%% -spec is_zone_olp_enabled(emqx_types:zone()) -> boolean(). is_zone_olp_enabled(Zone) -> case emqx_config:get_zone_conf(Zone, [overload_protection]) of @@ -156,3 +183,10 @@ is_zone_olp_enabled(Zone) -> _ -> false end. + +-spec init_cb_state(map()) -> cb_state(). +init_cb_state(Map) -> + Map#{ + ctrl_pid => undefined, + conn => undefined + }. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index fe6ff692c..d9c080c0d 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -17,6 +17,8 @@ %% MQTT/QUIC Stream -module(emqx_quic_stream). +-behaviour(quicer_stream). + %% emqx transport Callbacks -export([ type/1, @@ -32,13 +34,71 @@ peercert/1 ]). -wait({ConnOwner, Conn}) -> +-include("logger.hrl"). +-ifndef(BUILD_WITHOUT_QUIC). +-include_lib("quicer/include/quicer.hrl"). +-else. +%% STREAM SHUTDOWN FLAGS +-define(QUIC_STREAM_SHUTDOWN_FLAG_NONE, 0). +% Cleanly closes the send path. +-define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 1). +% Abruptly closes the send path. +-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 2). +% Abruptly closes the receive path. +-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 4). +% Abruptly closes both send and receive paths. +-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 6). +-define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE, 8). +-endif. + +-type cb_ret() :: gen_statem:event_handler_result(). +-type cb_data() :: emqtt_quic:cb_data(). +-type connection_handle() :: quicer:connection_handle(). +-type stream_handle() :: quicer:stream_handle(). + +-export([ + init_handoff/4, + new_stream/3, + start_completed/3, + send_complete/3, + peer_send_shutdown/3, + peer_send_aborted/3, + peer_receive_aborted/3, + send_shutdown_complete/3, + stream_closed/3, + peer_accepted/3, + passive/3, + handle_call/4 +]). + +-export_type([socket/0]). + +-opaque socket() :: {quic, connection_handle(), stream_handle(), socket_info()}. + +-type socket_info() :: #{ + is_orphan => boolean(), + ctrl_stream_start_flags => quicer:stream_open_flags(), + %% quicer:new_conn_props + _ => _ +}. + +-spec wait({pid(), quicer:connection_handle(), socket_info()}) -> + {ok, socket()} | {error, enotconn}. +wait({ConnOwner, Conn, ConnInfo}) -> {ok, Conn} = quicer:async_accept_stream(Conn, []), ConnOwner ! {self(), stream_acceptor_ready}, receive - %% from msquic - {quic, new_stream, Stream, _Props} -> - {ok, {quic, Conn, Stream}}; + %% New incoming stream, this is a *ctrl* stream + {quic, new_stream, Stream, #{is_orphan := IsOrphan, flags := StartFlags}} -> + SocketInfo = ConnInfo#{ + is_orphan => IsOrphan, + ctrl_stream_start_flags => StartFlags + }, + {ok, socket(Conn, Stream, SocketInfo)}; + %% connection closed event for stream acceptor + {quic, closed, undefined, undefined} -> + {error, enotconn}; + %% Connection owner process down {'EXIT', ConnOwner, _Reason} -> {error, enotconn} end. @@ -46,17 +106,17 @@ wait({ConnOwner, Conn}) -> type(_) -> quic. -peername({quic, Conn, _Stream}) -> +peername({quic, Conn, _Stream, _Info}) -> quicer:peername(Conn). -sockname({quic, Conn, _Stream}) -> +sockname({quic, Conn, _Stream, _Info}) -> quicer:sockname(Conn). peercert(_S) -> %% @todo but unsupported by msquic nossl. -getstat({quic, Conn, _Stream}, Stats) -> +getstat({quic, Conn, _Stream, _Info}, Stats) -> case quicer:getstat(Conn, Stats) of {error, _} -> {error, closed}; Res -> Res @@ -84,7 +144,7 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. -fast_close({quic, _Conn, Stream}) -> +fast_close({quic, _Conn, Stream, _Info}) -> %% Flush send buffer, gracefully shutdown quicer:async_shutdown_stream(Stream), ok. @@ -102,8 +162,131 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> Result end. -async_send({quic, _Conn, Stream}, Data, _Options) -> +async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> case quicer:send(Stream, Data) of {ok, _Len} -> ok; Other -> Other end. + +%%% +%%% quicer stream callbacks +%%% + +-spec init_handoff(stream_handle(), #{}, quicer:connection_handle(), #{}) -> cb_ret(). +init_handoff(_Stream, _StreamOpts, _Conn, _Flags) -> + %% stream owner already set while starts. + {stop, unimpl}. + +-spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret(). +new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) -> + {stop, unimpl}. + +-spec peer_accepted(stream_handle(), undefined, cb_data()) -> cb_ret(). +peer_accepted(_Stream, undefined, S) -> + %% We just ignore it + {ok, S}. + +-spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> + %% we abort send with same reason + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}; +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) -> + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}. + +-spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). +peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> + %% we abort receive with same reason + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + {ok, S}; +peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) -> + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + {ok, S}. + +-spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret(). +peer_send_shutdown(Stream, undefined, S) -> + ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0), + {ok, S}. + +-spec send_complete(stream_handle(), boolean(), cb_data()) -> cb_ret(). +send_complete(_Stream, false, S) -> + {ok, S}; +send_complete(_Stream, true = _IsCancelled, S) -> + ?SLOG(error, #{message => "send cancelled"}), + {ok, S}. + +-spec send_shutdown_complete(stream_handle(), boolean(), cb_data()) -> cb_ret(). +send_shutdown_complete(_Stream, _IsGraceful, S) -> + {ok, S}. + +-spec start_completed(stream_handle(), quicer:stream_start_completed_props(), cb_data()) -> + cb_ret(). +start_completed(_Stream, #{status := success, stream_id := StreamId} = Prop, S) -> + ?SLOG(debug, Prop), + {ok, S#{stream_id => StreamId}}; +start_completed(_Stream, #{status := stream_limit_reached, stream_id := _StreamId} = Prop, _S) -> + ?SLOG(error, #{message => start_completed}, Prop), + {stop, stream_limit_reached}; +start_completed(_Stream, #{status := Other} = Prop, S) -> + ?SLOG(error, Prop), + %% or we could retry? + {stop, {start_fail, Other}, S}. + +%% Local stream, Unidir +%% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data()) +%% -> cb_ret(). +%% handle_stream_data(Stream, Bin, Flags, #{ is_local := true +%% , parse_state := PS} = S) -> +%% ?SLOG(debug, #{data => Bin}, Flags), +%% case parse(Bin, PS, []) of +%% {keep_state, NewPS, Packets} -> +%% quicer:setopt(Stream, active, once), +%% {keep_state, S#{parse_state := NewPS}, +%% [{next_event, cast, P } || P <- lists:reverse(Packets)]}; +%% {stop, _} = Stop -> +%% Stop +%% end; +%% %% Remote stream +%% handle_stream_data(_Stream, _Bin, _Flags, +%% #{is_local := false, is_unidir := true, conn := _Conn} = _S) -> +%% {stop, unimpl}. + +-spec passive(stream_handle(), undefined, cb_data()) -> cb_ret(). +passive(_Stream, undefined, _S) -> + {stop, unimpl}. + +-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret(). +stream_closed( + _Stream, + #{ + is_conn_shutdown := IsConnShutdown, + is_app_closing := IsAppClosing, + is_shutdown_by_app := IsAppShutdown, + is_closed_remotely := IsRemote, + status := Status, + error := Code + }, + S +) when + is_boolean(IsConnShutdown) andalso + is_boolean(IsAppClosing) andalso + is_boolean(IsAppShutdown) andalso + is_boolean(IsRemote) andalso + is_atom(Status) andalso + is_integer(Code) +-> + %% @TODO for now we fake a sock_closed for + %% emqx_connection:process_msg to append + %% a msg to be processed + {ok, {sock_closed, Status}, S}. + +handle_call(_Stream, _Request, _Opts, S) -> + {error, unimpl, S}. + +%%% +%%% Internals +%%% +-spec socket(connection_handle(), stream_handle(), socket_info()) -> socket(). +socket(Conn, CtrlStream, Info) when is_map(Info) -> + {quic, Conn, CtrlStream, Info}. From 9f696928b6174bb17f4505ee8a1f4e72df028dcb Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 25 Nov 2022 15:15:52 +0100 Subject: [PATCH 016/135] feat(quic): multi streams --- apps/emqx/src/emqx_channel.erl | 1 + apps/emqx/src/emqx_connection.erl | 37 +- apps/emqx/src/emqx_quic_connection.erl | 139 +++++- apps/emqx/src/emqx_quic_data_stream.erl | 466 ++++++++++++++++++ apps/emqx/src/emqx_quic_stream.erl | 43 +- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 22 +- .../test/emqx_quic_multistreams_SUITE.erl | 190 +++++++ 7 files changed, 848 insertions(+), 50 deletions(-) create mode 100644 apps/emqx/src/emqx_quic_data_stream.erl create mode 100644 apps/emqx/test/emqx_quic_multistreams_SUITE.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index e82adc786..a12df9c64 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1136,6 +1136,7 @@ do_deliver(Publishes, Channel) when is_list(Publishes) -> {Packets, NChannel} = lists:foldl( fun(Publish, {Acc, Chann}) -> + %% @FIXME perf: list append with copy left list {Packets, NChann} = do_deliver(Publish, Chann), {Packets ++ Acc, NChann} end, diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 1c8b85808..980c41010 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -14,7 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/TCP|TLS Connection|QUIC Stream +%% This module interacts with the transport layer of MQTT +%% Transport: +%% - TCP connection +%% - TCP/TLS connection +%% - WebSocket +%% - QUIC Stream -module(emqx_connection). -include("emqx.hrl"). @@ -111,7 +116,13 @@ limiter_buffer :: queue:queue(pending_req()), %% limiter timers - limiter_timer :: undefined | reference() + limiter_timer :: undefined | reference(), + + %% QUIC conn pid if is a pid + quic_conn_pid :: maybe(pid()), + + %% QUIC control stream callback state + quic_ctrl_state :: map() }). -record(retry, { @@ -194,7 +205,7 @@ {ok, pid()}; ( emqx_quic_stream, - {ConnOwner :: pid(), quicer:connection_handler(), quicer:new_conn_props()}, + {ConnOwner :: pid(), quicer:connection_handle(), quicer:new_conn_props()}, emqx_quic_connection:cb_state() ) -> {ok, pid()}. @@ -334,6 +345,7 @@ init_state( }, ParseState = emqx_frame:initial_parse_state(FrameOpts), Serialize = emqx_frame:serialize_opts(), + %% Init Channel Channel = emqx_channel:init(ConnInfo, Opts), GcState = case emqx_config:get_zone_conf(Zone, [force_gc]) of @@ -364,7 +376,10 @@ init_state( zone = Zone, listener = Listener, limiter_buffer = queue:new(), - limiter_timer = undefined + limiter_timer = undefined, + %% for quic streams to inherit + quic_conn_pid = maps:get(conn_pid, Opts, undefined), + quic_ctrl_state = #{} }. run_loop( @@ -600,9 +615,20 @@ handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> handle_msg({connack, ConnAck}, State) -> handle_outgoing(ConnAck, State); handle_msg({close, Reason}, State) -> + %% @FIXME here it could be close due to appl error. ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}), handle_info({sock_closed, Reason}, close_socket(State)); -handle_msg({event, connected}, State = #state{channel = Channel}) -> +handle_msg( + {event, connected}, + State = #state{ + channel = Channel, + serialize = Serialize, + parse_state = PS, + quic_conn_pid = QuicConnPid + } +) -> + QuicConnPid =/= undefined andalso + emqx_quic_connection:activate_data_streams(QuicConnPid, {PS, Serialize, Channel}), ClientId = emqx_channel:info(clientid, Channel), emqx_cm:insert_channel_info(ClientId, info(State), stats(State)); handle_msg({event, disconnected}, State = #state{channel = Channel}) -> @@ -876,6 +902,7 @@ send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) ok; Error = {error, _Reason} -> %% Send an inet_reply to postpone handling the error + %% @FIXME: why not just return error? self() ! {inet_reply, Socket, Error}, ok end. diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index a5af3d4b3..de7776429 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -14,6 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% @doc impl. the quic connection owner process. -module(emqx_quic_connection). -include("logger.hrl"). @@ -41,15 +42,46 @@ new_stream/3 ]). +-export([activate_data_streams/2]). + +-export([ + handle_call/3, + handle_info/2 +]). + -type cb_state() :: #{ + %% connecion owner pid + conn_pid := pid(), + %% Pid of ctrl stream ctrl_pid := undefined | pid(), + %% quic connecion handle conn := undefined | quicer:conneciton_hanlder(), + %% streams that handoff from this process, excluding control stream + %% these streams could die/closed without effecting the connecion/session. + + %@TODO type? + streams := [{pid(), quicer:stream_handle()}], + %% New stream opts stream_opts := map(), + %% If conneciton is resumed from session ticket is_resumed => boolean(), + %% mqtt message serializer config + serialize => undefined, _ => _ }. -type cb_ret() :: quicer_lib:cb_ret(). +%% @doc Data streams initializions are started in parallel with control streams, data streams are blocked +%% for the activation from control stream after it is accepted as a legit conneciton. +%% For security, the initial number of allowed data streams from client should be limited by +%% 'peer_bidi_stream_count` & 'peer_unidi_stream_count` +-spec activate_data_streams(pid(), { + emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel() +}) -> ok. +activate_data_streams(ConnOwner, {PS, Serialize, Channel}) -> + gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity). + +%% @doc conneciton owner init callback -spec init(map() | list()) -> {ok, cb_state()}. init(ConnOpts) when is_list(ConnOpts) -> init(maps:from_list(ConnOpts)); @@ -64,6 +96,7 @@ closed(_Conn, #{is_peer_acked := _} = Prop, S) -> ?SLOG(debug, Prop), {stop, normal, S}. +%% @doc handle the new incoming connecion as the connecion acceptor. -spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. new_conn( @@ -75,15 +108,17 @@ new_conn( ?SLOG(debug, ConnInfo), case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of false -> - {ok, Pid} = emqx_connection:start_link( + %% Start control stream process + StartOption = S, + {ok, CtrlPid} = emqx_connection:start_link( emqx_quic_stream, {self(), Conn, maps:without([crypto_buffer], ConnInfo)}, - S + StartOption ), receive - {Pid, stream_acceptor_ready} -> + {CtrlPid, stream_acceptor_ready} -> ok = quicer:async_handshake(Conn), - {ok, S#{conn := Conn, ctrl_pid := Pid}}; + {ok, S#{conn := Conn, ctrl_pid := CtrlPid}}; {'EXIT', _Pid, _Reason} -> {error, stream_accept_error} end; @@ -92,6 +127,7 @@ new_conn( {error, overloaded} end. +%% @doc callback when connection is connected. -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. connected(Conn, Props, #{slow_start := false} = S) -> @@ -102,6 +138,7 @@ connected(_Conn, Props, S) -> ?SLOG(debug, Props), {ok, S}. +%% @doc callback when connection is resumed from 0-RTT -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret(). resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when is_function(ResumeFun) @@ -110,51 +147,77 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when resumed(_Conn, _Data, S) -> {ok, S#{is_resumed := true}}. +%% @doc callback for receiving nst, should never happen on server. -spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret(). nst_received(_Conn, _Data, S) -> %% As server we should not recv NST! {stop, no_nst_for_server, S}. +%% @doc callback for handling orphan data streams +%% depends on the connecion state and control stream state. -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret(). new_stream( Stream, - #{is_orphan := true} = Props, + #{is_orphan := true, flags := _Flags} = Props, #{ conn := Conn, streams := Streams, - stream_opts := SOpts - } = CBState + stream_opts := SOpts, + zone := Zone, + limiter := Limiter, + parse_state := PS, + channel := Channel, + serialize := Serialize + } = S ) -> - %% Spawn new stream - case quicer_stream:start_link(emqx_quic_stream, Stream, Conn, SOpts, Props) of - {ok, StreamOwner} -> - quicer_connection:handoff_stream(Stream, StreamOwner), - {ok, CBState#{streams := [{StreamOwner, Stream} | Streams]}}; - Other -> - Other - end. + %% Cherry pick options for data streams + SOpts1 = SOpts#{ + is_local => false, + zone => Zone, + % unused + limiter => Limiter, + parse_state => PS, + channel => Channel, + serialize => Serialize + }, + {ok, NewStreamOwner} = quicer_stream:start_link( + emqx_quic_data_stream, + Stream, + Conn, + SOpts1, + Props + ), + quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}), + %% @TODO keep them in ``inactive_streams' + {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}. +%% @doc callback for handling for remote connecion shutdown. -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). shutdown(Conn, _ErrorCode, S) -> - %% @TODO check spec what to do with the ErrorCode? + %% @TODO check spec what to set for the ErrorCode? quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. +%% @doc callback for handling for transport error, such as idle timeout -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) -> cb_ret(). transport_shutdown(_C, _DownInfo, S) -> %% @TODO some counter {ok, S}. +%% @doc callback for handling for peer addr changed. -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret(). peer_address_changed(_C, _NewAddr, S) -> + %% @TODO update session info? {ok, S}. +%% @doc callback for handling local addr change, currently unused -spec local_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state()) -> cb_ret(). local_address_changed(_C, _NewAddr, S) -> {ok, S}. +%% @doc callback for handling remote stream limit updates -spec streams_available( quicer:connection_handle(), {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()}, @@ -166,12 +229,43 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) -> peer_unidi_stream_count => UnidirCnt }}. --spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). -%% @TODO this is not going to get triggered. +%% @doc callback for handling request when remote wants for more streams +%% should cope with rate limiting +%% @TODO this is not going to get triggered in current version %% for https://github.com/microsoft/msquic/issues/3120 +-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). peer_needs_streams(_C, undefined, S) -> {ok, S}. +%% @doc handle API calls +handle_call( + {activate_data_streams, {PS, Serialize, Channel} = ActivateData}, + _From, + #{streams := Streams} = S +) -> + [emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) || {OwnerPid, _Stream} <- Streams], + {reply, ok, S#{ + %streams := [], %% @FIXME what ?????? + channel := Channel, + serialize := Serialize, + parse_state := PS + }}; +handle_call(_Req, _From, S) -> + {reply, {error, unimpl}, S}. + +%% @doc handle DOWN messages from streams. +%% @TODO handle DOWN from supervisor? +handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when + Reason =:= normal orelse + Reason =:= {shutdown, protocol_error} +-> + case proplists:is_defined(Pid, Streams) of + true -> + {ok, S}; + false -> + {stop, unknown_pid_down, S} + end. + %%% %%% Internals %%% @@ -185,8 +279,13 @@ is_zone_olp_enabled(Zone) -> end. -spec init_cb_state(map()) -> cb_state(). -init_cb_state(Map) -> +init_cb_state(#{zone := _Zone} = Map) -> Map#{ + conn_pid => self(), ctrl_pid => undefined, - conn => undefined + conn => undefined, + streams => [], + parse_state => undefined, + channel => undefined, + serialize => undefined }. diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl new file mode 100644 index 000000000..72f0e913f --- /dev/null +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -0,0 +1,466 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% +%% @doc QUIC data stream +%% Following the behaviour of emqx_connection: +%% The MQTT packets and their side effects are handled *atomically*. +%% + +-module(emqx_quic_data_stream). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("quicer/include/quicer.hrl"). +-include("emqx_mqtt.hrl"). +-include("logger.hrl"). +-behaviour(quicer_stream). + +%% Connection Callbacks +-export([ + init_handoff/4, + post_handoff/3, + new_stream/3, + start_completed/3, + send_complete/3, + peer_send_shutdown/3, + peer_send_aborted/3, + peer_receive_aborted/3, + send_shutdown_complete/3, + stream_closed/3, + peer_accepted/3, + passive/3 +]). + +-export([handle_stream_data/4]). + +-export([activate_data/2]). + +-export([ + handle_call/3, + handle_info/2, + handle_continue/2 +]). + +%% +%% @doc Activate the data handling. +%% Data handling is disabled before control stream allows the data processing. +-spec activate_data(pid(), { + emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel() +}) -> ok. +activate_data(StreamPid, {PS, Serialize, Channel}) -> + gen_server:call(StreamPid, {activate, {PS, Serialize, Channel}}, infinity). + +%% +%% @doc Handoff from previous owner, mostly from the connection owner. +%% @TODO parse_state doesn't look necessary since we have it in post_handoff +%% @TODO -spec +init_handoff( + Stream, + #{parse_state := PS} = _StreamOpts, + Connection, + #{is_orphan := true, flags := Flags} +) -> + {ok, init_state(Stream, Connection, Flags, PS)}. + +%% +%% @doc Post handoff data stream +%% +%% @TODO -spec +%% +post_handoff(Stream, {PS, Serialize, Channel}, S) -> + ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}), + quicer:setopt(Stream, active, true), + {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. + +%% +%% @doc when this proc is assigned to the owner of new stream +%% +new_stream(Stream, #{flags := Flags}, Connection) -> + {ok, init_state(Stream, Connection, Flags)}. + +%% +%% @doc for local initiated stream +%% +peer_accepted(_Stream, _Flags, S) -> + %% we just ignore it + {ok, S}. + +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> + %% we abort send with same reason + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}; +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) -> + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}. + +peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> + %% we abort receive with same reason + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + {ok, S}; +peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) -> + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + {ok, S}. + +peer_send_shutdown(Stream, _Flags, S) -> + ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0), + {ok, S}. + +send_complete(_Stream, false, S) -> + {ok, S}; +send_complete(_Stream, true = _IsCanceled, S) -> + {ok, S}. + +send_shutdown_complete(_Stream, _Flags, S) -> + {ok, S}. + +start_completed(_Stream, #{status := success, stream_id := StreamId}, S) -> + {ok, S#{stream_id => StreamId}}; +start_completed(_Stream, #{status := Other}, S) -> + %% or we could retry + {stop, {start_fail, Other}, S}. + +handle_stream_data( + Stream, + Bin, + _Flags, + #{ + is_unidir := false, + channel := undefined, + data_queue := Queue, + stream := Stream + } = State +) when is_binary(Bin) -> + {ok, State#{data_queue := [Bin | Queue]}}; +handle_stream_data( + _Stream, + Bin, + _Flags, + #{ + is_unidir := false, + channel := Channel, + parse_state := PS, + data_queue := QueuedData, + task_queue := TQ + } = State +) when + Channel =/= undefined +-> + {MQTTPackets, NewPS} = parse_incoming(list_to_binary(lists:reverse([Bin | QueuedData])), PS), + NewTQ = lists:foldl( + fun(Item, Acc) -> + queue:in(Item, Acc) + end, + TQ, + [{incoming, P} || P <- lists:reverse(MQTTPackets)] + ), + {{continue, handle_appl_msg}, State#{parse_state := NewPS, task_queue := NewTQ}}. + +%% Reserved for unidi streams +%% handle_stream_data(Stream, Bin, _Flags, #{is_unidir := true, peer_stream := PeerStream, conn := Conn} = State) -> +%% case PeerStream of +%% undefined -> +%% {ok, StreamProc} = quicer_stream:start_link(?MODULE, Conn, +%% [ {open_flag, ?QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL} +%% , {is_local, true} +%% ]), +%% {ok, _} = quicer_stream:send(StreamProc, Bin), +%% {ok, State#{peer_stream := StreamProc}}; +%% StreamProc when is_pid(StreamProc) -> +%% {ok, _} = quicer_stream:send(StreamProc, Bin), +%% {ok, State} +%% end. + +passive(_Stream, undefined, S) -> + {ok, S}. + +stream_closed( + _Stream, + #{ + is_conn_shutdown := IsConnShutdown, + is_app_closing := IsAppClosing, + is_shutdown_by_app := IsAppShutdown, + is_closed_remotely := IsRemote, + status := Status, + error := Code + }, + S +) when + is_boolean(IsConnShutdown) andalso + is_boolean(IsAppClosing) andalso + is_boolean(IsAppShutdown) andalso + is_boolean(IsRemote) andalso + is_atom(Status) andalso + is_integer(Code) +-> + {stop, normal, S}. + +handle_call(Call, _From, S) -> + do_handle_call(Call, S). + +handle_continue(handle_appl_msg, #{task_queue := Q} = S) -> + case queue:out(Q) of + {{value, Item}, Q2} -> + do_handle_appl_msg(Item, S#{task_queue := Q2}); + {empty, Q} -> + {ok, S} + end. + +do_handle_appl_msg( + {outgoing, Packets}, + #{ + channel := Channel, + stream := _Stream, + serialize := _Serialize + } = S +) when + Channel =/= undefined +-> + case handle_outgoing(Packets, S) of + {ok, Size} -> + ok = emqx_metrics:inc('bytes.sent', Size), + {{continue, handle_appl_msg}, S}; + {error, E1, E2} -> + {stop, {E1, E2}, S}; + {error, E} -> + {stop, E, S} + end; +do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = S) when + Channel =/= undefined +-> + with_channel(handle_in, [Packet], S); +do_handle_appl_msg({close, Reason}, S) -> + %% @TODO shall we abort shutdown or graceful shutdown? + with_channel(handle_info, [{sock_closed, Reason}], S); +do_handle_appl_msg({event, updated}, S) -> + %% Data stream don't care about connection state changes. + {{continue, handle_appl_msg}, S}. + +handle_info(Deliver = {deliver, _, _}, S) -> + Delivers = [Deliver], + with_channel(handle_deliver, [Delivers], S). + +with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when + Channel =/= undefined +-> + case apply(emqx_channel, Fun, Args ++ [Channel]) of + ok -> + {{continue, handle_appl_msg}, S}; + {ok, Msgs, NewChannel} when is_list(Msgs) -> + {{continue, handle_appl_msg}, S#{ + task_queue := queue:join(Q, queue:from_list(Msgs)), + channel := NewChannel + }}; + {ok, Msg, NewChannel} when is_record(Msg, mqtt_packet) -> + {{continue, handle_appl_msg}, S#{ + task_queue := queue:in({outgoing, Msg}, Q), channel := NewChannel + }}; + %% @FIXME WTH? + {ok, {outgoing, _} = Msg, NewChannel} -> + {{continue, handle_appl_msg}, S#{task_queue := queue:in(Msg, Q), channel := NewChannel}}; + {ok, NewChannel} -> + {{continue, handle_appl_msg}, S#{channel := NewChannel}}; + %% @TODO optimisation for shutdown wrap + {shutdown, Reason, NewChannel} -> + {stop, {shutdown, Reason}, S#{channel := NewChannel}}; + {shutdown, Reason, Msgs, NewChannel} when is_list(Msgs) -> + %% @TODO handle outgoing? + {stop, {shutdown, Reason}, S#{ + channel := NewChannel, + task_queue := queue:join(Q, queue:from_list(Msgs)) + }}; + {shutdown, Reason, Msg, NewChannel} -> + {stop, {shutdown, Reason}, S#{ + channel := NewChannel, + task_queue := queue:in(Msg, Q) + }} + end. + +%%% Internals +handle_outgoing(#mqtt_packet{} = P, S) -> + handle_outgoing([P], S); +handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir := false}) when + is_list(Packets) +-> + OutBin = [serialize_packet(P, Serialize) || P <- filter_disallowed_out(Packets)], + %% @TODO in which case shall we use sync send? + Res = quicer:async_send(Stream, OutBin), + ?TRACE("MQTT", "mqtt_packet_sent", #{packets => Packets}), + [ok = inc_outgoing_stats(P) || P <- Packets], + Res. + +serialize_packet(Packet, Serialize) -> + try emqx_frame:serialize_pkt(Packet, Serialize) of + <<>> -> + ?SLOG(warning, #{ + msg => "packet_is_discarded", + reason => "frame_is_too_large", + packet => emqx_packet:format(Packet, hidden) + }), + ok = emqx_metrics:inc('delivery.dropped.too_large'), + ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), + <<>>; + Data -> + Data + catch + %% Maybe Never happen. + throw:{?FRAME_SERIALIZE_ERROR, Reason} -> + ?SLOG(info, #{ + reason => Reason, + input_packet => Packet + }), + erlang:error({?FRAME_SERIALIZE_ERROR, Reason}); + error:Reason:Stacktrace -> + ?SLOG(error, #{ + input_packet => Packet, + exception => Reason, + stacktrace => Stacktrace + }), + erlang:error(?FRAME_SERIALIZE_ERROR) + end. + +-spec init_state( + quicer:stream_handle(), + quicer:connection_handle(), + quicer:new_stream_props() +) -> + % @TODO + map(). +init_state(Stream, Connection, OpenFlags) -> + init_state(Stream, Connection, OpenFlags, undefined). + +init_state(Stream, Connection, OpenFlags, PS) -> + %% quic stream handle + #{ + stream => Stream, + %% quic connection handle + conn => Connection, + %% if it is QUIC unidi stream + is_unidir => quicer:is_unidirectional(OpenFlags), + %% Frame Parse State + parse_state => PS, + %% Peer Stream handle in a pair for type unidir only + peer_stream => undefined, + %% if the stream is locally initiated. + is_local => false, + %% queue binary data when is NOT connected, in reversed order. + data_queue => [], + %% Channel from connection + %% `undefined' means the connection is not connected. + channel => undefined, + %% serialize opts for connection + serialize => undefined, + %% Current working queue + task_queue => queue:new() + }. + +-spec do_handle_call(term(), quicer_stream:cb_state()) -> quicer_stream:cb_ret(). +do_handle_call( + {activate, {PS, Serialize, Channel}}, + #{ + channel := undefined, + stream := Stream, + serialize := undefined + } = S +) -> + NewS = S#{channel := Channel, serialize := Serialize, parse_state := PS}, + %% We use quic protocol for flow control, and we don't check return val + case quicer:setopt(Stream, active, true) of + ok -> + {ok, NewS}; + {error, E} -> + ?SLOG(error, #{msg => "set stream active failed", error => E}), + {stop, E, NewS} + end; +do_handle_call(_Call, S) -> + {reply, {error, unimpl}, S}. + +%% @doc return reserved order of Packets +parse_incoming(Data, PS) -> + try + do_parse_incoming(Data, [], PS) + catch + throw:{?FRAME_PARSE_ERROR, Reason} -> + ?SLOG(info, #{ + reason => Reason, + input_bytes => Data + }), + {[{frame_error, Reason}], PS}; + error:Reason:Stacktrace -> + ?SLOG(error, #{ + input_bytes => Data, + reason => Reason, + stacktrace => Stacktrace + }), + {[{frame_error, Reason}], PS} + end. + +do_parse_incoming(<<>>, Packets, ParseState) -> + {Packets, ParseState}; +do_parse_incoming(Data, Packets, ParseState) -> + case emqx_frame:parse(Data, ParseState) of + {more, NParseState} -> + {Packets, NParseState}; + {ok, Packet, Rest, NParseState} -> + do_parse_incoming(Rest, [Packet | Packets], NParseState) + end. + +%% followings are copied from emqx_connection +-compile({inline, [inc_outgoing_stats/1]}). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); +inc_outgoing_stats(Packet = ?PACKET(Type)) -> + inc_counter(send_pkt, 1), + case Type of + ?PUBLISH -> + inc_counter(send_msg, 1), + inc_counter(outgoing_pubs, 1), + inc_qos_stats(send_msg, Packet); + _ -> + ok + end, + emqx_metrics:inc_sent(Packet). + +inc_counter(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. + +inc_qos_stats(Type, Packet) -> + case inc_qos_stats_key(Type, emqx_packet:qos(Packet)) of + undefined -> + ignore; + Key -> + inc_counter(Key, 1) + end. + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'; +%% for bad qos +inc_qos_stats_key(_, _) -> undefined. + +filter_disallowed_out(Packets) -> + lists:filter(fun is_datastream_out_pkt/1, Packets). + +is_datastream_out_pkt(#mqtt_packet{header = #mqtt_packet_header{type = Type}}) when + Type > 2 andalso Type < 12 +-> + true; +is_datastream_out_pkt(_) -> + false. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index d9c080c0d..70b01e643 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -78,17 +78,24 @@ -type socket_info() :: #{ is_orphan => boolean(), ctrl_stream_start_flags => quicer:stream_open_flags(), - %% quicer:new_conn_props + %% and quicer:new_conn_props() _ => _ }. --spec wait({pid(), quicer:connection_handle(), socket_info()}) -> - {ok, socket()} | {error, enotconn}. +%% for accepting +-spec wait + ({pid(), connection_handle(), socket_info()}) -> + {ok, socket()} | {error, enotconn}; + %% For handover + ({pid(), connection_handle(), stream_handle(), socket_info()}) -> + {ok, socket()} | {error, any()}. + +%%% For Accepting New Remote Stream wait({ConnOwner, Conn, ConnInfo}) -> {ok, Conn} = quicer:async_accept_stream(Conn, []), ConnOwner ! {self(), stream_acceptor_ready}, receive - %% New incoming stream, this is a *ctrl* stream + %% New incoming stream, this is a *control* stream {quic, new_stream, Stream, #{is_orphan := IsOrphan, flags := StartFlags}} -> SocketInfo = ConnInfo#{ is_orphan => IsOrphan, @@ -101,6 +108,14 @@ wait({ConnOwner, Conn, ConnInfo}) -> %% Connection owner process down {'EXIT', ConnOwner, _Reason} -> {error, enotconn} + end; +%% For ownership handover +wait({PrevOwner, Conn, Stream, SocketInfo}) -> + case quicer:wait_for_handoff(PrevOwner, Stream) of + ok -> + {ok, socket(Conn, Stream, SocketInfo)}; + owner_down -> + {error, owner_down} end. type(_) -> @@ -144,9 +159,10 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. -fast_close({quic, _Conn, Stream, _Info}) -> - %% Flush send buffer, gracefully shutdown - quicer:async_shutdown_stream(Stream), +fast_close({quic, Conn, _Stream, _Info}) -> + %% Since we shutdown the control stream, we shutdown the connection as well + %% @TODO supply some App Error Code + quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok. -spec ensure_ok_or_exit(atom(), list(term())) -> term(). @@ -187,21 +203,14 @@ peer_accepted(_Stream, undefined, S) -> {ok, S}. -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). -peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> - %% we abort send with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), - {ok, S}; -peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) -> +peer_receive_aborted(Stream, ErrorCode, S) -> quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). -peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> +peer_send_aborted(Stream, ErrorCode, S) -> %% we abort receive with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), - {ok, S}; -peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) -> - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret(). diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 07299bd42..0b493dff6 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -65,6 +65,7 @@ init_per_group(quic, Config) -> UdpPort = 1884, emqx_common_test_helpers:start_apps([]), emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), + emqx_logger:set_log_level(debug), [{port, UdpPort}, {conn_fun, quic_connect} | Config]; init_per_group(_, Config) -> emqx_common_test_helpers:stop_apps([]), @@ -78,14 +79,19 @@ end_per_group(_Group, _Config) -> init_per_suite(Config) -> %% Start Apps - %% dbg:tracer(process, {fun dbg:dhandler/2,group_leader()}), - %% dbg:p(all,c), - %% dbg:tp(emqx_quic_connection,cx), - %% dbg:tp(emqx_quic_stream,cx), - %% dbg:tp(emqtt_quic,cx), - %% dbg:tp(emqtt,cx), - %% dbg:tp(emqtt_quic_stream,cx), - %% dbg:tp(emqtt_quic_connection,cx), + dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + dbg:p(all, c), + dbg:tp(emqx_quic_connection, cx), + dbg:tp(quicer_connection, cx), + %% dbg:tp(emqx_quic_stream, cx), + %% dbg:tp(emqtt_quic, cx), + %% dbg:tp(emqtt, cx), + %% dbg:tp(emqtt_quic_stream, cx), + %% dbg:tp(emqtt_quic_connection, cx), + %% dbg:tp(emqx_cm, open_session, cx), + %% dbg:tpl(emqx_cm, lookup_channels, cx), + %% dbg:tpl(emqx_cm, register_channel, cx), + %% dbg:tpl(emqx_cm, unregister_channel, cx), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), Config. diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl new file mode 100644 index 000000000..bb19092f7 --- /dev/null +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -0,0 +1,190 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_quic_multistreams_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(TOPICS, [ + <<"TopicA">>, + <<"TopicA/B">>, + <<"Topic/C">>, + <<"TopicA/C">>, + <<"/TopicA">> +]). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap, {seconds, 30}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + UdpPort = 1884, + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), + %% @TODO remove + emqx_logger:set_log_level(debug), + + dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + dbg:p(all, c), + + %dbg:tp(emqx_quic_stream, cx), + %% dbg:tp(quicer_stream, cx), + %% dbg:tp(emqx_quic_data_stream, cx), + %% dbg:tp(emqx_channel, cx), + %% dbg:tp(emqx_packet,check,cx), + %% dbg:tp(emqx_frame,parse,cx), + %dbg:tp(emqx_quic_connection, cx), + [{port, UdpPort}, {conn_fun, quic_connect} | Config]. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> term() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_group(GroupName, Config0) -> +%% term() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_testcase(TestCase, Config0) -> +%% term() | {save_config,Config1} | {fail,Reason} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% @spec groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%% @end +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% @spec all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +all() -> + [ + tc_data_stream_sub + ]. + +%%-------------------------------------------------------------------- +%% @spec TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%% @end +%%-------------------------------------------------------------------- + +%% @doc Test MQTT Subscribe via data_stream +tc_data_stream_sub(Config) -> + Topic = lists:nth(1, ?TOPICS), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [1]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [{Topic, [{qos, qos1}]}]), + {ok, _, [2]} = emqtt:subscribe_via( + C, + {new_data_stream, []}, + #{}, + [{lists:nth(2, ?TOPICS), [{qos, qos2}]}] + ), + {ok, _} = emqtt:publish(C, Topic, <<"qos 2 1">>, 2), + {ok, _} = emqtt:publish(C, Topic, <<"qos 2 2">>, 2), + {ok, _} = emqtt:publish(C, Topic, <<"qos 2 3">>, 2), + Msgs = receive_messages(3), + ct:pal("recv msg: ~p", [Msgs]), + ?assertEqual(3, length(Msgs)), + ok = emqtt:disconnect(C). + +receive_messages(Count) -> + receive_messages(Count, []). + +receive_messages(0, Msgs) -> + Msgs; +receive_messages(Count, Msgs) -> + receive + {publish, Msg} -> + receive_messages(Count - 1, [Msg | Msgs]); + _Other -> + receive_messages(Count, Msgs) + after 1000 -> + Msgs + end. From 9b52beaee92b3d22d9402bc948dc41874ca785e2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 5 Dec 2022 11:04:03 +0100 Subject: [PATCH 017/135] fix(quic): handle fast_close while handshake fail --- apps/emqx/src/emqx_quic_stream.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 70b01e643..2469a2ea7 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -159,9 +159,13 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. +%% @TODO supply some App Error Code +fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) -> + %% handshake aborted. + quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), + ok; fast_close({quic, Conn, _Stream, _Info}) -> %% Since we shutdown the control stream, we shutdown the connection as well - %% @TODO supply some App Error Code quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok. From 7d9bd33de9a3452868f7281e33b2c97d64504b9f Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 7 Dec 2022 13:54:07 +0100 Subject: [PATCH 018/135] feat(quic): bump quicer version to 0.0.100 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 0ecd21715..3ac2b8758 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. %% @TODO revert +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/mix.exs b/mix.exs index a2df76701..ce798997b 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.16", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.100", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 9da71355b..1d342b403 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,8 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - %% @TODO revert - {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 04a8a49dbecc63dde7671ca1f8ce80aa04db3f09 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 13 Dec 2022 09:49:02 +0100 Subject: [PATCH 019/135] test: update testcase for new emqtt --- apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 0b493dff6..5a9abc7f4 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -919,7 +919,7 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(Config) -> emqtt, connected, fun - (cast, ?PUBLISH_PACKET(?QOS_2, _PacketId), _State) -> + (cast, {?PUBLISH_PACKET(?QOS_2, _PacketId), _Via}, _State) -> ok = counters:add(CRef, 1, 1), {stop, {shutdown, for_testing}}; (Arg1, ARg2, Arg3) -> From 5bdcb0562d988e0cd5f01d6e188376cb93ea446a Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 14 Dec 2022 23:30:13 +0100 Subject: [PATCH 020/135] feat(quic): workaround to flushing the send buffer after conn shutdown Could not find a way to ensure msquic flush the send buffer after calling ConnectionShutdown. So just close the ctrl stream and let conn owner shutdown the conn. --- apps/emqx/src/emqx_quic_connection.erl | 9 +++++++++ apps/emqx/src/emqx_quic_stream.erl | 10 +++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index de7776429..480f59e3a 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -255,6 +255,15 @@ handle_call(_Req, _From, S) -> %% @doc handle DOWN messages from streams. %% @TODO handle DOWN from supervisor? +handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> + case Reason of + normal -> + quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); + _ -> + %% @TODO have some reasons mappings here. + quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1) + end, + {ok, S}; handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when Reason =:= normal orelse Reason =:= {shutdown, protocol_error} diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 2469a2ea7..555637d20 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -164,9 +164,13 @@ fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) -> %% handshake aborted. quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok; -fast_close({quic, Conn, _Stream, _Info}) -> - %% Since we shutdown the control stream, we shutdown the connection as well - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), +fast_close({quic, _Conn, Stream, _Info}) -> + %% Force flush + quicer:async_shutdown_stream(Stream), + %% @FIXME Since we shutdown the control stream, we shutdown the connection as well + %% *BUT* Msquic does not flush the send buffer if we shutdown the connection after + %% gracefully shutdown the stream. + % quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok. -spec ensure_ok_or_exit(atom(), list(term())) -> term(). From 1840a7f9237e75e10ee3f2d4b12e6d267608ec88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 15 Dec 2022 22:23:10 +0100 Subject: [PATCH 021/135] test(quic): improve coverage --- apps/emqx/include/emqx_quic.hrl | 24 + apps/emqx/src/emqx_quic_connection.erl | 33 +- apps/emqx/src/emqx_quic_data_stream.erl | 4 + apps/emqx/src/emqx_quic_stream.erl | 16 +- apps/emqx/test/emqtt_quic_SUITE.erl | 1291 +++++++++++++++++++++++ 5 files changed, 1344 insertions(+), 24 deletions(-) create mode 100644 apps/emqx/include/emqx_quic.hrl create mode 100644 apps/emqx/test/emqtt_quic_SUITE.erl diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl new file mode 100644 index 000000000..302f2704d --- /dev/null +++ b/apps/emqx/include/emqx_quic.hrl @@ -0,0 +1,24 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_QUIC_HRL). +-define(EMQX_QUIC_HRL, true). + +%% MQTT Over QUIC Shutdown Error code. +-define(MQTT_QUIC_CONN_NOERROR, 0). +-define(MQTT_QUIC_CONN_ERROR_OVERLOADED, 2). + +-endif. diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 480f59e3a..e632e6b1a 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -20,6 +20,7 @@ -include("logger.hrl"). -ifndef(BUILD_WITHOUT_QUIC). -include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_quic.hrl"). -else. -define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0). -endif. @@ -36,9 +37,9 @@ local_address_changed/3, peer_address_changed/3, streams_available/3, - peer_needs_streams/3, + % @TODO wait for newer quicer + %peer_needs_streams/3, resumed/3, - nst_received/3, new_stream/3 ]). @@ -120,11 +121,16 @@ new_conn( ok = quicer:async_handshake(Conn), {ok, S#{conn := Conn, ctrl_pid := CtrlPid}}; {'EXIT', _Pid, _Reason} -> - {error, stream_accept_error} + {stop, stream_accept_error, S} end; true -> emqx_metrics:inc('olp.new_conn'), - {error, overloaded} + quicer:async_shutdown_connection( + Conn, + ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, + ?MQTT_QUIC_CONN_ERROR_OVERLOADED + ), + {stop, normal, S} end. %% @doc callback when connection is connected. @@ -132,8 +138,8 @@ new_conn( {ok, cb_state()} | {error, any()}. connected(Conn, Props, #{slow_start := false} = S) -> ?SLOG(debug, Props), - {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), - {ok, S}; + {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), + {ok, S#{ctrl_pid => Pid}}; connected(_Conn, Props, S) -> ?SLOG(debug, Props), {ok, S}. @@ -147,12 +153,6 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when resumed(_Conn, _Data, S) -> {ok, S#{is_resumed := true}}. -%% @doc callback for receiving nst, should never happen on server. --spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret(). -nst_received(_Conn, _Data, S) -> - %% As server we should not recv NST! - {stop, no_nst_for_server, S}. - %% @doc callback for handling orphan data streams %% depends on the connecion state and control stream state. -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret(). @@ -233,9 +233,9 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) -> %% should cope with rate limiting %% @TODO this is not going to get triggered in current version %% for https://github.com/microsoft/msquic/issues/3120 --spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). -peer_needs_streams(_C, undefined, S) -> - {ok, S}. +%% -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). +%% peer_needs_streams(_C, undefined, S) -> +%% {ok, S}. %% @doc handle API calls handle_call( @@ -296,5 +296,6 @@ init_cb_state(#{zone := _Zone} = Map) -> streams => [], parse_state => undefined, channel => undefined, - serialize => undefined + serialize => undefined, + is_resumed => false }. diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 72f0e913f..094680b19 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -240,6 +240,10 @@ do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = Channel =/= undefined -> with_channel(handle_in, [Packet], S); +do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) when + Channel =/= undefined +-> + with_channel(handle_in, [FE], S); do_handle_appl_msg({close, Reason}, S) -> %% @TODO shall we abort shutdown or graceful shutdown? with_channel(handle_info, [{sock_closed, Reason}], S); diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 555637d20..714ef337f 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -108,15 +108,15 @@ wait({ConnOwner, Conn, ConnInfo}) -> %% Connection owner process down {'EXIT', ConnOwner, _Reason} -> {error, enotconn} - end; -%% For ownership handover -wait({PrevOwner, Conn, Stream, SocketInfo}) -> - case quicer:wait_for_handoff(PrevOwner, Stream) of - ok -> - {ok, socket(Conn, Stream, SocketInfo)}; - owner_down -> - {error, owner_down} end. +%% UNUSED, for ownership handover, +%% wait({PrevOwner, Conn, Stream, SocketInfo}) -> +%% case quicer:wait_for_handoff(PrevOwner, Stream) of +%% ok -> +%% {ok, socket(Conn, Stream, SocketInfo)}; +%% owner_down -> +%% {error, owner_down} +%% end. type(_) -> quic. diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl new file mode 100644 index 000000000..28e9bcd7b --- /dev/null +++ b/apps/emqx/test/emqtt_quic_SUITE.erl @@ -0,0 +1,1291 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqtt_quic_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("quicer/include/quicer.hrl"). + +suite() -> + [{timetrap, {seconds, 30}}]. + +all() -> + [ + {group, mstream}, + {group, shutdown}, + {group, misc} + ]. + +groups() -> + [ + {mstream, [], [{group, profiles}]}, + + {profiles, [], [ + {group, profile_low_latency}, + {group, profile_max_throughput} + ]}, + {profile_low_latency, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {profile_max_throughput, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {pub_qos0, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos1, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos2, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {sub_qos0, [{group, qos}]}, + {sub_qos1, [{group, qos}]}, + {sub_qos2, [{group, qos}]}, + {qos, [ + t_multi_streams_sub, + t_multi_streams_pub_parallel, + t_multi_streams_sub_pub_async, + t_multi_streams_sub_pub_sync, + t_multi_streams_unsub, + t_multi_streams_corr_topic, + t_multi_streams_unsub_via_other, + t_multi_streams_shutdown_data_stream_abortive, + t_multi_streams_dup_sub, + t_multi_streams_packet_boundary, + t_multi_streams_packet_malform + ]}, + + {shutdown, [ + {group, graceful_shutdown}, + {group, abort_recv_shutdown}, + {group, abort_send_shutdown}, + {group, abort_send_recv_shutdown} + ]}, + + {graceful_shutdown, [{group, ctrl_stream_shutdown}]}, + {abort_recv_shutdown, [{group, ctrl_stream_shutdown}]}, + {abort_send_shutdown, [{group, ctrl_stream_shutdown}]}, + {abort_send_recv_shutdown, [{group, ctrl_stream_shutdown}]}, + + {ctrl_stream_shutdown, [ + t_multi_streams_shutdown_ctrl_stream, + t_multi_streams_shutdown_ctrl_stream_then_reconnect, + t_multi_streams_remote_shutdown, + t_multi_streams_remote_shutdown_with_reconnect + ]}, + {misc, [ + t_conn_silent_close, + t_client_conn_bump_streams, + t_olp_true, + t_olp_reject, + t_conn_resume, + t_conn_without_ctrl_stream + ]} + ]. + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([]), + UdpPort = 14567, + start_emqx_quic(UdpPort), + %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + %% dbg:p(all, c), + %% dbg:tp(emqx_quic_connection, cx), + %% dbg:tp(emqx_quic_stream, cx), + %% dbg:tp(emqtt, cx), + %% dbg:tpl(emqtt_quic_stream, cx), + %% dbg:tpl(emqx_quic_stream, cx), + %% dbg:tpl(emqx_quic_data_stream, cx), + %% dbg:tpl(emqtt, cx), + [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. + +end_per_suite(_) -> + ok. + +init_per_group(pub_qos0, Config) -> + [{pub_qos, 0} | Config]; +init_per_group(sub_qos0, Config) -> + [{sub_qos, 0} | Config]; +init_per_group(pub_qos1, Config) -> + [{pub_qos, 1} | Config]; +init_per_group(sub_qos1, Config) -> + [{sub_qos, 1} | Config]; +init_per_group(pub_qos2, Config) -> + [{pub_qos, 2} | Config]; +init_per_group(sub_qos2, Config) -> + [{sub_qos, 2} | Config]; +init_per_group(abort_send_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config]; +init_per_group(abort_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config]; +init_per_group(abort_send_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config]; +init_per_group(graceful_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config]; +init_per_group(profile_max_throughput, Config) -> + quicer:reg_open(quic_execution_profile_type_max_throughput), + Config; +init_per_group(profile_low_latency, Config) -> + quicer:reg_open(quic_execution_profile_low_latency), + Config; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(_, Config) -> + emqx_common_test_helpers:start_apps([]), + Config. + +t_quic_sock(Config) -> + Port = 4567, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, Sock} = emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + quic_server:stop(Server). + +t_quic_sock_fail(_Config) -> + Port = 4567, + Error1 = + {error, + {transport_down, #{ + error => 2, + status => connection_refused + }}}, + Error2 = {error, {transport_down, #{error => 1, status => unreachable}}}, + case + emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ) + of + Error1 -> + ok; + Error2 -> + ok; + Other -> + ct:fail("unexpected return ~p", [Other]) + end. + +t_0_rtt(Config) -> + Port = 4568, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + NST = + receive + {quic, nst_received, Conn, Ticket} -> + Ticket + end, + {ok, Sock2} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, NST} + ], + 3000 + ), + send_and_recv_with(Sock2), + ok = emqtt_quic:close(Sock2), + quic_server:stop(Server). + +t_0_rtt_fail(Config) -> + Port = 4569, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + <<_Head:16, Left/binary>> = + receive + {quic, nst_received, Conn, Ticket} when is_binary(Ticket) -> + Ticket + end, + + Error = {error, {not_found, invalid_parameter}}, + Error = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, Left} + ], + 3000 + ), + quic_server:stop(Server). + +t_multi_streams_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + receive + {publish, #{ + client_pid := C, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C). + +t_multi_streams_pub_parallel(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +t_multi_streams_packet_boundary(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + LargePart3 = binary:copy(<<"stream data3">>, 2000), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(3), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := LargePart3, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + ok = emqtt:disconnect(C). + +%% @doc test that one malformed stream will not close the entire connection +t_multi_streams_packet_malform(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []), + {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), + + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + LargePart3 = binary:copy(<<"stream data3">>, 2000), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(3), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := LargePart3, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + + case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of + {ok, 10} -> ok; + {error, cancelled} -> ok; + {error, stm_send_error, aborted} -> ok + end, + + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + {error, stm_send_error, aborted} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_async(Config) -> + Topic = atom_to_binary(?FUNCTION_NAME), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic2, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_sync(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + Via1 = undefined, + ok; + {ok, #{reason_code := 0, via := Via1}} -> + ok + end, + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [ + {qos, PubQos} + ]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := Via2}} -> + ?assert(Via1 =/= Via2), + ok + end, + ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos, + via := SVia1 + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 4">>, + qos := RecQos, + via := SVia2 + }} + ], + lists:sort(PubRecvs) + ), + ok = emqtt:disconnect(C). + +t_multi_streams_dup_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value( + extra, emqtt:info(C) + ), + ?assertEqual(2, length(emqx_broker:subscribers(Topic))), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via1}} -> + ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }} + ], + lists:sort(PubRecvs) + ), + + RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs], + + ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]), + %% Can recv in any order + ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias), + + %% Shutdown one stream + quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500), + timer:sleep(100), + + ?assertEqual(1, length(emqx_broker:subscribers(Topic))), + + ok = emqtt:disconnect(C). + +t_multi_streams_corr_topic(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := PubVia}} -> ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs + ), + ok = emqtt:disconnect(C). + +t_multi_streams_unsub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + emqtt:unsubscribe_via(C, SubVia, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} -> + ok + end, + + timeout = recv_pub(1), + ok = emqtt:disconnect(C). + +t_multi_streams_unsub_via_other(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + + %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed" + {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia2}} -> ok + end, + + PubRecvs2 = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs2 + ), + ok = emqtt:disconnect(C). + +t_multi_streams_shutdown_data_stream_abortive(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia =/= SVia2), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + {quic, _Conn, DataStream} = PubVia, + quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 500, 100), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_shutdown_ctrl_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + unlink(C), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000), + timer:sleep(500), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(200), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_remote_shutdown(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + + timer:sleep(200), + start_emqx_quic(?config(port, Config)), + + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_remote_shutdown_with_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + + timer:sleep(200), + + start_emqx_quic(?config(port, Config)), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_conn_silent_close(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + %% quic idle timeout + 1s + timer:sleep(16000), + Topic = atom_to_binary(?FUNCTION_NAME), + ?assertException( + exit, + noproc, + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}]) + ). + +t_client_conn_bump_streams(Config) -> + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)), + ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}). + +t_olp_true(Config) -> + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + ok = meck:unload(emqx_olp). + +t_olp_reject(Config) -> + erlang:process_flag(trap_exit, true), + emqx_config:put_zone_conf(default, [overload_protection, enable], true), + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + ?assertEqual( + {error, + {transport_down, #{ + error => 346, + status => + user_canceled + }}}, + emqtt:quic_connect(C) + ), + ok = meck:unload(emqx_olp), + emqx_config:put_zone_conf(default, [overload_protection, enable], false). + +t_conn_resume(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C). + +t_conn_without_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, Conn} = quicer:connect( + {127, 0, 0, 1}, + ?config(port, Config), + [{alpn, ["mqtt"]}, {verify, none}], + 3000 + ), + receive + {quic, transport_shutdown, Conn, _} -> ok + end. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- +send_and_recv_with(Sock) -> + {ok, {IP, _}} = emqtt_quic:sockname(Sock), + ?assert(lists:member(tuple_size(IP), [4, 8])), + ok = emqtt_quic:send(Sock, <<"ping">>), + emqtt_quic:setopts(Sock, [{active, false}]), + {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0), + ok = emqtt_quic:setopts(Sock, [{active, 100}]), + {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]), + %% connection level counters, not stream level + [{send_cnt, _}, {recv_cnt, _}] = Stats. + +certfile(Config) -> + filename:join([test_dir(Config), "certs", "test.crt"]). + +keyfile(Config) -> + filename:join([test_dir(Config), "certs", "test.key"]). + +test_dir(Config) -> + filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))). + +recv_pub(Count) -> + recv_pub(Count, []). + +recv_pub(0, Acc) -> + lists:reverse(Acc); +recv_pub(Count, Acc) -> + receive + {publish, _Prop} = Pub -> + recv_pub(Count - 1, [Pub | Acc]) + after 100 -> + timeout + end. + +all_tc() -> + code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")), + emqx_common_test_helpers:all(?MODULE). + +-spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2. +calc_qos(PubQos, SubQos) -> + if + PubQos > SubQos -> + SubQos; + SubQos > PubQos -> + PubQos; + true -> + PubQos + end. +-spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer(). +calc_pkt_id(0, _Id) -> + undefined; +calc_pkt_id(1, Id) -> + Id; +calc_pkt_id(2, Id) -> + Id. + +-spec start_emqx_quic(inet:port_number()) -> ok. +start_emqx_quic(UdpPort) -> + emqx_common_test_helpers:start_apps([]), + application:ensure_all_started(quicer), + emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort). + +-spec stop_emqx() -> ok. +stop_emqx() -> + emqx_common_test_helpers:stop_apps([]). From ceac5a0ec7b054c7db1ff618a57ea8b9bbb7c00f Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 20 Dec 2022 20:35:59 +0100 Subject: [PATCH 022/135] feat(quic): bump to quicer 0.0.101 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 3ac2b8758..873b599cd 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/mix.exs b/mix.exs index ce798997b..088b06728 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.100", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.101", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 1d342b403..e1b94a954 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 0544a3ca0ce95424e70fbd5bc71ec3022c75c736 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 22 Dec 2022 09:49:20 +0100 Subject: [PATCH 023/135] fix(quic): setops on stream and handle peer needs stream - setopts should go for stream - handle peer_needs_streams for none msquic clients --- apps/emqx/src/emqx_quic_connection.erl | 25 ++++++++++++++----------- apps/emqx/src/emqx_quic_stream.erl | 6 +++--- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index e632e6b1a..8cdd9d5e6 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -37,8 +37,7 @@ local_address_changed/3, peer_address_changed/3, streams_available/3, - % @TODO wait for newer quicer - %peer_needs_streams/3, + peer_needs_streams/3, resumed/3, new_stream/3 ]). @@ -233,9 +232,12 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) -> %% should cope with rate limiting %% @TODO this is not going to get triggered in current version %% for https://github.com/microsoft/msquic/issues/3120 -%% -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). -%% peer_needs_streams(_C, undefined, S) -> -%% {ok, S}. +-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). +peer_needs_streams(_C, undefined, S) -> + ?SLOG(info, #{ + msg => "ignore: peer need more streames", info => maps:with([conn_pid, ctrl_pid], S) + }), + {ok, S}. %% @doc handle API calls handle_call( @@ -255,7 +257,7 @@ handle_call(_Req, _From, S) -> %% @doc handle DOWN messages from streams. %% @TODO handle DOWN from supervisor? -handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> +handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> case Reason of normal -> quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); @@ -264,12 +266,13 @@ handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Con quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1) end, {ok, S}; -handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when - Reason =:= normal orelse - Reason =:= {shutdown, protocol_error} --> +handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) -> case proplists:is_defined(Pid, Streams) of - true -> + true when + Reason =:= normal orelse + Reason =:= {shutdown, protocol_error} orelse + Reason =:= killed + -> {ok, S}; false -> {stop, unknown_pid_down, S} diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 714ef337f..6fb7b0816 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -137,13 +137,13 @@ getstat({quic, Conn, _Stream, _Info}, Stats) -> Res -> Res end. -setopts(Socket, Opts) -> +setopts({quic, _Conn, Stream, _Info}, Opts) -> lists:foreach( fun ({Opt, V}) when is_atom(Opt) -> - quicer:setopt(Socket, Opt, V); + quicer:setopt(Stream, Opt, V); (Opt) when is_atom(Opt) -> - quicer:setopt(Socket, Opt, true) + quicer:setopt(Stream, Opt, true) end, Opts ), From 0173121a309adea318e06799bd71508d4643aa60 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 22 Dec 2022 22:20:29 +0100 Subject: [PATCH 024/135] feat(quic): improve coverage and remove unused code --- apps/emqx/src/emqx_quic_connection.erl | 19 ++- apps/emqx/src/emqx_quic_data_stream.erl | 18 ++- apps/emqx/test/emqtt_quic_SUITE.erl | 151 +++++++++++++++++++++++- 3 files changed, 173 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 8cdd9d5e6..588648483 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -135,20 +135,17 @@ new_conn( %% @doc callback when connection is connected. -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. -connected(Conn, Props, #{slow_start := false} = S) -> - ?SLOG(debug, Props), - {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), - {ok, S#{ctrl_pid => Pid}}; connected(_Conn, Props, S) -> ?SLOG(debug, Props), {ok, S}. %% @doc callback when connection is resumed from 0-RTT -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret(). -resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when - is_function(ResumeFun) --> - ResumeFun(Conn, Data, S); +%% reserve resume conn with callback. +%% resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when +%% is_function(ResumeFun) +%% -> +%% ResumeFun(Conn, Data, S); resumed(_Conn, _Data, S) -> {ok, S#{is_resumed := true}}. @@ -245,9 +242,11 @@ handle_call( _From, #{streams := Streams} = S ) -> - [emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) || {OwnerPid, _Stream} <- Streams], + [ + catch emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) + || {OwnerPid, _Stream} <- Streams + ], {reply, ok, S#{ - %streams := [], %% @FIXME what ?????? channel := Channel, serialize := Serialize, parse_state := PS diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 094680b19..24dd71c29 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -68,11 +68,11 @@ activate_data(StreamPid, {PS, Serialize, Channel}) -> %% @TODO -spec init_handoff( Stream, - #{parse_state := PS} = _StreamOpts, + _StreamOpts, Connection, #{is_orphan := true, flags := Flags} ) -> - {ok, init_state(Stream, Connection, Flags, PS)}. + {ok, init_state(Stream, Connection, Flags)}. %% %% @doc Post handoff data stream @@ -239,6 +239,7 @@ do_handle_appl_msg( do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = S) when Channel =/= undefined -> + ok = inc_incoming_stats(Packet), with_channel(handle_in, [Packet], S); do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) when Channel =/= undefined @@ -422,6 +423,19 @@ do_parse_incoming(Data, Packets, ParseState) -> end. %% followings are copied from emqx_connection +-compile({inline, [inc_incoming_stats/1]}). +inc_incoming_stats(Packet = ?PACKET(Type)) -> + inc_counter(recv_pkt, 1), + case Type =:= ?PUBLISH of + true -> + inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), + inc_counter(incoming_pubs, 1); + false -> + ok + end, + emqx_metrics:inc_recv(Packet). + -compile({inline, [inc_outgoing_stats/1]}). inc_outgoing_stats({error, message_too_large}) -> inc_counter('send_msg.dropped', 1), diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl index 28e9bcd7b..6c19ecdad 100644 --- a/apps/emqx/test/emqtt_quic_SUITE.erl +++ b/apps/emqx/test/emqtt_quic_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). suite() -> [{timetrap, {seconds, 30}}]. @@ -79,7 +80,10 @@ groups() -> t_multi_streams_shutdown_data_stream_abortive, t_multi_streams_dup_sub, t_multi_streams_packet_boundary, - t_multi_streams_packet_malform + t_multi_streams_packet_malform, + t_multi_streams_kill_sub_stream, + t_multi_streams_packet_too_large, + t_conn_change_client_addr ]}, {shutdown, [ @@ -537,12 +541,84 @@ t_multi_streams_packet_malform(Config) -> timer:sleep(200), ?assert(is_list(emqtt:info(C))), - {error, stm_send_error, aborted} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), + {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>), + timer:sleep(200), ?assert(is_list(emqtt:info(C))), ok = emqtt:disconnect(C). +t_multi_streams_packet_too_large(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + meck:new(emqx_frame, [passthrough, no_history]), + ok = meck:expect( + emqx_frame, + serialize_opts, + fun(#mqtt_packet_connect{proto_ver = ProtoVer}) -> + #{version => ProtoVer, max_size => 1024} + end + ), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + binary:copy(<<"stream data 1">>, 1024), + [{qos, PubQos}], + undefined + ), + timeout = recv_pub(1), + ?assert(is_list(emqtt:info(C))), + ok = meck:unload(emqx_frame), + ok = emqtt:disconnect(C). + +t_conn_change_client_addr(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := _PktId1, + payload := <<"stream data 1">>, + qos := RecQos + }} + ], + recv_pub(1) + ), + NewPort = select_port(), + {ok, OldAddr} = quicer:sockname(Conn), + ?assertEqual( + ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort)) + ), + {ok, NewAddr} = quicer:sockname(Conn), + ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]), + ?assertNotEqual(OldAddr, NewAddr), + ?assert(is_list(emqtt:info(C))), + ok = emqtt:disconnect(C). + t_multi_streams_sub_pub_async(Config) -> Topic = atom_to_binary(?FUNCTION_NAME), PubQos = ?config(pub_qos, Config), @@ -815,6 +891,57 @@ t_multi_streams_unsub(Config) -> timeout = recv_pub(1), ok = emqtt:disconnect(C). +t_multi_streams_kill_sub_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + [TopicStreamOwner] = emqx_broker:subscribers(Topic), + exit(TopicStreamOwner, kill), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 -> + ok + end, + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia2}} -> + ok + end, + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + topic := Topic2, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + recv_pub(1) + ), + ?assertEqual(timeout, recv_pub(1)), + ok. + t_multi_streams_unsub_via_other(Config) -> PubQos = ?config(pub_qos, Config), SubQos = ?config(sub_qos, Config), @@ -1208,7 +1335,9 @@ t_conn_resume(Config) -> {nst, NST} | Config ]), - {ok, _} = emqtt:quic_connect(C). + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). t_conn_without_ctrl_stream(Config) -> erlang:process_flag(trap_exit, true), @@ -1289,3 +1418,19 @@ start_emqx_quic(UdpPort) -> -spec stop_emqx() -> ok. stop_emqx() -> emqx_common_test_helpers:stop_apps([]). + +%% select a random port picked by OS +-spec select_port() -> inet:port_number(). +select_port() -> + {ok, S} = gen_udp:open(0, [{reuseaddr, true}]), + {ok, {_, Port}} = inet:sockname(S), + gen_udp:close(S), + case os:type() of + {unix, darwin} -> + %% in MacOS, still get address_in_use after close port + timer:sleep(500); + _ -> + skip + end, + ct:pal("select port: ~p", [Port]), + Port. From 71d3148544ac98c7d2ea6bd3be3d32b276354607 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Jan 2023 15:15:34 +0100 Subject: [PATCH 025/135] feat(quic): stream use active_n 10 --- apps/emqx/src/emqx_quic_data_stream.erl | 3 ++- apps/emqx/src/emqx_quic_stream.erl | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 24dd71c29..61c13bdee 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -182,7 +182,8 @@ handle_stream_data( %% {ok, State} %% end. -passive(_Stream, undefined, S) -> +passive(Stream, undefined, S) -> + quicer:setopt(Stream, active, 10), {ok, S}. stream_closed( diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 6fb7b0816..667ddb2b0 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -270,8 +270,9 @@ start_completed(_Stream, #{status := Other} = Prop, S) -> %% {stop, unimpl}. -spec passive(stream_handle(), undefined, cb_data()) -> cb_ret(). -passive(_Stream, undefined, _S) -> - {stop, unimpl}. +passive(Stream, undefined, S) -> + quicer:setopt(Stream, active, 10), + {ok, S}. -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret(). stream_closed( From 1e8b2e247e9d31c754412622ce8cfcf4eb0f4bea Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 6 Jan 2023 13:09:25 +0100 Subject: [PATCH 026/135] feat(quic): 0-RTT multi-streams data --- apps/emqx/src/emqx_quic_connection.erl | 3 +- apps/emqx/src/emqx_quic_data_stream.erl | 37 ++++++----- apps/emqx/src/emqx_quic_stream.erl | 35 +--------- apps/emqx/test/emqtt_quic_SUITE.erl | 88 +++++++++++++++++++++++-- 4 files changed, 104 insertions(+), 59 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 588648483..ef0d9b2e3 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -174,7 +174,8 @@ new_stream( limiter => Limiter, parse_state => PS, channel => Channel, - serialize => Serialize + serialize => Serialize, + quic_event_mask => ?QUICER_STREAM_EVENT_MASK_START_COMPLETE }, {ok, NewStreamOwner} = quicer_stream:start_link( emqx_quic_data_stream, diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 61c13bdee..bea0d37e1 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -25,14 +25,12 @@ -include_lib("quicer/include/quicer.hrl"). -include("emqx_mqtt.hrl"). -include("logger.hrl"). --behaviour(quicer_stream). +-behaviour(quicer_remote_stream). %% Connection Callbacks -export([ init_handoff/4, post_handoff/3, - new_stream/3, - start_completed/3, send_complete/3, peer_send_shutdown/3, peer_send_aborted/3, @@ -79,17 +77,15 @@ init_handoff( %% %% @TODO -spec %% +post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Channel}, S) -> + %% Channel isn't ready yet. + %% Data stream should wait for activate call with ?MODULE:activate_data/2 + {ok, S}; post_handoff(Stream, {PS, Serialize, Channel}, S) -> ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}), quicer:setopt(Stream, active, true), {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. -%% -%% @doc when this proc is assigned to the owner of new stream -%% -new_stream(Stream, #{flags := Flags}, Connection) -> - {ok, init_state(Stream, Connection, Flags)}. - %% %% @doc for local initiated stream %% @@ -125,12 +121,6 @@ send_complete(_Stream, true = _IsCanceled, S) -> send_shutdown_complete(_Stream, _Flags, S) -> {ok, S}. -start_completed(_Stream, #{status := success, stream_id := StreamId}, S) -> - {ok, S#{stream_id => StreamId}}; -start_completed(_Stream, #{status := Other}, S) -> - %% or we could retry - {stop, {start_fail, Other}, S}. - handle_stream_data( Stream, Bin, @@ -208,7 +198,18 @@ stream_closed( {stop, normal, S}. handle_call(Call, _From, S) -> - do_handle_call(Call, S). + case do_handle_call(Call, S) of + {ok, NewS} -> + {reply, ok, NewS}; + {error, Reason, NewS} -> + {reply, {error, Reason}, NewS}; + {{continue, _} = Cont, NewS} -> + {reply, ok, NewS, Cont}; + {hibernate, NewS} -> + {reply, ok, NewS, hibernate}; + {stop, Reason, NewS} -> + {stop, Reason, {stopped, Reason}, NewS} + end. handle_continue(handle_appl_msg, #{task_queue := Q} = S) -> case queue:out(Q) of @@ -390,8 +391,8 @@ do_handle_call( ?SLOG(error, #{msg => "set stream active failed", error => E}), {stop, E, NewS} end; -do_handle_call(_Call, S) -> - {reply, {error, unimpl}, S}. +do_handle_call(_Call, _S) -> + {error, unimpl}. %% @doc return reserved order of Packets parse_incoming(Data, PS) -> diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 667ddb2b0..ee764cdc5 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -17,7 +17,7 @@ %% MQTT/QUIC Stream -module(emqx_quic_stream). --behaviour(quicer_stream). +-behaviour(quicer_remote_stream). %% emqx transport Callbacks -export([ @@ -57,18 +57,14 @@ -type stream_handle() :: quicer:stream_handle(). -export([ - init_handoff/4, new_stream/3, - start_completed/3, send_complete/3, peer_send_shutdown/3, peer_send_aborted/3, peer_receive_aborted/3, send_shutdown_complete/3, stream_closed/3, - peer_accepted/3, - passive/3, - handle_call/4 + passive/3 ]). -export_type([socket/0]). @@ -195,21 +191,10 @@ async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> %%% %%% quicer stream callbacks %%% - --spec init_handoff(stream_handle(), #{}, quicer:connection_handle(), #{}) -> cb_ret(). -init_handoff(_Stream, _StreamOpts, _Conn, _Flags) -> - %% stream owner already set while starts. - {stop, unimpl}. - -spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret(). new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) -> {stop, unimpl}. --spec peer_accepted(stream_handle(), undefined, cb_data()) -> cb_ret(). -peer_accepted(_Stream, undefined, S) -> - %% We just ignore it - {ok, S}. - -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). peer_receive_aborted(Stream, ErrorCode, S) -> quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), @@ -237,19 +222,6 @@ send_complete(_Stream, true = _IsCancelled, S) -> send_shutdown_complete(_Stream, _IsGraceful, S) -> {ok, S}. --spec start_completed(stream_handle(), quicer:stream_start_completed_props(), cb_data()) -> - cb_ret(). -start_completed(_Stream, #{status := success, stream_id := StreamId} = Prop, S) -> - ?SLOG(debug, Prop), - {ok, S#{stream_id => StreamId}}; -start_completed(_Stream, #{status := stream_limit_reached, stream_id := _StreamId} = Prop, _S) -> - ?SLOG(error, #{message => start_completed}, Prop), - {stop, stream_limit_reached}; -start_completed(_Stream, #{status := Other} = Prop, S) -> - ?SLOG(error, Prop), - %% or we could retry? - {stop, {start_fail, Other}, S}. - %% Local stream, Unidir %% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data()) %% -> cb_ret(). @@ -299,9 +271,6 @@ stream_closed( %% a msg to be processed {ok, {sock_closed, Status}, S}. -handle_call(_Stream, _Request, _Opts, S) -> - {error, unimpl, S}. - %%% %%% Internals %%% diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl index 6c19ecdad..cfb9d4ae4 100644 --- a/apps/emqx/test/emqtt_quic_SUITE.erl +++ b/apps/emqx/test/emqtt_quic_SUITE.erl @@ -77,12 +77,12 @@ groups() -> t_multi_streams_unsub, t_multi_streams_corr_topic, t_multi_streams_unsub_via_other, - t_multi_streams_shutdown_data_stream_abortive, t_multi_streams_dup_sub, t_multi_streams_packet_boundary, t_multi_streams_packet_malform, t_multi_streams_kill_sub_stream, t_multi_streams_packet_too_large, + t_multi_streams_sub_0_rtt, t_conn_change_client_addr ]}, @@ -93,10 +93,22 @@ groups() -> {group, abort_send_recv_shutdown} ]}, - {graceful_shutdown, [{group, ctrl_stream_shutdown}]}, - {abort_recv_shutdown, [{group, ctrl_stream_shutdown}]}, - {abort_send_shutdown, [{group, ctrl_stream_shutdown}]}, - {abort_send_recv_shutdown, [{group, ctrl_stream_shutdown}]}, + {graceful_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, {ctrl_stream_shutdown, [ t_multi_streams_shutdown_ctrl_stream, @@ -104,6 +116,8 @@ groups() -> t_multi_streams_remote_shutdown, t_multi_streams_remote_shutdown_with_reconnect ]}, + + {data_stream_shutdown, [t_multi_streams_shutdown_data_stream]}, {misc, [ t_conn_silent_close, t_client_conn_bump_streams, @@ -1004,7 +1018,7 @@ t_multi_streams_unsub_via_other(Config) -> ), ok = emqtt:disconnect(C). -t_multi_streams_shutdown_data_stream_abortive(Config) -> +t_multi_streams_shutdown_data_stream(Config) -> PubQos = ?config(pub_qos, Config), SubQos = ?config(sub_qos, Config), RecQos = calc_qos(PubQos, SubQos), @@ -1045,7 +1059,7 @@ t_multi_streams_shutdown_data_stream_abortive(Config) -> #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), {quic, _Conn, DataStream} = PubVia, - quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 500, 100), + quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), timer:sleep(500), %% Still alive ?assert(is_list(emqtt:info(C))). @@ -1351,6 +1365,66 @@ t_conn_without_ctrl_stream(Config) -> {quic, transport_shutdown, Conn, _} -> ok end. +t_data_stream_race_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). + +t_multi_streams_sub_0_rtt(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + <<"qos 2 1">>, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 5764994436956276aafe84359b98b8978457903c Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 6 Jan 2023 13:10:33 +0100 Subject: [PATCH 027/135] feat(quic): bump to quicer 0.0.103 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 873b599cd..faf668b26 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/mix.exs b/mix.exs index 088b06728..181f02633 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.101", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.103", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index e1b94a954..adf930ac5 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From f65ac5422e3c850be9104805781db27b054e332a Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 8 Jan 2023 22:24:32 +0100 Subject: [PATCH 028/135] test(quic): improve coverage --- apps/emqx/src/emqx_quic_data_stream.erl | 6 +- apps/emqx/src/emqx_quic_stream.erl | 2 +- apps/emqx/test/emqtt_quic_SUITE.erl | 196 ++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index bea0d37e1..2aa3ad4f7 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -83,7 +83,7 @@ post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Cha {ok, S}; post_handoff(Stream, {PS, Serialize, Channel}, S) -> ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}), - quicer:setopt(Stream, active, true), + quicer:setopt(Stream, active, 10), {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. %% @@ -301,8 +301,8 @@ handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir : is_list(Packets) -> OutBin = [serialize_packet(P, Serialize) || P <- filter_disallowed_out(Packets)], - %% @TODO in which case shall we use sync send? - Res = quicer:async_send(Stream, OutBin), + %% Send data async but still want send feedback via {quic, send_complete, ...} + Res = quicer:async_send(Stream, OutBin, ?QUICER_SEND_FLAG_SYNC), ?TRACE("MQTT", "mqtt_packet_sent", #{packets => Packets}), [ok = inc_outgoing_stats(P) || P <- Packets], Res. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index ee764cdc5..88cf4b7c3 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -183,7 +183,7 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> end. async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> - case quicer:send(Stream, Data) of + case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of {ok, _Len} -> ok; Other -> Other end. diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl index cfb9d4ae4..f926c2f3e 100644 --- a/apps/emqx/test/emqtt_quic_SUITE.erl +++ b/apps/emqx/test/emqtt_quic_SUITE.erl @@ -71,7 +71,9 @@ groups() -> {sub_qos2, [{group, qos}]}, {qos, [ t_multi_streams_sub, + t_multi_streams_pub_5x100, t_multi_streams_pub_parallel, + t_multi_streams_pub_parallel_no_blocking, t_multi_streams_sub_pub_async, t_multi_streams_sub_pub_sync, t_multi_streams_unsub, @@ -83,6 +85,8 @@ groups() -> t_multi_streams_kill_sub_stream, t_multi_streams_packet_too_large, t_multi_streams_sub_0_rtt, + t_multi_streams_sub_0_rtt_large_payload, + t_multi_streams_sub_0_rtt_stream_data_cont, t_conn_change_client_addr ]}, @@ -347,6 +351,36 @@ t_multi_streams_sub(Config) -> end, ok = emqtt:disconnect(C). +t_multi_streams_pub_5x100(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + PubVias = lists:map( + fun(_N) -> + {ok, Via} = emqtt:start_data_stream(C, []), + Via + end, + lists:seq(1, 5) + ), + [ + begin + case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + 0 == (N rem 10) andalso timer:sleep(10) + end + || N <- lists:seq(1, 100), PVia <- PubVias + ], + ?assert(timeout =/= recv_pub(500)), + ok = emqtt:disconnect(C). + t_multi_streams_pub_parallel(Config) -> PubQos = ?config(pub_qos, Config), SubQos = ?config(sub_qos, Config), @@ -400,6 +434,60 @@ t_multi_streams_pub_parallel(Config) -> ), ok = emqtt:disconnect(C). +%% @doc test two pub streams, one send incomplete MQTT packet() can not block another. +t_multi_streams_pub_parallel_no_blocking(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId2 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + Drop = <<"stream data 1">>, + meck:new(emqtt_quic, [passthrough, no_history]), + meck:expect(emqtt_quic, send, fun(Sock, IoList) -> + case lists:last(IoList) == Drop of + true -> + ct:pal("meck droping ~p", [Drop]), + meck:passthrough([Sock, IoList -- [Drop]]); + false -> + meck:passthrough([Sock, IoList]) + end + end), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + Drop, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + meck:unload(emqtt_quic), + ?assertEqual(timeout, recv_pub(1)), + ok = emqtt:disconnect(C). + t_multi_streams_packet_boundary(Config) -> PubQos = ?config(pub_qos, Config), SubQos = ?config(sub_qos, Config), @@ -1425,6 +1513,114 @@ t_multi_streams_sub_0_rtt(Config) -> ok = emqtt:disconnect(C), ok = emqtt:disconnect(C0). +t_multi_streams_sub_0_rtt_large_payload(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +%% @doc verify data stream can continue after 0-RTT handshake +t_multi_streams_sub_0_rtt_stream_data_cont(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + Payload2 = <<"2nd part", Payload/binary>>, + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload2, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + receive + {publish, #{ + client_pid := C0, + payload := Payload2, + qos := RecQos, + topic := Topic + }} -> + ok; + Other2 -> + ct:fail("unexpected recv ~p", [Other2]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 22dcf5907e6f278bd665640aeb2987b56e68928f Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 8 Jan 2023 22:25:29 +0100 Subject: [PATCH 029/135] feat(quic): bump to quicer 0.0.104 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index faf668b26..e942e1a5c 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/mix.exs b/mix.exs index 181f02633..bf300761f 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.103", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.104", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index adf930ac5..e99f83683 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 00f615a1e33289d870d7ffada7d17952b05bcf95 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 9 Jan 2023 09:17:03 +0100 Subject: [PATCH 030/135] chore(quic): clean code --- apps/emqx/include/emqx_quic.hrl | 2 +- apps/emqx/src/emqx_connection.erl | 14 +- apps/emqx/src/emqx_quic_connection.erl | 43 +- apps/emqx/src/emqx_quic_data_stream.erl | 119 +- apps/emqx/src/emqx_quic_stream.erl | 77 +- apps/emqx/test/emqtt_quic_SUITE.erl | 1706 --------------- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 13 - .../test/emqx_quic_multistreams_SUITE.erl | 1880 +++++++++++++++-- 8 files changed, 1817 insertions(+), 2037 deletions(-) delete mode 100644 apps/emqx/test/emqtt_quic_SUITE.erl diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl index 302f2704d..3366b8938 100644 --- a/apps/emqx/include/emqx_quic.hrl +++ b/apps/emqx/include/emqx_quic.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 980c41010..be420d65e 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -119,10 +119,7 @@ limiter_timer :: undefined | reference(), %% QUIC conn pid if is a pid - quic_conn_pid :: maybe(pid()), - - %% QUIC control stream callback state - quic_ctrl_state :: map() + quic_conn_pid :: maybe(pid()) }). -record(retry, { @@ -378,8 +375,7 @@ init_state( limiter_buffer = queue:new(), limiter_timer = undefined, %% for quic streams to inherit - quic_conn_pid = maps:get(conn_pid, Opts, undefined), - quic_ctrl_state = #{} + quic_conn_pid = maps:get(conn_pid, Opts, undefined) }. run_loop( @@ -928,12 +924,6 @@ handle_info({sock_error, Reason}, State) -> handle_info({sock_closed, Reason}, close_socket(State)); handle_info({quic, Event, Handle, Prop}, State) -> emqx_quic_stream:Event(Handle, Prop, State); -%% handle_info({quic, peer_send_shutdown, _Stream}, State) -> -%% handle_info({sock_closed, force}, close_socket(State)); -%% handle_info({quic, closed, _Channel, ReasonFlag}, State) -> -%% handle_info({sock_closed, ReasonFlag}, State); -%% handle_info({quic, closed, _Stream}, State) -> -%% handle_info({sock_closed, force}, State); handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index ef0d9b2e3..69d16cbc3 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -17,13 +17,11 @@ %% @doc impl. the quic connection owner process. -module(emqx_quic_connection). --include("logger.hrl"). -ifndef(BUILD_WITHOUT_QUIC). + +-include("logger.hrl"). -include_lib("quicer/include/quicer.hrl"). -include_lib("emqx/include/emqx_quic.hrl"). --else. --define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0). --endif. -behavior(quicer_connection). @@ -55,10 +53,9 @@ %% Pid of ctrl stream ctrl_pid := undefined | pid(), %% quic connecion handle - conn := undefined | quicer:conneciton_hanlder(), - %% streams that handoff from this process, excluding control stream - %% these streams could die/closed without effecting the connecion/session. - + conn := undefined | quicer:conneciton_handle(), + %% Data streams that handoff from this process + %% these streams could die/close without effecting the connecion/session. %@TODO type? streams := [{pid(), quicer:stream_handle()}], %% New stream opts @@ -82,22 +79,20 @@ activate_data_streams(ConnOwner, {PS, Serialize, Channel}) -> gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity). %% @doc conneciton owner init callback --spec init(map() | list()) -> {ok, cb_state()}. -init(ConnOpts) when is_list(ConnOpts) -> - init(maps:from_list(ConnOpts)); +-spec init(map()) -> {ok, cb_state()}. init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> init(S#{stream_opts := maps:from_list(SOpts)}); init(ConnOpts) when is_map(ConnOpts) -> {ok, init_cb_state(ConnOpts)}. --spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) -> +-spec closed(quicer:conneciton_handle(), quicer:conn_closed_props(), cb_state()) -> {stop, normal, cb_state()}. closed(_Conn, #{is_peer_acked := _} = Prop, S) -> ?SLOG(debug, Prop), {stop, normal, S}. %% @doc handle the new incoming connecion as the connecion acceptor. --spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) -> +-spec new_conn(quicer:connection_handle(), quicer:new_conn_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. new_conn( Conn, @@ -133,7 +128,7 @@ new_conn( end. %% @doc callback when connection is connected. --spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) -> +-spec connected(quicer:connection_handle(), quicer:connected_props(), cb_state()) -> {ok, cb_state()} | {error, any()}. connected(_Conn, Props, S) -> ?SLOG(debug, Props), @@ -185,21 +180,21 @@ new_stream( Props ), quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}), - %% @TODO keep them in ``inactive_streams' + %% @TODO maybe keep them in `inactive_streams' {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}. -%% @doc callback for handling for remote connecion shutdown. +%% @doc callback for handling remote connecion shutdown. -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). -shutdown(Conn, _ErrorCode, S) -> - %% @TODO check spec what to set for the ErrorCode? +shutdown(Conn, ErrorCode, S) -> + ErrorCode =/= 0 andalso ?SLOG(debug, #{error_code => ErrorCode, state => S}), quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. -%% @doc callback for handling for transport error, such as idle timeout +%% @doc callback for handling transport error, such as idle timeout -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) -> cb_ret(). -transport_shutdown(_C, _DownInfo, S) -> - %% @TODO some counter +transport_shutdown(_C, DownInfo, S) when is_map(DownInfo) -> + ?SLOG(debug, DownInfo), {ok, S}. %% @doc callback for handling for peer addr changed. @@ -238,6 +233,7 @@ peer_needs_streams(_C, undefined, S) -> {ok, S}. %% @doc handle API calls +-spec handle_call(Req :: term(), gen_server:from(), cb_state()) -> cb_ret(). handle_call( {activate_data_streams, {PS, Serialize, Channel} = ActivateData}, _From, @@ -256,7 +252,6 @@ handle_call(_Req, _From, S) -> {reply, {error, unimpl}, S}. %% @doc handle DOWN messages from streams. -%% @TODO handle DOWN from supervisor? handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> case Reason of normal -> @@ -302,3 +297,7 @@ init_cb_state(#{zone := _Zone} = Map) -> serialize => undefined, is_resumed => false }. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 2aa3ad4f7..e3f6b7adc 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,11 +21,14 @@ %% -module(emqx_quic_data_stream). + +-ifndef(BUILD_WITHOUT_QUIC). +-behaviour(quicer_remote_stream). + -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("quicer/include/quicer.hrl"). -include("emqx_mqtt.hrl"). -include("logger.hrl"). --behaviour(quicer_remote_stream). %% Connection Callbacks -export([ @@ -37,12 +40,12 @@ peer_receive_aborted/3, send_shutdown_complete/3, stream_closed/3, - peer_accepted/3, passive/3 ]). -export([handle_stream_data/4]). +%% gen_server API -export([activate_data/2]). -export([ @@ -51,9 +54,19 @@ handle_continue/2 ]). +-type cb_ret() :: quicer_stream:cb_ret(). +-type cb_state() :: quicer_stream:cb_state(). +-type error_code() :: quicer:error_code(). +-type connection_handle() :: quicer:connection_handle(). +-type stream_handle() :: quicer:stream_handle(). +-type handoff_data() :: { + emqx_frame:parse_state() | undefined, + emqx_frame:serialize_opts() | undefined, + emqx_channel:channel() | undefined +}. %% %% @doc Activate the data handling. -%% Data handling is disabled before control stream allows the data processing. +%% Note, data handling is disabled before finishing the validation over control stream. -spec activate_data(pid(), { emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel() }) -> ok. @@ -61,9 +74,12 @@ activate_data(StreamPid, {PS, Serialize, Channel}) -> gen_server:call(StreamPid, {activate, {PS, Serialize, Channel}}, infinity). %% -%% @doc Handoff from previous owner, mostly from the connection owner. -%% @TODO parse_state doesn't look necessary since we have it in post_handoff -%% @TODO -spec +%% @doc Handoff from previous owner, from the connection owner. +%% Note, unlike control stream, there is no acceptor for data streams. +%% The connection owner get new stream, spawn new proc and then handover to it. +%% +-spec init_handoff(stream_handle(), map(), connection_handle(), quicer:new_stream_props()) -> + {ok, cb_state()}. init_handoff( Stream, _StreamOpts, @@ -75,10 +91,9 @@ init_handoff( %% %% @doc Post handoff data stream %% -%% @TODO -spec -%% +-spec post_handoff(stream_handle(), handoff_data(), cb_state()) -> cb_ret(). post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Channel}, S) -> - %% Channel isn't ready yet. + %% When the channel isn't ready yet. %% Data stream should wait for activate call with ?MODULE:activate_data/2 {ok, S}; post_handoff(Stream, {PS, Serialize, Channel}, S) -> @@ -86,53 +101,35 @@ post_handoff(Stream, {PS, Serialize, Channel}, S) -> quicer:setopt(Stream, active, 10), {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. -%% -%% @doc for local initiated stream -%% -peer_accepted(_Stream, _Flags, S) -> - %% we just ignore it - {ok, S}. - -peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> +-spec peer_receive_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> %% we abort send with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), - {ok, S}; -peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) -> quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) -> +-spec peer_send_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). +peer_send_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> %% we abort receive with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), - {ok, S}; -peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) -> quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), {ok, S}. -peer_send_shutdown(Stream, _Flags, S) -> +-spec peer_send_shutdown(stream_handle(), undefined, cb_state()) -> cb_ret(). +peer_send_shutdown(Stream, undefined, S) -> ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0), {ok, S}. +-spec send_complete(stream_handle(), IsCanceled :: boolean(), cb_state()) -> cb_ret(). send_complete(_Stream, false, S) -> {ok, S}; send_complete(_Stream, true = _IsCanceled, S) -> {ok, S}. +-spec send_shutdown_complete(stream_handle(), error_code(), cb_state()) -> cb_ret(). send_shutdown_complete(_Stream, _Flags, S) -> {ok, S}. -handle_stream_data( - Stream, - Bin, - _Flags, - #{ - is_unidir := false, - channel := undefined, - data_queue := Queue, - stream := Stream - } = State -) when is_binary(Bin) -> - {ok, State#{data_queue := [Bin | Queue]}}; +-spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_state()) -> + cb_ret(). handle_stream_data( _Stream, Bin, @@ -145,6 +142,7 @@ handle_stream_data( task_queue := TQ } = State ) when + %% assert get stream data only after channel is created Channel =/= undefined -> {MQTTPackets, NewPS} = parse_incoming(list_to_binary(lists:reverse([Bin | QueuedData])), PS), @@ -157,25 +155,12 @@ handle_stream_data( ), {{continue, handle_appl_msg}, State#{parse_state := NewPS, task_queue := NewTQ}}. -%% Reserved for unidi streams -%% handle_stream_data(Stream, Bin, _Flags, #{is_unidir := true, peer_stream := PeerStream, conn := Conn} = State) -> -%% case PeerStream of -%% undefined -> -%% {ok, StreamProc} = quicer_stream:start_link(?MODULE, Conn, -%% [ {open_flag, ?QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL} -%% , {is_local, true} -%% ]), -%% {ok, _} = quicer_stream:send(StreamProc, Bin), -%% {ok, State#{peer_stream := StreamProc}}; -%% StreamProc when is_pid(StreamProc) -> -%% {ok, _} = quicer_stream:send(StreamProc, Bin), -%% {ok, State} -%% end. - +-spec passive(stream_handle(), undefined, cb_state()) -> cb_ret(). passive(Stream, undefined, S) -> quicer:setopt(Stream, active, 10), {ok, S}. +-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_state()) -> cb_ret(). stream_closed( _Stream, #{ @@ -197,28 +182,20 @@ stream_closed( -> {stop, normal, S}. +-spec handle_call(Request :: term(), From :: {pid(), term()}, cb_state()) -> cb_ret(). handle_call(Call, _From, S) -> - case do_handle_call(Call, S) of - {ok, NewS} -> - {reply, ok, NewS}; - {error, Reason, NewS} -> - {reply, {error, Reason}, NewS}; - {{continue, _} = Cont, NewS} -> - {reply, ok, NewS, Cont}; - {hibernate, NewS} -> - {reply, ok, NewS, hibernate}; - {stop, Reason, NewS} -> - {stop, Reason, {stopped, Reason}, NewS} - end. + do_handle_call(Call, S). +-spec handle_continue(Continue :: term(), cb_state()) -> cb_ret(). handle_continue(handle_appl_msg, #{task_queue := Q} = S) -> case queue:out(Q) of {{value, Item}, Q2} -> do_handle_appl_msg(Item, S#{task_queue := Q2}); - {empty, Q} -> + {empty, _Q} -> {ok, S} end. +%%% Internals do_handle_appl_msg( {outgoing, Packets}, #{ @@ -248,7 +225,7 @@ do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) -> with_channel(handle_in, [FE], S); do_handle_appl_msg({close, Reason}, S) -> - %% @TODO shall we abort shutdown or graceful shutdown? + %% @TODO shall we abort shutdown or graceful shutdown here? with_channel(handle_info, [{sock_closed, Reason}], S); do_handle_appl_msg({event, updated}, S) -> %% Data stream don't care about connection state changes. @@ -294,7 +271,6 @@ with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when }} end. -%%% Internals handle_outgoing(#mqtt_packet{} = P, S) -> handle_outgoing([P], S); handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir := false}) when @@ -373,7 +349,7 @@ init_state(Stream, Connection, OpenFlags, PS) -> task_queue => queue:new() }. --spec do_handle_call(term(), quicer_stream:cb_state()) -> quicer_stream:cb_ret(). +-spec do_handle_call(term(), cb_state()) -> cb_ret(). do_handle_call( {activate, {PS, Serialize, Channel}}, #{ @@ -386,7 +362,7 @@ do_handle_call( %% We use quic protocol for flow control, and we don't check return val case quicer:setopt(Stream, active, true) of ok -> - {ok, NewS}; + {reply, ok, NewS}; {error, E} -> ?SLOG(error, #{msg => "set stream active failed", error => E}), {stop, E, NewS} @@ -484,3 +460,6 @@ is_datastream_out_pkt(#mqtt_packet{header = #mqtt_packet_header{type = Type}}) w true; is_datastream_out_pkt(_) -> false. +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 88cf4b7c3..a8ef7d41d 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -17,8 +17,12 @@ %% MQTT/QUIC Stream -module(emqx_quic_stream). +-ifndef(BUILD_WITHOUT_QUIC). + -behaviour(quicer_remote_stream). +-include("logger.hrl"). + %% emqx transport Callbacks -export([ type/1, @@ -33,31 +37,14 @@ sockname/1, peercert/1 ]). - --include("logger.hrl"). --ifndef(BUILD_WITHOUT_QUIC). -include_lib("quicer/include/quicer.hrl"). --else. -%% STREAM SHUTDOWN FLAGS --define(QUIC_STREAM_SHUTDOWN_FLAG_NONE, 0). -% Cleanly closes the send path. --define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 1). -% Abruptly closes the send path. --define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 2). -% Abruptly closes the receive path. --define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 4). -% Abruptly closes both send and receive paths. --define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 6). --define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE, 8). --endif. --type cb_ret() :: gen_statem:event_handler_result(). --type cb_data() :: emqtt_quic:cb_data(). +-type cb_ret() :: quicer_stream:cb_ret(). +-type cb_data() :: quicer_stream:cb_state(). -type connection_handle() :: quicer:connection_handle(). -type stream_handle() :: quicer:stream_handle(). -export([ - new_stream/3, send_complete/3, peer_send_shutdown/3, peer_send_aborted/3, @@ -79,13 +66,8 @@ }. %% for accepting --spec wait - ({pid(), connection_handle(), socket_info()}) -> - {ok, socket()} | {error, enotconn}; - %% For handover - ({pid(), connection_handle(), stream_handle(), socket_info()}) -> - {ok, socket()} | {error, any()}. - +-spec wait({pid(), connection_handle(), socket_info()}) -> + {ok, socket()} | {error, enotconn}. %%% For Accepting New Remote Stream wait({ConnOwner, Conn, ConnInfo}) -> {ok, Conn} = quicer:async_accept_stream(Conn, []), @@ -105,15 +87,8 @@ wait({ConnOwner, Conn, ConnInfo}) -> {'EXIT', ConnOwner, _Reason} -> {error, enotconn} end. -%% UNUSED, for ownership handover, -%% wait({PrevOwner, Conn, Stream, SocketInfo}) -> -%% case quicer:wait_for_handoff(PrevOwner, Stream) of -%% ok -> -%% {ok, socket(Conn, Stream, SocketInfo)}; -%% owner_down -> -%% {error, owner_down} -%% end. +-spec type(_) -> quic. type(_) -> quic. @@ -155,7 +130,7 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. -%% @TODO supply some App Error Code +%% @TODO supply some App Error Code from caller fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) -> %% handshake aborted. quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), @@ -185,15 +160,13 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of {ok, _Len} -> ok; + {error, X, Y} -> {error, {X, Y}}; Other -> Other end. %%% %%% quicer stream callbacks %%% --spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret(). -new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) -> - {stop, unimpl}. -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). peer_receive_aborted(Stream, ErrorCode, S) -> @@ -222,28 +195,12 @@ send_complete(_Stream, true = _IsCancelled, S) -> send_shutdown_complete(_Stream, _IsGraceful, S) -> {ok, S}. -%% Local stream, Unidir -%% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data()) -%% -> cb_ret(). -%% handle_stream_data(Stream, Bin, Flags, #{ is_local := true -%% , parse_state := PS} = S) -> -%% ?SLOG(debug, #{data => Bin}, Flags), -%% case parse(Bin, PS, []) of -%% {keep_state, NewPS, Packets} -> -%% quicer:setopt(Stream, active, once), -%% {keep_state, S#{parse_state := NewPS}, -%% [{next_event, cast, P } || P <- lists:reverse(Packets)]}; -%% {stop, _} = Stop -> -%% Stop -%% end; -%% %% Remote stream -%% handle_stream_data(_Stream, _Bin, _Flags, -%% #{is_local := false, is_unidir := true, conn := _Conn} = _S) -> -%% {stop, unimpl}. - -spec passive(stream_handle(), undefined, cb_data()) -> cb_ret(). passive(Stream, undefined, S) -> - quicer:setopt(Stream, active, 10), + case quicer:setopt(Stream, active, 10) of + ok -> ok; + Error -> ?SLOG(error, #{message => "set active error", error => Error}) + end, {ok, S}. -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret(). @@ -277,3 +234,7 @@ stream_closed( -spec socket(connection_handle(), stream_handle(), socket_info()) -> socket(). socket(Conn, CtrlStream, Info) when is_map(Info) -> {quic, Conn, CtrlStream, Info}. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl deleted file mode 100644 index f926c2f3e..000000000 --- a/apps/emqx/test/emqtt_quic_SUITE.erl +++ /dev/null @@ -1,1706 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqtt_quic_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("quicer/include/quicer.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - -suite() -> - [{timetrap, {seconds, 30}}]. - -all() -> - [ - {group, mstream}, - {group, shutdown}, - {group, misc} - ]. - -groups() -> - [ - {mstream, [], [{group, profiles}]}, - - {profiles, [], [ - {group, profile_low_latency}, - {group, profile_max_throughput} - ]}, - {profile_low_latency, [], [ - {group, pub_qos0}, - {group, pub_qos1}, - {group, pub_qos2} - ]}, - {profile_max_throughput, [], [ - {group, pub_qos0}, - {group, pub_qos1}, - {group, pub_qos2} - ]}, - {pub_qos0, [], [ - {group, sub_qos0}, - {group, sub_qos1}, - {group, sub_qos2} - ]}, - {pub_qos1, [], [ - {group, sub_qos0}, - {group, sub_qos1}, - {group, sub_qos2} - ]}, - {pub_qos2, [], [ - {group, sub_qos0}, - {group, sub_qos1}, - {group, sub_qos2} - ]}, - {sub_qos0, [{group, qos}]}, - {sub_qos1, [{group, qos}]}, - {sub_qos2, [{group, qos}]}, - {qos, [ - t_multi_streams_sub, - t_multi_streams_pub_5x100, - t_multi_streams_pub_parallel, - t_multi_streams_pub_parallel_no_blocking, - t_multi_streams_sub_pub_async, - t_multi_streams_sub_pub_sync, - t_multi_streams_unsub, - t_multi_streams_corr_topic, - t_multi_streams_unsub_via_other, - t_multi_streams_dup_sub, - t_multi_streams_packet_boundary, - t_multi_streams_packet_malform, - t_multi_streams_kill_sub_stream, - t_multi_streams_packet_too_large, - t_multi_streams_sub_0_rtt, - t_multi_streams_sub_0_rtt_large_payload, - t_multi_streams_sub_0_rtt_stream_data_cont, - t_conn_change_client_addr - ]}, - - {shutdown, [ - {group, graceful_shutdown}, - {group, abort_recv_shutdown}, - {group, abort_send_shutdown}, - {group, abort_send_recv_shutdown} - ]}, - - {graceful_shutdown, [ - {group, ctrl_stream_shutdown}, - {group, data_stream_shutdown} - ]}, - {abort_recv_shutdown, [ - {group, ctrl_stream_shutdown}, - {group, data_stream_shutdown} - ]}, - {abort_send_shutdown, [ - {group, ctrl_stream_shutdown}, - {group, data_stream_shutdown} - ]}, - {abort_send_recv_shutdown, [ - {group, ctrl_stream_shutdown}, - {group, data_stream_shutdown} - ]}, - - {ctrl_stream_shutdown, [ - t_multi_streams_shutdown_ctrl_stream, - t_multi_streams_shutdown_ctrl_stream_then_reconnect, - t_multi_streams_remote_shutdown, - t_multi_streams_remote_shutdown_with_reconnect - ]}, - - {data_stream_shutdown, [t_multi_streams_shutdown_data_stream]}, - {misc, [ - t_conn_silent_close, - t_client_conn_bump_streams, - t_olp_true, - t_olp_reject, - t_conn_resume, - t_conn_without_ctrl_stream - ]} - ]. - -init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - UdpPort = 14567, - start_emqx_quic(UdpPort), - %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - %% dbg:p(all, c), - %% dbg:tp(emqx_quic_connection, cx), - %% dbg:tp(emqx_quic_stream, cx), - %% dbg:tp(emqtt, cx), - %% dbg:tpl(emqtt_quic_stream, cx), - %% dbg:tpl(emqx_quic_stream, cx), - %% dbg:tpl(emqx_quic_data_stream, cx), - %% dbg:tpl(emqtt, cx), - [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. - -end_per_suite(_) -> - ok. - -init_per_group(pub_qos0, Config) -> - [{pub_qos, 0} | Config]; -init_per_group(sub_qos0, Config) -> - [{sub_qos, 0} | Config]; -init_per_group(pub_qos1, Config) -> - [{pub_qos, 1} | Config]; -init_per_group(sub_qos1, Config) -> - [{sub_qos, 1} | Config]; -init_per_group(pub_qos2, Config) -> - [{pub_qos, 2} | Config]; -init_per_group(sub_qos2, Config) -> - [{sub_qos, 2} | Config]; -init_per_group(abort_send_shutdown, Config) -> - [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config]; -init_per_group(abort_recv_shutdown, Config) -> - [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config]; -init_per_group(abort_send_recv_shutdown, Config) -> - [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config]; -init_per_group(graceful_shutdown, Config) -> - [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config]; -init_per_group(profile_max_throughput, Config) -> - quicer:reg_open(quic_execution_profile_type_max_throughput), - Config; -init_per_group(profile_low_latency, Config) -> - quicer:reg_open(quic_execution_profile_low_latency), - Config; -init_per_group(_, Config) -> - Config. - -end_per_group(_, Config) -> - Config. - -init_per_testcase(_, Config) -> - emqx_common_test_helpers:start_apps([]), - Config. - -t_quic_sock(Config) -> - Port = 4567, - SslOpts = [ - {cert, certfile(Config)}, - {key, keyfile(Config)}, - {idle_timeout_ms, 10000}, - % QUIC_SERVER_RESUME_AND_ZERORTT - {server_resumption_level, 2}, - {peer_bidi_stream_count, 10}, - {alpn, ["mqtt"]} - ], - Server = quic_server:start_link(Port, SslOpts), - timer:sleep(500), - {ok, Sock} = emqtt_quic:connect( - "localhost", - Port, - [{alpn, ["mqtt"]}, {active, false}], - 3000 - ), - send_and_recv_with(Sock), - ok = emqtt_quic:close(Sock), - quic_server:stop(Server). - -t_quic_sock_fail(_Config) -> - Port = 4567, - Error1 = - {error, - {transport_down, #{ - error => 2, - status => connection_refused - }}}, - Error2 = {error, {transport_down, #{error => 1, status => unreachable}}}, - case - emqtt_quic:connect( - "localhost", - Port, - [{alpn, ["mqtt"]}, {active, false}], - 3000 - ) - of - Error1 -> - ok; - Error2 -> - ok; - Other -> - ct:fail("unexpected return ~p", [Other]) - end. - -t_0_rtt(Config) -> - Port = 4568, - SslOpts = [ - {cert, certfile(Config)}, - {key, keyfile(Config)}, - {idle_timeout_ms, 10000}, - % QUIC_SERVER_RESUME_AND_ZERORTT - {server_resumption_level, 2}, - {peer_bidi_stream_count, 10}, - {alpn, ["mqtt"]} - ], - Server = quic_server:start_link(Port, SslOpts), - timer:sleep(500), - {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( - "localhost", - Port, - [ - {alpn, ["mqtt"]}, - {active, false}, - {quic_event_mask, 1} - ], - 3000 - ), - send_and_recv_with(Sock), - ok = emqtt_quic:close(Sock), - NST = - receive - {quic, nst_received, Conn, Ticket} -> - Ticket - end, - {ok, Sock2} = emqtt_quic:connect( - "localhost", - Port, - [ - {alpn, ["mqtt"]}, - {active, false}, - {nst, NST} - ], - 3000 - ), - send_and_recv_with(Sock2), - ok = emqtt_quic:close(Sock2), - quic_server:stop(Server). - -t_0_rtt_fail(Config) -> - Port = 4569, - SslOpts = [ - {cert, certfile(Config)}, - {key, keyfile(Config)}, - {idle_timeout_ms, 10000}, - % QUIC_SERVER_RESUME_AND_ZERORTT - {server_resumption_level, 2}, - {peer_bidi_stream_count, 10}, - {alpn, ["mqtt"]} - ], - Server = quic_server:start_link(Port, SslOpts), - timer:sleep(500), - {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( - "localhost", - Port, - [ - {alpn, ["mqtt"]}, - {active, false}, - {quic_event_mask, 1} - ], - 3000 - ), - send_and_recv_with(Sock), - ok = emqtt_quic:close(Sock), - <<_Head:16, Left/binary>> = - receive - {quic, nst_received, Conn, Ticket} when is_binary(Ticket) -> - Ticket - end, - - Error = {error, {not_found, invalid_parameter}}, - Error = emqtt_quic:connect( - "localhost", - Port, - [ - {alpn, ["mqtt"]}, - {active, false}, - {nst, Left} - ], - 3000 - ), - quic_server:stop(Server). - -t_multi_streams_sub(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of - ok when PubQos == 0 -> ok; - {ok, _} -> ok - end, - receive - {publish, #{ - client_pid := C, - payload := <<"qos 2 1">>, - qos := RecQos, - topic := Topic - }} -> - ok; - Other -> - ct:fail("unexpected recv ~p", [Other]) - after 100 -> - ct:fail("not received") - end, - ok = emqtt:disconnect(C). - -t_multi_streams_pub_5x100(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - - PubVias = lists:map( - fun(_N) -> - {ok, Via} = emqtt:start_data_stream(C, []), - Via - end, - lists:seq(1, 5) - ), - [ - begin - case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of - ok when PubQos == 0 -> ok; - {ok, _} -> ok - end, - 0 == (N rem 10) andalso timer:sleep(10) - end - || N <- lists:seq(1, 100), PVia <- PubVias - ], - ?assert(timeout =/= recv_pub(500)), - ok = emqtt:disconnect(C). - -t_multi_streams_pub_parallel(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - PktId2 = calc_pkt_id(RecQos, 2), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - <<"stream data 1">>, - [{qos, PubQos}], - undefined - ), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - <<"stream data 2">>, - [{qos, PubQos}], - undefined - ), - PubRecvs = recv_pub(2), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data", _/binary>>, - qos := RecQos, - topic := Topic - }}, - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<"stream data", _/binary>>, - qos := RecQos, - topic := Topic - }} - ], - PubRecvs - ), - Payloads = [P || {publish, #{payload := P}} <- PubRecvs], - ?assert( - [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse - [<<"stream data 2">>, <<"stream data 1">>] == Payloads - ), - ok = emqtt:disconnect(C). - -%% @doc test two pub streams, one send incomplete MQTT packet() can not block another. -t_multi_streams_pub_parallel_no_blocking(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId2 = calc_pkt_id(RecQos, 1), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - Drop = <<"stream data 1">>, - meck:new(emqtt_quic, [passthrough, no_history]), - meck:expect(emqtt_quic, send, fun(Sock, IoList) -> - case lists:last(IoList) == Drop of - true -> - ct:pal("meck droping ~p", [Drop]), - meck:passthrough([Sock, IoList -- [Drop]]); - false -> - meck:passthrough([Sock, IoList]) - end - end), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - Drop, - [{qos, PubQos}], - undefined - ), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - <<"stream data 2">>, - [{qos, PubQos}], - undefined - ), - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<"stream data 2">>, - qos := RecQos, - topic := Topic - }} - ], - PubRecvs - ), - meck:unload(emqtt_quic), - ?assertEqual(timeout, recv_pub(1)), - ok = emqtt:disconnect(C). - -t_multi_streams_packet_boundary(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - PktId2 = calc_pkt_id(RecQos, 2), - PktId3 = calc_pkt_id(RecQos, 3), - Topic = atom_to_binary(?FUNCTION_NAME), - - %% make quicer to batch job - quicer:reg_open(quic_execution_profile_type_max_throughput), - - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - - {ok, PubVia} = emqtt:start_data_stream(C, []), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - <<"stream data 1">>, - [{qos, PubQos}], - undefined - ), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - <<"stream data 2">>, - [{qos, PubQos}], - undefined - ), - LargePart3 = binary:copy(<<"stream data3">>, 2000), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - LargePart3, - [{qos, PubQos}], - undefined - ), - PubRecvs = recv_pub(3), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 1">>, - qos := RecQos, - topic := Topic - }}, - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<"stream data 2">>, - qos := RecQos, - topic := Topic - }}, - {publish, #{ - client_pid := C, - packet_id := PktId3, - payload := LargePart3, - qos := RecQos, - topic := Topic - }} - ], - PubRecvs - ), - ok = emqtt:disconnect(C). - -%% @doc test that one malformed stream will not close the entire connection -t_multi_streams_packet_malform(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - PktId2 = calc_pkt_id(RecQos, 2), - PktId3 = calc_pkt_id(RecQos, 3), - Topic = atom_to_binary(?FUNCTION_NAME), - - %% make quicer to batch job - quicer:reg_open(quic_execution_profile_type_max_throughput), - - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - - {ok, PubVia} = emqtt:start_data_stream(C, []), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - <<"stream data 1">>, - [{qos, PubQos}], - undefined - ), - - {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []), - {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), - - ok = emqtt:publish_async( - C, - PubVia, - Topic, - <<"stream data 2">>, - [{qos, PubQos}], - undefined - ), - LargePart3 = binary:copy(<<"stream data3">>, 2000), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - LargePart3, - [{qos, PubQos}], - undefined - ), - PubRecvs = recv_pub(3), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 1">>, - qos := RecQos, - topic := Topic - }}, - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<"stream data 2">>, - qos := RecQos, - topic := Topic - }}, - {publish, #{ - client_pid := C, - packet_id := PktId3, - payload := LargePart3, - qos := RecQos, - topic := Topic - }} - ], - PubRecvs - ), - - case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of - {ok, 10} -> ok; - {error, cancelled} -> ok; - {error, stm_send_error, aborted} -> ok - end, - - timer:sleep(200), - ?assert(is_list(emqtt:info(C))), - - {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>), - - timer:sleep(200), - ?assert(is_list(emqtt:info(C))), - - ok = emqtt:disconnect(C). - -t_multi_streams_packet_too_large(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - Topic = atom_to_binary(?FUNCTION_NAME), - meck:new(emqx_frame, [passthrough, no_history]), - ok = meck:expect( - emqx_frame, - serialize_opts, - fun(#mqtt_packet_connect{proto_ver = ProtoVer}) -> - #{version => ProtoVer, max_size => 1024} - end - ), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - - {ok, PubVia} = emqtt:start_data_stream(C, []), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - binary:copy(<<"stream data 1">>, 1024), - [{qos, PubQos}], - undefined - ), - timeout = recv_pub(1), - ?assert(is_list(emqtt:info(C))), - ok = meck:unload(emqx_frame), - ok = emqtt:disconnect(C). - -t_conn_change_client_addr(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), - - {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - <<"stream data 1">>, - [{qos, PubQos}], - undefined - ), - - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := _PktId1, - payload := <<"stream data 1">>, - qos := RecQos - }} - ], - recv_pub(1) - ), - NewPort = select_port(), - {ok, OldAddr} = quicer:sockname(Conn), - ?assertEqual( - ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort)) - ), - {ok, NewAddr} = quicer:sockname(Conn), - ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]), - ?assertNotEqual(OldAddr, NewAddr), - ?assert(is_list(emqtt:info(C))), - ok = emqtt:disconnect(C). - -t_multi_streams_sub_pub_async(Config) -> - Topic = atom_to_binary(?FUNCTION_NAME), - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - <<"stream data 1">>, - [{qos, PubQos}], - undefined - ), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic2, - <<"stream data 2">>, - [{qos, PubQos}], - undefined - ), - PubRecvs = recv_pub(2), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data", _/binary>>, - qos := RecQos - }}, - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data", _/binary>>, - qos := RecQos - }} - ], - PubRecvs - ), - Payloads = [P || {publish, #{payload := P}} <- PubRecvs], - ?assert( - [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse - [<<"stream data 2">>, <<"stream data 1">>] == Payloads - ), - ok = emqtt:disconnect(C). - -t_multi_streams_sub_pub_sync(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - Via1 = undefined, - ok; - {ok, #{reason_code := 0, via := Via1}} -> - ok - end, - case - emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [ - {qos, PubQos} - ]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := Via2}} -> - ?assert(Via1 =/= Via2), - ok - end, - ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]), - PubRecvs = recv_pub(2), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 3">>, - qos := RecQos, - via := SVia1 - }}, - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 4">>, - qos := RecQos, - via := SVia2 - }} - ], - lists:sort(PubRecvs) - ), - ok = emqtt:disconnect(C). - -t_multi_streams_dup_sub(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - - #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value( - extra, emqtt:info(C) - ), - ?assertEqual(2, length(emqx_broker:subscribers(Topic))), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := 0, via := _Via1}} -> - ok - end, - PubRecvs = recv_pub(2), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 3">>, - qos := RecQos - }}, - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<"stream data 3">>, - qos := RecQos - }} - ], - lists:sort(PubRecvs) - ), - - RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs], - - ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]), - %% Can recv in any order - ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias), - - %% Shutdown one stream - quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500), - timer:sleep(100), - - ?assertEqual(1, length(emqx_broker:subscribers(Topic))), - - ok = emqtt:disconnect(C). - -t_multi_streams_corr_topic(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - PktId2 = calc_pkt_id(RecQos, 2), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := 0, via := _Via}} -> - ok - end, - - #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), - ?assert(PubVia =/= SubVia), - - case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := PubVia}} -> ok - end, - PubRecvs = recv_pub(2), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }}, - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<6, 7, 8, 9>>, - qos := RecQos - }} - ], - PubRecvs - ), - ok = emqtt:disconnect(C). - -t_multi_streams_unsub(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := 0, via := _PVia}} -> - ok - end, - - #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), - ?assert(PubVia =/= SubVia), - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - emqtt:unsubscribe_via(C, SubVia, Topic), - - case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} -> - ok - end, - - timeout = recv_pub(1), - ok = emqtt:disconnect(C). - -t_multi_streams_kill_sub_stream(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - [TopicStreamOwner] = emqx_broker:subscribers(Topic), - exit(TopicStreamOwner, kill), - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 -> - ok - end, - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> - ok; - {ok, #{reason_code := 0, via := _PVia2}} -> - ok - end, - - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - topic := Topic2, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - recv_pub(1) - ), - ?assertEqual(timeout, recv_pub(1)), - ok. - -t_multi_streams_unsub_via_other(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - PktId2 = calc_pkt_id(RecQos, 2), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), - - %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed" - {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic), - - case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia2}} -> ok - end, - - PubRecvs2 = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId2, - payload := <<6, 7, 8, 9>>, - qos := RecQos - }} - ], - PubRecvs2 - ), - ok = emqtt:disconnect(C). - -t_multi_streams_shutdown_data_stream(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - ?assert(SVia =/= SVia2), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), - {quic, _Conn, DataStream} = PubVia, - quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), - timer:sleep(500), - %% Still alive - ?assert(is_list(emqtt:info(C))). - -t_multi_streams_shutdown_ctrl_stream(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - unlink(C), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000), - timer:sleep(500), - %% Client should be closed - ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). - -t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> - erlang:process_flag(trap_exit, true), - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {reconnect, true}, - %% speedup test - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - ?assert(SVia2 =/= SVia), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100), - timer:sleep(200), - %% Client should be closed - ?assert(is_list(emqtt:info(C))). - -t_multi_streams_remote_shutdown(Config) -> - erlang:process_flag(trap_exit, true), - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {reconnect, false}, - %% speedup test - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - ?assert(SVia2 =/= SVia), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - - ok = stop_emqx(), - - timer:sleep(200), - start_emqx_quic(?config(port, Config)), - - %% Client should be closed - ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). - -t_multi_streams_remote_shutdown_with_reconnect(Config) -> - erlang:process_flag(trap_exit, true), - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - PktId1 = calc_pkt_id(RecQos, 1), - - Topic = atom_to_binary(?FUNCTION_NAME), - Topic2 = <>, - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {reconnect, true}, - %% speedup test - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ - {Topic2, [{qos, SubQos}]} - ]), - - ?assert(SVia2 =/= SVia), - - case - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) - of - ok when PubQos == 0 -> ok; - {ok, #{reason_code := 0, via := _PVia}} -> ok - end, - - PubRecvs = recv_pub(1), - ?assertMatch( - [ - {publish, #{ - client_pid := C, - packet_id := PktId1, - payload := <<1, 2, 3, 4, 5>>, - qos := RecQos - }} - ], - PubRecvs - ), - - {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - - ok = stop_emqx(), - - timer:sleep(200), - - start_emqx_quic(?config(port, Config)), - %% Client should be closed - ?assert(is_list(emqtt:info(C))). - -t_conn_silent_close(Config) -> - erlang:process_flag(trap_exit, true), - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - %% quic idle timeout + 1s - timer:sleep(16000), - Topic = atom_to_binary(?FUNCTION_NAME), - ?assertException( - exit, - noproc, - emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}]) - ). - -t_client_conn_bump_streams(Config) -> - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)), - ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}). - -t_olp_true(Config) -> - meck:new(emqx_olp, [passthrough, no_history]), - ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - ok = meck:unload(emqx_olp). - -t_olp_reject(Config) -> - erlang:process_flag(trap_exit, true), - emqx_config:put_zone_conf(default, [overload_protection, enable], true), - meck:new(emqx_olp, [passthrough, no_history]), - ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - ?assertEqual( - {error, - {transport_down, #{ - error => 346, - status => - user_canceled - }}}, - emqtt:quic_connect(C) - ), - ok = meck:unload(emqx_olp), - emqx_config:put_zone_conf(default, [overload_protection, enable], false). - -t_conn_resume(Config) -> - erlang:process_flag(trap_exit, true), - {ok, C0} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - - {ok, _} = emqtt:quic_connect(C0), - #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), - emqtt:disconnect(C0), - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5}, - {nst, NST} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - Cid = proplists:get_value(clientid, emqtt:info(C)), - ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). - -t_conn_without_ctrl_stream(Config) -> - erlang:process_flag(trap_exit, true), - {ok, Conn} = quicer:connect( - {127, 0, 0, 1}, - ?config(port, Config), - [{alpn, ["mqtt"]}, {verify, none}], - 3000 - ), - receive - {quic, transport_shutdown, Conn, _} -> ok - end. - -t_data_stream_race_ctrl_stream(Config) -> - erlang:process_flag(trap_exit, true), - {ok, C0} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5} - | Config - ]), - {ok, _} = emqtt:quic_connect(C0), - #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), - emqtt:disconnect(C0), - {ok, C} = emqtt:start_link([ - {proto_ver, v5}, - {connect_timeout, 5}, - {nst, NST} - | Config - ]), - {ok, _} = emqtt:quic_connect(C), - Cid = proplists:get_value(clientid, emqtt:info(C)), - ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). - -t_multi_streams_sub_0_rtt(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - Topic = atom_to_binary(?FUNCTION_NAME), - {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C0), - {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - ok = emqtt:open_quic_connection(C), - ok = emqtt:quic_mqtt_connect(C), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - #{}, - <<"qos 2 1">>, - [{qos, PubQos}], - infinity, - fun(_) -> ok end - ), - {ok, _} = emqtt:quic_connect(C), - receive - {publish, #{ - client_pid := C0, - payload := <<"qos 2 1">>, - qos := RecQos, - topic := Topic - }} -> - ok; - Other -> - ct:fail("unexpected recv ~p", [Other]) - after 100 -> - ct:fail("not received") - end, - ok = emqtt:disconnect(C), - ok = emqtt:disconnect(C0). - -t_multi_streams_sub_0_rtt_large_payload(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - Topic = atom_to_binary(?FUNCTION_NAME), - Payload = binary:copy(<<"qos 2 1">>, 1600), - {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C0), - {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - ok = emqtt:open_quic_connection(C), - ok = emqtt:quic_mqtt_connect(C), - ok = emqtt:publish_async( - C, - {new_data_stream, []}, - Topic, - #{}, - Payload, - [{qos, PubQos}], - infinity, - fun(_) -> ok end - ), - {ok, _} = emqtt:quic_connect(C), - receive - {publish, #{ - client_pid := C0, - payload := Payload, - qos := RecQos, - topic := Topic - }} -> - ok; - Other -> - ct:fail("unexpected recv ~p", [Other]) - after 100 -> - ct:fail("not received") - end, - ok = emqtt:disconnect(C), - ok = emqtt:disconnect(C0). - -%% @doc verify data stream can continue after 0-RTT handshake -t_multi_streams_sub_0_rtt_stream_data_cont(Config) -> - PubQos = ?config(pub_qos, Config), - SubQos = ?config(sub_qos, Config), - RecQos = calc_qos(PubQos, SubQos), - Topic = atom_to_binary(?FUNCTION_NAME), - Payload = binary:copy(<<"qos 2 1">>, 1600), - {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), - {ok, _} = emqtt:quic_connect(C0), - {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ - {Topic, [{qos, SubQos}]} - ]), - {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), - ok = emqtt:open_quic_connection(C), - ok = emqtt:quic_mqtt_connect(C), - {ok, PubVia} = emqtt:start_data_stream(C, []), - ok = emqtt:publish_async( - C, - PubVia, - Topic, - #{}, - Payload, - [{qos, PubQos}], - infinity, - fun(_) -> ok end - ), - {ok, _} = emqtt:quic_connect(C), - receive - {publish, #{ - client_pid := C0, - payload := Payload, - qos := RecQos, - topic := Topic - }} -> - ok; - Other -> - ct:fail("unexpected recv ~p", [Other]) - after 100 -> - ct:fail("not received") - end, - Payload2 = <<"2nd part", Payload/binary>>, - ok = emqtt:publish_async( - C, - PubVia, - Topic, - #{}, - Payload2, - [{qos, PubQos}], - infinity, - fun(_) -> ok end - ), - receive - {publish, #{ - client_pid := C0, - payload := Payload2, - qos := RecQos, - topic := Topic - }} -> - ok; - Other2 -> - ct:fail("unexpected recv ~p", [Other2]) - after 100 -> - ct:fail("not received") - end, - ok = emqtt:disconnect(C), - ok = emqtt:disconnect(C0). - -%%-------------------------------------------------------------------- -%% Helper functions -%%-------------------------------------------------------------------- -send_and_recv_with(Sock) -> - {ok, {IP, _}} = emqtt_quic:sockname(Sock), - ?assert(lists:member(tuple_size(IP), [4, 8])), - ok = emqtt_quic:send(Sock, <<"ping">>), - emqtt_quic:setopts(Sock, [{active, false}]), - {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0), - ok = emqtt_quic:setopts(Sock, [{active, 100}]), - {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]), - %% connection level counters, not stream level - [{send_cnt, _}, {recv_cnt, _}] = Stats. - -certfile(Config) -> - filename:join([test_dir(Config), "certs", "test.crt"]). - -keyfile(Config) -> - filename:join([test_dir(Config), "certs", "test.key"]). - -test_dir(Config) -> - filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))). - -recv_pub(Count) -> - recv_pub(Count, []). - -recv_pub(0, Acc) -> - lists:reverse(Acc); -recv_pub(Count, Acc) -> - receive - {publish, _Prop} = Pub -> - recv_pub(Count - 1, [Pub | Acc]) - after 100 -> - timeout - end. - -all_tc() -> - code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")), - emqx_common_test_helpers:all(?MODULE). - --spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2. -calc_qos(PubQos, SubQos) -> - if - PubQos > SubQos -> - SubQos; - SubQos > PubQos -> - PubQos; - true -> - PubQos - end. --spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer(). -calc_pkt_id(0, _Id) -> - undefined; -calc_pkt_id(1, Id) -> - Id; -calc_pkt_id(2, Id) -> - Id. - --spec start_emqx_quic(inet:port_number()) -> ok. -start_emqx_quic(UdpPort) -> - emqx_common_test_helpers:start_apps([]), - application:ensure_all_started(quicer), - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort). - --spec stop_emqx() -> ok. -stop_emqx() -> - emqx_common_test_helpers:stop_apps([]). - -%% select a random port picked by OS --spec select_port() -> inet:port_number(). -select_port() -> - {ok, S} = gen_udp:open(0, [{reuseaddr, true}]), - {ok, {_, Port}} = inet:sockname(S), - gen_udp:close(S), - case os:type() of - {unix, darwin} -> - %% in MacOS, still get address_in_use after close port - timer:sleep(500); - _ -> - skip - end, - ct:pal("select port: ~p", [Port]), - Port. diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 5a9abc7f4..0199bbc10 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -79,19 +79,6 @@ end_per_group(_Group, _Config) -> init_per_suite(Config) -> %% Start Apps - dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - dbg:p(all, c), - dbg:tp(emqx_quic_connection, cx), - dbg:tp(quicer_connection, cx), - %% dbg:tp(emqx_quic_stream, cx), - %% dbg:tp(emqtt_quic, cx), - %% dbg:tp(emqtt, cx), - %% dbg:tp(emqtt_quic_stream, cx), - %% dbg:tp(emqtt_quic_connection, cx), - %% dbg:tp(emqx_cm, open_session, cx), - %% dbg:tpl(emqx_cm, lookup_channels, cx), - %% dbg:tpl(emqx_cm, register_channel, cx), - %% dbg:tpl(emqx_cm, unregister_channel, cx), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), Config. diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index bb19092f7..b6d3c661c 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -13,178 +13,1748 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_quic_multistreams_SUITE). -compile(export_all). +-compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). --define(TOPICS, [ - <<"TopicA">>, - <<"TopicA/B">>, - <<"Topic/C">>, - <<"TopicA/C">>, - <<"/TopicA">> -]). - -%%-------------------------------------------------------------------- -%% @spec suite() -> Info -%% Info = [tuple()] -%% @end -%%-------------------------------------------------------------------- suite() -> [{timetrap, {seconds, 30}}]. -%%-------------------------------------------------------------------- -%% @spec init_per_suite(Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% Config0 = Config1 = [tuple()] -%% Reason = term() -%% @end -%%-------------------------------------------------------------------- -init_per_suite(Config) -> - UdpPort = 1884, - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), - %% @TODO remove - emqx_logger:set_log_level(debug), - - dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - dbg:p(all, c), - - %dbg:tp(emqx_quic_stream, cx), - %% dbg:tp(quicer_stream, cx), - %% dbg:tp(emqx_quic_data_stream, cx), - %% dbg:tp(emqx_channel, cx), - %% dbg:tp(emqx_packet,check,cx), - %% dbg:tp(emqx_frame,parse,cx), - %dbg:tp(emqx_quic_connection, cx), - [{port, UdpPort}, {conn_fun, quic_connect} | Config]. - -%%-------------------------------------------------------------------- -%% @spec end_per_suite(Config0) -> term() | {save_config,Config1} -%% Config0 = Config1 = [tuple()] -%% @end -%%-------------------------------------------------------------------- -end_per_suite(_Config) -> - ok. - -%%-------------------------------------------------------------------- -%% @spec init_per_group(GroupName, Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% GroupName = atom() -%% Config0 = Config1 = [tuple()] -%% Reason = term() -%% @end -%%-------------------------------------------------------------------- -init_per_group(_GroupName, Config) -> - Config. - -%%-------------------------------------------------------------------- -%% @spec end_per_group(GroupName, Config0) -> -%% term() | {save_config,Config1} -%% GroupName = atom() -%% Config0 = Config1 = [tuple()] -%% @end -%%-------------------------------------------------------------------- -end_per_group(_GroupName, _Config) -> - ok. - -%%-------------------------------------------------------------------- -%% @spec init_per_testcase(TestCase, Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% TestCase = atom() -%% Config0 = Config1 = [tuple()] -%% Reason = term() -%% @end -%%-------------------------------------------------------------------- -init_per_testcase(_TestCase, Config) -> - Config. - -%%-------------------------------------------------------------------- -%% @spec end_per_testcase(TestCase, Config0) -> -%% term() | {save_config,Config1} | {fail,Reason} -%% TestCase = atom() -%% Config0 = Config1 = [tuple()] -%% Reason = term() -%% @end -%%-------------------------------------------------------------------- -end_per_testcase(_TestCase, _Config) -> - ok. - -%%-------------------------------------------------------------------- -%% @spec groups() -> [Group] -%% Group = {GroupName,Properties,GroupsAndTestCases} -%% GroupName = atom() -%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] -%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] -%% TestCase = atom() -%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} -%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | -%% repeat_until_any_ok | repeat_until_any_fail -%% N = integer() | forever -%% @end -%%-------------------------------------------------------------------- -groups() -> - []. - -%%-------------------------------------------------------------------- -%% @spec all() -> GroupsAndTestCases | {skip,Reason} -%% GroupsAndTestCases = [{group,GroupName} | TestCase] -%% GroupName = atom() -%% TestCase = atom() -%% Reason = term() -%% @end -%%-------------------------------------------------------------------- all() -> [ - tc_data_stream_sub + {group, mstream}, + {group, shutdown}, + {group, misc} ]. -%%-------------------------------------------------------------------- -%% @spec TestCase(Config0) -> -%% ok | exit() | {skip,Reason} | {comment,Comment} | -%% {save_config,Config1} | {skip_and_save,Reason,Config1} -%% Config0 = Config1 = [tuple()] -%% Reason = term() -%% Comment = term() -%% @end -%%-------------------------------------------------------------------- +groups() -> + [ + {mstream, [], [{group, profiles}]}, -%% @doc Test MQTT Subscribe via data_stream -tc_data_stream_sub(Config) -> - Topic = lists:nth(1, ?TOPICS), + {profiles, [], [ + {group, profile_low_latency}, + {group, profile_max_throughput} + ]}, + {profile_low_latency, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {profile_max_throughput, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {pub_qos0, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos1, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos2, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {sub_qos0, [{group, qos}]}, + {sub_qos1, [{group, qos}]}, + {sub_qos2, [{group, qos}]}, + {qos, [ + t_multi_streams_sub, + t_multi_streams_pub_5x100, + t_multi_streams_pub_parallel, + t_multi_streams_pub_parallel_no_blocking, + t_multi_streams_sub_pub_async, + t_multi_streams_sub_pub_sync, + t_multi_streams_unsub, + t_multi_streams_corr_topic, + t_multi_streams_unsub_via_other, + t_multi_streams_dup_sub, + t_multi_streams_packet_boundary, + t_multi_streams_packet_malform, + t_multi_streams_kill_sub_stream, + t_multi_streams_packet_too_large, + t_multi_streams_sub_0_rtt, + t_multi_streams_sub_0_rtt_large_payload, + t_multi_streams_sub_0_rtt_stream_data_cont, + t_conn_change_client_addr + ]}, + + {shutdown, [ + {group, graceful_shutdown}, + {group, abort_recv_shutdown}, + {group, abort_send_shutdown}, + {group, abort_send_recv_shutdown} + ]}, + + {graceful_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + + {ctrl_stream_shutdown, [ + t_multi_streams_shutdown_ctrl_stream, + t_multi_streams_shutdown_ctrl_stream_then_reconnect, + t_multi_streams_remote_shutdown, + t_multi_streams_remote_shutdown_with_reconnect + ]}, + + {data_stream_shutdown, [ + t_multi_streams_shutdown_pub_data_stream, + t_multi_streams_shutdown_sub_data_stream + ]}, + {misc, [ + t_conn_silent_close, + t_client_conn_bump_streams, + t_olp_true, + t_olp_reject, + t_conn_resume, + t_conn_without_ctrl_stream + ]} + ]. + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([]), + UdpPort = 14567, + start_emqx_quic(UdpPort), + %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + %% dbg:p(all, c), + %% dbg:tpl(quicer_stream, handle_info, c), + %% dbg:tp(emqx_quic_connection, cx), + %% dbg:tp(emqx_quic_stream, cx), + %% dbg:tp(emqtt, cx), + %% dbg:tpl(emqtt_quic_stream, cx), + %% dbg:tpl(emqx_quic_stream, cx), + %% dbg:tpl(emqx_quic_data_stream, cx), + %% dbg:tpl(emqtt, cx), + [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. + +end_per_suite(_) -> + ok. + +init_per_group(pub_qos0, Config) -> + [{pub_qos, 0} | Config]; +init_per_group(sub_qos0, Config) -> + [{sub_qos, 0} | Config]; +init_per_group(pub_qos1, Config) -> + [{pub_qos, 1} | Config]; +init_per_group(sub_qos1, Config) -> + [{sub_qos, 1} | Config]; +init_per_group(pub_qos2, Config) -> + [{pub_qos, 2} | Config]; +init_per_group(sub_qos2, Config) -> + [{sub_qos, 2} | Config]; +init_per_group(abort_send_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config]; +init_per_group(abort_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config]; +init_per_group(abort_send_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config]; +init_per_group(graceful_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config]; +init_per_group(profile_max_throughput, Config) -> + quicer:reg_open(quic_execution_profile_type_max_throughput), + Config; +init_per_group(profile_low_latency, Config) -> + quicer:reg_open(quic_execution_profile_low_latency), + Config; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(_, Config) -> + emqx_common_test_helpers:start_apps([]), + Config. + +t_quic_sock(Config) -> + Port = 4567, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, Sock} = emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + quic_server:stop(Server). + +t_quic_sock_fail(_Config) -> + Port = 4567, + Error1 = + {error, + {transport_down, #{ + error => 2, + status => connection_refused + }}}, + Error2 = {error, {transport_down, #{error => 1, status => unreachable}}}, + case + emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ) + of + Error1 -> + ok; + Error2 -> + ok; + Other -> + ct:fail("unexpected return ~p", [Other]) + end. + +t_0_rtt(Config) -> + Port = 4568, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + NST = + receive + {quic, nst_received, Conn, Ticket} -> + Ticket + end, + {ok, Sock2} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, NST} + ], + 3000 + ), + send_and_recv_with(Sock2), + ok = emqtt_quic:close(Sock2), + quic_server:stop(Server). + +t_0_rtt_fail(Config) -> + Port = 4569, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + <<_Head:16, Left/binary>> = + receive + {quic, nst_received, Conn, Ticket} when is_binary(Ticket) -> + Ticket + end, + + Error = {error, {not_found, invalid_parameter}}, + Error = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, Left} + ], + 3000 + ), + quic_server:stop(Server). + +t_multi_streams_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), {ok, _} = emqtt:quic_connect(C), - {ok, _, [1]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [{Topic, [{qos, qos1}]}]), - {ok, _, [2]} = emqtt:subscribe_via( - C, - {new_data_stream, []}, - #{}, - [{lists:nth(2, ?TOPICS), [{qos, qos2}]}] - ), - {ok, _} = emqtt:publish(C, Topic, <<"qos 2 1">>, 2), - {ok, _} = emqtt:publish(C, Topic, <<"qos 2 2">>, 2), - {ok, _} = emqtt:publish(C, Topic, <<"qos 2 3">>, 2), - Msgs = receive_messages(3), - ct:pal("recv msg: ~p", [Msgs]), - ?assertEqual(3, length(Msgs)), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + receive + {publish, #{ + client_pid := C, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, ok = emqtt:disconnect(C). -receive_messages(Count) -> - receive_messages(Count, []). +t_multi_streams_pub_5x100(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), -receive_messages(0, Msgs) -> - Msgs; -receive_messages(Count, Msgs) -> + PubVias = lists:map( + fun(_N) -> + {ok, Via} = emqtt:start_data_stream(C, []), + Via + end, + lists:seq(1, 5) + ), + CtrlVia = proplists:get_value(socket, emqtt:info(C)), + [ + begin + case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + 0 == (N rem 10) andalso timer:sleep(10) + end + || %% also publish on control stream + N <- lists:seq(1, 100), + PVia <- [CtrlVia | PubVias] + ], + ?assert(timeout =/= recv_pub(600)), + ok = emqtt:disconnect(C). + +t_multi_streams_pub_parallel(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +%% @doc test two pub streams, one send incomplete MQTT packet() can not block another. +t_multi_streams_pub_parallel_no_blocking(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId2 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + Drop = <<"stream data 1">>, + meck:new(emqtt_quic, [passthrough, no_history]), + meck:expect(emqtt_quic, send, fun(Sock, IoList) -> + case lists:last(IoList) == Drop of + true -> + ct:pal("meck droping ~p", [Drop]), + meck:passthrough([Sock, IoList -- [Drop]]); + false -> + meck:passthrough([Sock, IoList]) + end + end), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + Drop, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + meck:unload(emqtt_quic), + ?assertEqual(timeout, recv_pub(1)), + ok = emqtt:disconnect(C). + +t_multi_streams_packet_boundary(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + LargePart3 = binary:copy(<<"stream data3">>, 2000), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(3), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := LargePart3, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + ok = emqtt:disconnect(C). + +%% @doc test that one malformed stream will not close the entire connection +t_multi_streams_packet_malform(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []), + {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), + + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + LargePart3 = binary:copy(<<"stream data3">>, 2000), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(3), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := LargePart3, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + + case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of + {ok, 10} -> ok; + {error, cancelled} -> ok; + {error, stm_send_error, aborted} -> ok + end, + + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>), + + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + ok = emqtt:disconnect(C). + +t_multi_streams_packet_too_large(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + meck:new(emqx_frame, [passthrough, no_history]), + ok = meck:expect( + emqx_frame, + serialize_opts, + fun(#mqtt_packet_connect{proto_ver = ProtoVer}) -> + #{version => ProtoVer, max_size => 1024} + end + ), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + binary:copy(<<"stream data 1">>, 1024), + [{qos, PubQos}], + undefined + ), + timeout = recv_pub(1), + ?assert(is_list(emqtt:info(C))), + ok = meck:unload(emqx_frame), + ok = emqtt:disconnect(C). + +t_conn_change_client_addr(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := _PktId1, + payload := <<"stream data 1">>, + qos := RecQos + }} + ], + recv_pub(1) + ), + NewPort = select_port(), + {ok, OldAddr} = quicer:sockname(Conn), + ?assertEqual( + ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort)) + ), + {ok, NewAddr} = quicer:sockname(Conn), + ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]), + ?assertNotEqual(OldAddr, NewAddr), + ?assert(is_list(emqtt:info(C))), + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_async(Config) -> + Topic = atom_to_binary(?FUNCTION_NAME), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic2, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_sync(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + Via1 = undefined, + ok; + {ok, #{reason_code := 0, via := Via1}} -> + ok + end, + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [ + {qos, PubQos} + ]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := Via2}} -> + ?assert(Via1 =/= Via2), + ok + end, + ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos, + via := SVia1 + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 4">>, + qos := RecQos, + via := SVia2 + }} + ], + lists:sort(PubRecvs) + ), + ok = emqtt:disconnect(C). + +t_multi_streams_dup_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value( + extra, emqtt:info(C) + ), + ?assertEqual(2, length(emqx_broker:subscribers(Topic))), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via1}} -> + ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }} + ], + lists:sort(PubRecvs) + ), + + RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs], + + ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]), + %% Can recv in any order + ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias), + + %% Shutdown one stream + quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500), + timer:sleep(100), + + ?assertEqual(1, length(emqx_broker:subscribers(Topic))), + + ok = emqtt:disconnect(C). + +t_multi_streams_corr_topic(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := PubVia}} -> ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs + ), + ok = emqtt:disconnect(C). + +t_multi_streams_unsub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + emqtt:unsubscribe_via(C, SubVia, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} -> + ok + end, + + timeout = recv_pub(1), + ok = emqtt:disconnect(C). + +t_multi_streams_kill_sub_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + [TopicStreamOwner] = emqx_broker:subscribers(Topic), + exit(TopicStreamOwner, kill), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 -> + ok + end, + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia2}} -> + ok + end, + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + topic := Topic2, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + recv_pub(1) + ), + ?assertEqual(timeout, recv_pub(1)), + ok. + +t_multi_streams_unsub_via_other(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + + %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed" + {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia2}} -> ok + end, + + PubRecvs2 = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs2 + ), + ok = emqtt:disconnect(C). + +t_multi_streams_shutdown_pub_data_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia =/= SVia2), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + {quic, _Conn, DataStream} = PubVia, + quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_shutdown_sub_data_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia =/= SVia2), + {quic, _Conn, DataStream} = SVia2, + quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 500, 100), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [_PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_shutdown_ctrl_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + unlink(C), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000), + timer:sleep(500), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(200), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_remote_shutdown(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + + timer:sleep(200), + start_emqx_quic(?config(port, Config)), + + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_remote_shutdown_with_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + + timer:sleep(200), + + start_emqx_quic(?config(port, Config)), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_conn_silent_close(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + %% quic idle timeout + 1s + timer:sleep(16000), + Topic = atom_to_binary(?FUNCTION_NAME), + ?assertException( + exit, + noproc, + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}]) + ). + +t_client_conn_bump_streams(Config) -> + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)), + ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}). + +t_olp_true(Config) -> + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + ok = meck:unload(emqx_olp). + +t_olp_reject(Config) -> + erlang:process_flag(trap_exit, true), + emqx_config:put_zone_conf(default, [overload_protection, enable], true), + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + ?assertEqual( + {error, + {transport_down, #{ + error => 346, + status => + user_canceled + }}}, + emqtt:quic_connect(C) + ), + ok = meck:unload(emqx_olp), + emqx_config:put_zone_conf(default, [overload_protection, enable], false). + +t_conn_resume(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). + +t_conn_without_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, Conn} = quicer:connect( + {127, 0, 0, 1}, + ?config(port, Config), + [{alpn, ["mqtt"]}, {verify, none}], + 3000 + ), receive - {publish, Msg} -> - receive_messages(Count - 1, [Msg | Msgs]); - _Other -> - receive_messages(Count, Msgs) - after 1000 -> - Msgs + {quic, transport_shutdown, Conn, _} -> ok end. + +t_data_stream_race_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). + +t_multi_streams_sub_0_rtt(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + <<"qos 2 1">>, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +t_multi_streams_sub_0_rtt_large_payload(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +%% @doc verify data stream can continue after 0-RTT handshake +t_multi_streams_sub_0_rtt_stream_data_cont(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + Payload2 = <<"2nd part", Payload/binary>>, + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload2, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + receive + {publish, #{ + client_pid := C0, + payload := Payload2, + qos := RecQos, + topic := Topic + }} -> + ok; + Other2 -> + ct:fail("unexpected recv ~p", [Other2]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- +send_and_recv_with(Sock) -> + {ok, {IP, _}} = emqtt_quic:sockname(Sock), + ?assert(lists:member(tuple_size(IP), [4, 8])), + ok = emqtt_quic:send(Sock, <<"ping">>), + emqtt_quic:setopts(Sock, [{active, false}]), + {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0), + ok = emqtt_quic:setopts(Sock, [{active, 100}]), + {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]), + %% connection level counters, not stream level + [{send_cnt, _}, {recv_cnt, _}] = Stats. + +certfile(Config) -> + filename:join([test_dir(Config), "certs", "test.crt"]). + +keyfile(Config) -> + filename:join([test_dir(Config), "certs", "test.key"]). + +test_dir(Config) -> + filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))). + +recv_pub(Count) -> + recv_pub(Count, []). + +recv_pub(0, Acc) -> + lists:reverse(Acc); +recv_pub(Count, Acc) -> + receive + {publish, _Prop} = Pub -> + recv_pub(Count - 1, [Pub | Acc]) + after 100 -> + timeout + end. + +all_tc() -> + code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")), + emqx_common_test_helpers:all(?MODULE). + +-spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2. +calc_qos(PubQos, SubQos) -> + if + PubQos > SubQos -> + SubQos; + SubQos > PubQos -> + PubQos; + true -> + PubQos + end. +-spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer(). +calc_pkt_id(0, _Id) -> + undefined; +calc_pkt_id(1, Id) -> + Id; +calc_pkt_id(2, Id) -> + Id. + +-spec start_emqx_quic(inet:port_number()) -> ok. +start_emqx_quic(UdpPort) -> + emqx_common_test_helpers:start_apps([]), + application:ensure_all_started(quicer), + emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort). + +-spec stop_emqx() -> ok. +stop_emqx() -> + emqx_common_test_helpers:stop_apps([]). + +%% select a random port picked by OS +-spec select_port() -> inet:port_number(). +select_port() -> + {ok, S} = gen_udp:open(0, [{reuseaddr, true}]), + {ok, {_, Port}} = inet:sockname(S), + gen_udp:close(S), + case os:type() of + {unix, darwin} -> + %% in MacOS, still get address_in_use after close port + timer:sleep(500); + _ -> + skip + end, + ct:pal("select port: ~p", [Port]), + Port. From 2a6cdd9da6b1daa5242d07b3cad363cc4c68ef75 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 10 Jan 2023 16:34:25 +0100 Subject: [PATCH 031/135] test(quic): enhance large payload test --- apps/emqx/src/emqx_channel.erl | 1 - apps/emqx/src/emqx_connection.erl | 3 +- .../test/emqx_quic_multistreams_SUITE.erl | 107 ++++++++++++++++-- 3 files changed, 98 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index a12df9c64..e82adc786 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1136,7 +1136,6 @@ do_deliver(Publishes, Channel) when is_list(Publishes) -> {Packets, NChannel} = lists:foldl( fun(Publish, {Acc, Chann}) -> - %% @FIXME perf: list append with copy left list {Packets, NChann} = do_deliver(Publish, Chann), {Packets ++ Acc, NChann} end, diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index be420d65e..ff3ee81a9 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -118,7 +118,7 @@ %% limiter timers limiter_timer :: undefined | reference(), - %% QUIC conn pid if is a pid + %% QUIC conn owner pid if in use. quic_conn_pid :: maybe(pid()) }). @@ -336,7 +336,6 @@ init_state( Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg), FrameOpts = #{ - %% @TODO:q what is strict_mode? strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size]) }, diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index b6d3c661c..025790ef7 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -661,14 +661,14 @@ t_multi_streams_packet_too_large(Config) -> PubQos = ?config(pub_qos, Config), SubQos = ?config(sub_qos, Config), Topic = atom_to_binary(?FUNCTION_NAME), - meck:new(emqx_frame, [passthrough, no_history]), - ok = meck:expect( - emqx_frame, - serialize_opts, - fun(#mqtt_packet_connect{proto_ver = ProtoVer}) -> - #{version => ProtoVer, max_size => 1024} - end - ), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + + OldMax = emqx_config:get_zone_conf(default, [mqtt, max_packet_size]), + emqx_config:put_zone_conf(default, [mqtt, max_packet_size], 1000), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), {ok, _} = emqtt:quic_connect(C), {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), @@ -678,13 +678,95 @@ t_multi_streams_packet_too_large(Config) -> C, PubVia, Topic, - binary:copy(<<"stream data 1">>, 1024), + <<"stream data 1">>, [{qos, PubQos}], undefined ), + + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + + {ok, PubVia2} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia2, + Topic, + binary:copy(<<"too large">>, 200), + [{qos, PubQos}], + undefined + ), + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + timeout = recv_pub(1), + + %% send large payload on stream 1 + ok = emqtt:publish_async( + C, + PubVia, + Topic, + binary:copy(<<"too large">>, 200), + [{qos, PubQos}], + undefined + ), + timer:sleep(200), timeout = recv_pub(1), ?assert(is_list(emqtt:info(C))), - ok = meck:unload(emqx_frame), + + %% Connection could be kept + {error, stm_send_error, _} = quicer:send(via_stream(PubVia), <<1>>), + {error, stm_send_error, _} = quicer:send(via_stream(PubVia2), <<1>>), + %% We could send data over new stream + {ok, PubVia3} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia3, + Topic, + <<"stream data 3">>, + [{qos, PubQos}], + undefined + ), + [ + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := <<"stream data 3">>, + qos := RecQos, + topic := Topic + }} + ] = recv_pub(1), + timer:sleep(200), + + ?assert(is_list(emqtt:info(C))), + + emqx_config:put_zone_conf(default, [mqtt, max_packet_size], OldMax), ok = emqtt:disconnect(C). t_conn_change_client_addr(Config) -> @@ -1758,3 +1840,8 @@ select_port() -> end, ct:pal("select port: ~p", [Port]), Port. + +-spec via_stream({quic, quicer:connection_handle(), quicer:stream_handle()}) -> + quicer:stream_handle(). +via_stream({quic, _Conn, Stream}) -> + Stream. From 1692a16778731711db58ac17eea2a400f810e6d6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Jan 2023 16:24:06 +0100 Subject: [PATCH 032/135] feat(quic): handle ctrl stream normal shutdown --- apps/emqx/include/emqx_quic.hrl | 1 + apps/emqx/src/emqx_connection.erl | 3 +- apps/emqx/src/emqx_quic_connection.erl | 27 +++- apps/emqx/src/emqx_quic_stream.erl | 14 ++- .../test/emqx_quic_multistreams_SUITE.erl | 118 +++++++++++++++++- 5 files changed, 152 insertions(+), 11 deletions(-) diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl index 3366b8938..a16784d5d 100644 --- a/apps/emqx/include/emqx_quic.hrl +++ b/apps/emqx/include/emqx_quic.hrl @@ -19,6 +19,7 @@ %% MQTT Over QUIC Shutdown Error code. -define(MQTT_QUIC_CONN_NOERROR, 0). +-define(MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN, 1). -define(MQTT_QUIC_CONN_ERROR_OVERLOADED, 2). -endif. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index ff3ee81a9..2916f37bb 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -921,7 +921,8 @@ handle_info({sock_error, Reason}, State) -> false -> ok end, handle_info({sock_closed, Reason}, close_socket(State)); -handle_info({quic, Event, Handle, Prop}, State) -> +%% handle QUIC control stream events +handle_info({quic, Event, Handle, Prop}, State) when is_atom(Event) -> emqx_quic_stream:Event(Handle, Prop, State); handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 69d16cbc3..7538307e8 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -179,7 +179,13 @@ new_stream( SOpts1, Props ), - quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}), + case quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}) of + ok -> + ok; + E -> + %% Only log, keep connecion alive. + ?SLOG(error, #{message => "new stream handoff failed", stream => Stream, error => E}) + end, %% @TODO maybe keep them in `inactive_streams' {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}. @@ -200,7 +206,7 @@ transport_shutdown(_C, DownInfo, S) when is_map(DownInfo) -> %% @doc callback for handling for peer addr changed. -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret(). peer_address_changed(_C, _NewAddr, S) -> - %% @TODO update session info? + %% @TODO update conn info in emqx_quic_stream {ok, S}. %% @doc callback for handling local addr change, currently unused @@ -224,7 +230,7 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) -> %% @doc callback for handling request when remote wants for more streams %% should cope with rate limiting %% @TODO this is not going to get triggered in current version -%% for https://github.com/microsoft/msquic/issues/3120 +%% ref: https://github.com/microsoft/msquic/issues/3120 -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). peer_needs_streams(_C, undefined, S) -> ?SLOG(info, #{ @@ -240,6 +246,10 @@ handle_call( #{streams := Streams} = S ) -> [ + %% Try to activate streams individually if failed, stream will shutdown on its own. + %% we dont care about the return val here. + %% note, this is only used after control stream pass the validation. The data streams + %% that are called here are assured to be inactived (data processing hasn't been started). catch emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) || {OwnerPid, _Stream} <- Streams ], @@ -255,10 +265,15 @@ handle_call(_Req, _From, S) -> handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> case Reason of normal -> - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); + quicer:async_shutdown_connection( + Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, ?MQTT_QUIC_CONN_NOERROR + ); _ -> - %% @TODO have some reasons mappings here. - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1) + quicer:async_shutdown_connection( + Conn, + ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, + ?MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN + ) end, {ok, S}; handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) -> diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index a8ef7d41d..d1b205cf0 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/QUIC Stream +%% MQTT/QUIC control Stream -module(emqx_quic_stream). -ifndef(BUILD_WITHOUT_QUIC). @@ -38,6 +38,7 @@ peercert/1 ]). -include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_quic.hrl"). -type cb_ret() :: quicer_stream:cb_ret(). -type cb_data() :: quicer_stream:cb_state(). @@ -223,10 +224,17 @@ stream_closed( is_atom(Status) andalso is_integer(Code) -> - %% @TODO for now we fake a sock_closed for + %% For now we fake a sock_closed for %% emqx_connection:process_msg to append %% a msg to be processed - {ok, {sock_closed, Status}, S}. + Reason = + case Code of + ?MQTT_QUIC_CONN_NOERROR -> + normal; + _ -> + Status + end, + {ok, {sock_closed, Reason}, S}. %%% %%% Internals diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 025790ef7..17f4cbbc2 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -118,6 +118,8 @@ groups() -> t_multi_streams_shutdown_ctrl_stream, t_multi_streams_shutdown_ctrl_stream_then_reconnect, t_multi_streams_remote_shutdown, + t_multi_streams_emqx_ctrl_kill, + t_multi_streams_emqx_ctrl_exit_normal, t_multi_streams_remote_shutdown_with_reconnect ]}, @@ -1327,7 +1329,13 @@ t_multi_streams_shutdown_ctrl_stream(Config) -> ), {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000), + Flag = ?config(stream_shutdown_flag, Config), + AppErrorCode = + case Flag of + ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL -> 0; + _ -> 500 + end, + quicer:shutdown_stream(Ctrlstream, Flag, AppErrorCode, 1000), timer:sleep(500), %% Client should be closed ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). @@ -1384,6 +1392,114 @@ t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> %% Client should be closed ?assert(is_list(emqtt:info(C))). +t_multi_streams_emqx_ctrl_kill(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + ClientId = proplists:get_value(clientid, emqtt:info(C)), + [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId), + exit(TransPid, kill), + + timer:sleep(200), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_emqx_ctrl_exit_normal(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + ClientId = proplists:get_value(clientid, emqtt:info(C)), + [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId), + + emqx_connection:stop(TransPid), + timer:sleep(200), + %% Client exit normal. + ?assertMatch({'EXIT', {normal, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + t_multi_streams_remote_shutdown(Config) -> erlang:process_flag(trap_exit, true), PubQos = ?config(pub_qos, Config), From 98a72d40ce7232735e11bb178f128722bf4085a3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Jan 2023 16:24:37 +0100 Subject: [PATCH 033/135] fix(emqx_connection): do not raise an exception for normal shutdown --- apps/emqx/src/emqx_connection.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 2916f37bb..88c7d28e2 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -680,6 +680,12 @@ maybe_raise_exception(#{ stacktrace := Stacktrace }) -> erlang:raise(Exception, Context, Stacktrace); +maybe_raise_exception({shutdown, normal}) -> + ok; +maybe_raise_exception(normal) -> + ok; +maybe_raise_exception(shutdown) -> + ok; maybe_raise_exception(Reason) -> exit(Reason). From de810e04fd1bd0d13681a7a4da06f183a35986dd Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Jan 2023 16:53:03 +0100 Subject: [PATCH 034/135] chore(quic): clean test code --- apps/emqx/src/emqx_quic_connection.erl | 2 +- apps/emqx/src/emqx_quic_stream.erl | 8 +++++--- apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 1 - apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 10 ---------- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 7538307e8..ae195cd6b 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -23,7 +23,7 @@ -include_lib("quicer/include/quicer.hrl"). -include_lib("emqx/include/emqx_quic.hrl"). --behavior(quicer_connection). +-behaviour(quicer_connection). -export([ init/1, diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index d1b205cf0..5f7f93866 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -14,7 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/QUIC control Stream +%% MQTT over QUIC +%% multistreams: This is the control stream. +%% single stream: This is the only main stream. +%% callbacks are from emqx_connection process rather than quicer_stream -module(emqx_quic_stream). -ifndef(BUILD_WITHOUT_QUIC). @@ -66,10 +69,9 @@ _ => _ }. -%% for accepting +%%% For Accepting New Remote Stream -spec wait({pid(), connection_handle(), socket_info()}) -> {ok, socket()} | {error, enotconn}. -%%% For Accepting New Remote Stream wait({ConnOwner, Conn, ConnInfo}) -> {ok, Conn} = quicer:async_accept_stream(Conn, []), ConnOwner ! {self(), stream_acceptor_ready}, diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 0199bbc10..d3de74f72 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -65,7 +65,6 @@ init_per_group(quic, Config) -> UdpPort = 1884, emqx_common_test_helpers:start_apps([]), emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), - emqx_logger:set_log_level(debug), [{port, UdpPort}, {conn_fun, quic_connect} | Config]; init_per_group(_, Config) -> emqx_common_test_helpers:stop_apps([]), diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 17f4cbbc2..593613fcc 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -141,16 +141,6 @@ init_per_suite(Config) -> emqx_common_test_helpers:start_apps([]), UdpPort = 14567, start_emqx_quic(UdpPort), - %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - %% dbg:p(all, c), - %% dbg:tpl(quicer_stream, handle_info, c), - %% dbg:tp(emqx_quic_connection, cx), - %% dbg:tp(emqx_quic_stream, cx), - %% dbg:tp(emqtt, cx), - %% dbg:tpl(emqtt_quic_stream, cx), - %% dbg:tpl(emqx_quic_stream, cx), - %% dbg:tpl(emqx_quic_data_stream, cx), - %% dbg:tpl(emqtt, cx), [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. end_per_suite(_) -> From 88cdfcc4a6b7d8e19ebb99ef13454a55f6554678 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Jan 2023 21:04:34 +0100 Subject: [PATCH 035/135] test(quic): excl. multistream SUITE when BUILD_WITHOUT_QUIC --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 593613fcc..1cafdccd8 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -15,6 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_quic_multistreams_SUITE). +-ifndef(BUILD_WITHOUT_QUIC). + -compile(export_all). -compile(nowarn_export_all). @@ -1951,3 +1953,7 @@ select_port() -> quicer:stream_handle(). via_stream({quic, _Conn, Stream}) -> Stream. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. From 9e9ae50ab90a77b829875fc6b7a2cb56234be2cf Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Jan 2023 22:14:43 +0100 Subject: [PATCH 036/135] chore: qzhuyan/emqtt vsn 534541b --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index bf300761f..715d62226 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.7.0", override: true}, + {:emqtt, github: "qzhuyan/emqtt", tag: "1.7.1-pre", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index 76402897b..917d11ab7 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/multi-streams"}}} %% @TODO revert + , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} From 282d1a6829adaed7bbb664e3502b328d058787a7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 12 Jan 2023 14:58:45 +0100 Subject: [PATCH 037/135] ci: build dialyzer PLT with quicer, jq and bcrypt --- apps/emqx/rebar.config.script | 15 ++++++++++++++- rebar.config.erl | 25 ++++++++++++++----------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index e942e1a5c..e1afbf61c 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -26,6 +26,19 @@ end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}. +Dialyzer = fun(Config) -> + {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), + {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig), + Extra = OldExtra ++ [quicer || IsQuicSupp()], + NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig], + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, NewDialyzerConfig} + ) + end. + ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), {erl_opts, ErlOpts0} = lists:keyfind(erl_opts, 1, C), @@ -43,4 +56,4 @@ ExtraDeps = fun(C) -> ) end, -ExtraDeps(CONFIG). +Dialyzer(ExtraDeps(CONFIG)). diff --git a/rebar.config.erl b/rebar.config.erl index e99f83683..6f4371c7b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -548,17 +548,20 @@ dialyzer(Config) -> AppsToExclude = AppNames -- KnownApps, - case length(AppsToAnalyse) > 0 of - true -> - lists:keystore( - dialyzer, - 1, - Config, - {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]} - ); - false -> - Config - end. + Extra = + [bcrypt || provide_bcrypt_dep()] ++ + [jq || is_jq_supported()] ++ + [quicer || is_quicer_supported()], + NewDialyzerConfig = + OldDialyzerConfig ++ + [{exclude_apps, AppsToExclude} || length(AppsToAnalyse) > 0] ++ + [{plt_extra_apps, Extra} || length(Extra) > 0], + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, NewDialyzerConfig} + ). coveralls() -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of From 381eb8ec682ae9740398333aae0069a5ef1d8840 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 13 Jan 2023 10:02:21 +0100 Subject: [PATCH 038/135] chore(quic): fix dialyzer --- apps/emqx/src/emqx_connection.erl | 7 ++++++- apps/emqx/src/emqx_listeners.erl | 7 +++++-- apps/emqx/src/emqx_quic_connection.erl | 28 +++++++++++-------------- apps/emqx/src/emqx_quic_data_stream.erl | 8 +++---- apps/emqx/src/emqx_quic_stream.erl | 13 ++++++------ 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 88c7d28e2..9e0099414 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -929,7 +929,12 @@ handle_info({sock_error, Reason}, State) -> handle_info({sock_closed, Reason}, close_socket(State)); %% handle QUIC control stream events handle_info({quic, Event, Handle, Prop}, State) when is_atom(Event) -> - emqx_quic_stream:Event(Handle, Prop, State); + case emqx_quic_stream:Event(Handle, Prop, State) of + {{continue, Msgs}, NewState} -> + {ok, Msgs, NewState}; + Other -> + Other + end; handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 45f3b2cfd..ccf6a667a 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -386,13 +386,16 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> listener => {quic, ListenerName}, limiter => limiter(Opts) }, - StreamOpts = [{stream_callback, emqx_quic_stream}], + StreamOpts = #{ + stream_callback => emqx_quic_stream, + active => 1 + }, Id = listener_id(quic, ListenerName), add_limiter_bucket(Id, Opts), quicer:start_listener( Id, ListenOn, - {ListenOpts, ConnectionOpts, StreamOpts} + {maps:from_list(ListenOpts), ConnectionOpts, StreamOpts} ); [] -> {ok, {skipped, quic_app_missing}} diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index ae195cd6b..39d6a2c2f 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -93,7 +93,7 @@ closed(_Conn, #{is_peer_acked := _} = Prop, S) -> %% @doc handle the new incoming connecion as the connecion acceptor. -spec new_conn(quicer:connection_handle(), quicer:new_conn_props(), cb_state()) -> - {ok, cb_state()} | {error, any()}. + {ok, cb_state()} | {error, any(), cb_state()}. new_conn( Conn, #{version := _Vsn} = ConnInfo, @@ -119,7 +119,7 @@ new_conn( end; true -> emqx_metrics:inc('olp.new_conn'), - quicer:async_shutdown_connection( + _ = quicer:async_shutdown_connection( Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, ?MQTT_QUIC_CONN_ERROR_OVERLOADED @@ -129,7 +129,7 @@ new_conn( %% @doc callback when connection is connected. -spec connected(quicer:connection_handle(), quicer:connected_props(), cb_state()) -> - {ok, cb_state()} | {error, any()}. + {ok, cb_state()} | {error, any(), cb_state()}. connected(_Conn, Props, S) -> ?SLOG(debug, Props), {ok, S}. @@ -193,7 +193,7 @@ new_stream( -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). shutdown(Conn, ErrorCode, S) -> ErrorCode =/= 0 andalso ?SLOG(debug, #{error_code => ErrorCode, state => S}), - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. %% @doc callback for handling transport error, such as idle timeout @@ -245,7 +245,7 @@ handle_call( _From, #{streams := Streams} = S ) -> - [ + _ = [ %% Try to activate streams individually if failed, stream will shutdown on its own. %% we dont care about the return val here. %% note, this is only used after control stream pass the validation. The data streams @@ -263,18 +263,14 @@ handle_call(_Req, _From, S) -> %% @doc handle DOWN messages from streams. handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> - case Reason of - normal -> - quicer:async_shutdown_connection( - Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, ?MQTT_QUIC_CONN_NOERROR - ); - _ -> - quicer:async_shutdown_connection( - Conn, - ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, + Code = + case Reason of + normal -> + ?MQTT_QUIC_CONN_NOERROR; + _ -> ?MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN - ) - end, + end, + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, Code), {ok, S}; handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) -> case proplists:is_defined(Pid, Streams) of diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index e3f6b7adc..2e90edcfb 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -98,19 +98,19 @@ post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Cha {ok, S}; post_handoff(Stream, {PS, Serialize, Channel}, S) -> ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}), - quicer:setopt(Stream, active, 10), + _ = quicer:setopt(Stream, active, 10), {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. -spec peer_receive_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). peer_receive_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> %% we abort send with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -spec peer_send_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). peer_send_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> %% we abort receive with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), {ok, S}. -spec peer_send_shutdown(stream_handle(), undefined, cb_state()) -> cb_ret(). @@ -157,7 +157,7 @@ handle_stream_data( -spec passive(stream_handle(), undefined, cb_state()) -> cb_ret(). passive(Stream, undefined, S) -> - quicer:setopt(Stream, active, 10), + _ = quicer:setopt(Stream, active, 10), {ok, S}. -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_state()) -> cb_ret(). diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 5f7f93866..f60345fe9 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -136,11 +136,11 @@ getopts(_Socket, _Opts) -> %% @TODO supply some App Error Code from caller fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) -> %% handshake aborted. - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok; fast_close({quic, _Conn, Stream, _Info}) -> %% Force flush - quicer:async_shutdown_stream(Stream), + _ = quicer:async_shutdown_stream(Stream), %% @FIXME Since we shutdown the control stream, we shutdown the connection as well %% *BUT* Msquic does not flush the send buffer if we shutdown the connection after %% gracefully shutdown the stream. @@ -173,13 +173,13 @@ async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). peer_receive_aborted(Stream, ErrorCode, S) -> - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). peer_send_aborted(Stream, ErrorCode, S) -> %% we abort receive with same reason - quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), {ok, S}. -spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret(). @@ -206,7 +206,8 @@ passive(Stream, undefined, S) -> end, {ok, S}. --spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret(). +-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> + {{continue, term()}, cb_data()}. stream_closed( _Stream, #{ @@ -236,7 +237,7 @@ stream_closed( _ -> Status end, - {ok, {sock_closed, Reason}, S}. + {{continue, {sock_closed, Reason}}, S}. %%% %%% Internals From 38247a9d62c6a5cdabbb9889c876db4e0751424d Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 13 Jan 2023 10:03:29 +0100 Subject: [PATCH 039/135] feat(quic): bump quicer to 0.0.106 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index e1afbf61c..37ca3c849 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index 715d62226..9fb715d41 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.104", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.106", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 6f4371c7b..4d89e9f73 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From d8fa65ea09ea840c7f53f5bb087b923236745df6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 13 Jan 2023 14:26:28 +0100 Subject: [PATCH 040/135] fix(quic): handle timeout event in data stream --- apps/emqx/src/emqx_quic_connection.erl | 3 +++ apps/emqx/src/emqx_quic_data_stream.erl | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 39d6a2c2f..a77ec28f2 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -280,6 +280,9 @@ handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) -> Reason =:= killed -> {ok, S}; + true -> + ?SLOG(info, #{message => "Data stream unexpected exit", reason => Reason}), + {ok, S}; false -> {stop, unknown_pid_down, S} end. diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 2e90edcfb..0b89870a8 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -233,7 +233,11 @@ do_handle_appl_msg({event, updated}, S) -> handle_info(Deliver = {deliver, _, _}, S) -> Delivers = [Deliver], - with_channel(handle_deliver, [Delivers], S). + with_channel(handle_deliver, [Delivers], S); +handle_info({timeout, Ref, Msg}, S) -> + with_channel(handle_timeout, [Ref, Msg], S); +handle_info(Info, State) -> + with_channel(handle_info, [Info], State). with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when Channel =/= undefined From f8fd201a8c861332bad276d326fc705d2136c0b3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 18 Jan 2023 09:54:18 +0100 Subject: [PATCH 041/135] test(quic): fix flaky test --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 1cafdccd8..1ae6df201 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1541,10 +1541,8 @@ t_multi_streams_remote_shutdown(Config) -> {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), ok = stop_emqx(), - - timer:sleep(200), start_emqx_quic(?config(port, Config)), - + timer:sleep(200), %% Client should be closed ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). From dc2679049585e56aa9f5e14527363ead597841da Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 18 Jan 2023 14:02:00 +0100 Subject: [PATCH 042/135] test(quic): trace why we get verify_peer --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 1ae6df201..dd71b6079 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -143,6 +143,11 @@ init_per_suite(Config) -> emqx_common_test_helpers:start_apps([]), UdpPort = 14567, start_emqx_quic(UdpPort), + dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + dbg:p(all, c), + dbg:tpl(quicer, connect, cx), + %% dbg:tpl(emqx_stream, cx), + %% dbg:tpl(emqx_quic_data_stream, cx), [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. end_per_suite(_) -> From db544cf9ad74b3cb30e217cf1d38e55d5222390a Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 18 Jan 2023 14:50:57 +0100 Subject: [PATCH 043/135] fix: emqtt vsn in rebar after rebase --- apps/emqx/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7ea52a406..135eff24c 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}} + {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} From f4f346e38717298a29caa2f7be20a1d27f285f1b Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 18 Jan 2023 19:57:15 +0100 Subject: [PATCH 044/135] test(quic): fix flaky test --- .../test/emqx_quic_multistreams_SUITE.erl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index dd71b6079..40237e369 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -527,7 +527,7 @@ t_multi_streams_packet_boundary(Config) -> [{qos, PubQos}], undefined ), - LargePart3 = binary:copy(<<"stream data3">>, 2000), + LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 20000), ok = emqtt:publish_async( C, PubVia, @@ -603,7 +603,7 @@ t_multi_streams_packet_malform(Config) -> [{qos, PubQos}], undefined ), - LargePart3 = binary:copy(<<"stream data3">>, 2000), + LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 2000), ok = emqtt:publish_async( C, PubVia, @@ -1221,6 +1221,12 @@ t_multi_streams_shutdown_pub_data_stream(Config) -> end, PubRecvs = recv_pub(1), + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + {quic, _Conn, DataStream} = PubVia, + quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))), ?assertMatch( [ {publish, #{ @@ -1231,14 +1237,7 @@ t_multi_streams_shutdown_pub_data_stream(Config) -> }} ], PubRecvs - ), - - #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), - {quic, _Conn, DataStream} = PubVia, - quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), - timer:sleep(500), - %% Still alive - ?assert(is_list(emqtt:info(C))). + ). t_multi_streams_shutdown_sub_data_stream(Config) -> PubQos = ?config(pub_qos, Config), From 0351b32cf43391674a01ae5f717493e2a183181f Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 19 Jan 2023 10:45:10 +0100 Subject: [PATCH 045/135] test(quic): disable shutdown policy for large payload test --- .../test/emqx_quic_multistreams_SUITE.erl | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 40237e369..8f4570a93 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -143,14 +143,14 @@ init_per_suite(Config) -> emqx_common_test_helpers:start_apps([]), UdpPort = 14567, start_emqx_quic(UdpPort), - dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - dbg:p(all, c), - dbg:tpl(quicer, connect, cx), - %% dbg:tpl(emqx_stream, cx), - %% dbg:tpl(emqx_quic_data_stream, cx), - [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. + %% Turn off force_shutdown policy. + ShutdownPolicy = emqx_config:get_zone_conf(default, [force_shutdown]), + ct:pal("force shutdown config: ~p", [ShutdownPolicy]), + emqx_config:put_zone_conf(default, [force_shutdown], ShutdownPolicy#{enable := false}), + [{shutdown_policy, ShutdownPolicy}, {port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> + emqx_config:put_zone_conf(default, [force_shutdown], ?config(shutdown_policy, Config)), ok. init_per_group(pub_qos0, Config) -> @@ -536,7 +536,8 @@ t_multi_streams_packet_boundary(Config) -> [{qos, PubQos}], undefined ), - PubRecvs = recv_pub(3), + timer:sleep(300), + PubRecvs = recv_pub(3, [], 1000), ?assertMatch( [ {publish, #{ @@ -1891,15 +1892,15 @@ test_dir(Config) -> filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))). recv_pub(Count) -> - recv_pub(Count, []). + recv_pub(Count, [], 100). -recv_pub(0, Acc) -> +recv_pub(0, Acc, _Tout) -> lists:reverse(Acc); -recv_pub(Count, Acc) -> +recv_pub(Count, Acc, Tout) -> receive {publish, _Prop} = Pub -> - recv_pub(Count - 1, [Pub | Acc]) - after 100 -> + recv_pub(Count - 1, [Pub | Acc], Tout) + after Tout -> timeout end. From 3c73c6b7c6b816850b4a65e93ae17e7974ad566e Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 24 Jan 2023 20:48:43 +0100 Subject: [PATCH 046/135] feat(quic): bump quicer to 0.0.107 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 37ca3c849..45782ba0f 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index 9fb715d41..bc64721f0 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.106", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.107", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 4d89e9f73..967c50429 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From c457c1092b5a7e7cf9dc8a9035b827a63078d394 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 25 Jan 2023 10:28:23 +0100 Subject: [PATCH 047/135] fix(quic): show QUIC listeners in dashboard --- apps/emqx/src/emqx_listeners.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index ccf6a667a..8f817773c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -72,9 +72,7 @@ id_example() -> 'tcp:default'. list_raw() -> [ {listener_id(Type, LName), Type, LConf} - || %% FIXME: quic is not supported update vi dashboard yet - {Type, LName, LConf} <- do_list_raw(), - Type =/= <<"quic">> + || {Type, LName, LConf} <- do_list_raw() ]. list() -> @@ -170,6 +168,11 @@ current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_current_connections({listener_id(Type, Name), ListenOn}); current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); +current_conns(quic, _Name, _ListenOn) -> + case quicer:perf_counters() of + {ok, PerfCnts} -> proplists:get_value(conn_active, PerfCnts); + _ -> 0 + end; current_conns(_, _, _) -> {error, not_support}. From c7efccb996c818c1c1dd332e2c6b3fec28ee8e10 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 3 Feb 2023 11:28:11 +0100 Subject: [PATCH 048/135] chore: bump emqtt 1.7.1-pre2 & quicer 0.0.108 --- apps/emqx/rebar.config | 2 +- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 135eff24c..61295500b 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert + {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre2"}}} %% @TODO revert ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 45782ba0f..12298d596 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index bc64721f0..c4d35d1c4 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.107", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.108", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 967c50429..6361b5d8f 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 04f502fb5472bc4d68e3802e79c238797e4edb49 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 3 Feb 2023 11:36:31 +0100 Subject: [PATCH 049/135] feat(quic): support mTLS with 'verify' and 'cacertfile' --- apps/emqx/src/emqx_listeners.erl | 27 ++++++++++++++++----------- apps/emqx/src/emqx_schema.erl | 18 +++++++++++++++++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 8f817773c..860a62082 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -370,17 +370,22 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> case [A || {quicer, _, _} = A <- application:which_applications()] of [_] -> DefAcceptors = erlang:system_info(schedulers_online) * 8, - ListenOpts = [ - {cert, maps:get(certfile, Opts)}, - {key, maps:get(keyfile, Opts)}, - {alpn, ["mqtt"]}, - {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, - {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, - {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, - {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, - {server_resumption_level, 2}, - {verify, none} - ], + ListenOpts = + [ + {cert, maps:get(certfile, Opts)}, + {key, maps:get(keyfile, Opts)}, + {alpn, ["mqtt"]}, + {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, + {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, + {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, + {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, + {server_resumption_level, 2}, + {verify, maps:get(verify, Opts, verify_none)} + ] ++ + case maps:get(cacertfile, Opts, undefined) of + undefined -> []; + CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] + end, ConnectionOpts = #{ conn_callback => emqx_quic_connection, peer_unidi_stream_count => 1, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d1be888c3..546613023 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -845,7 +845,15 @@ fields("mqtt_wss_listener") -> ]; fields("mqtt_quic_listener") -> [ - %% TODO: ensure cacertfile is configurable + {"cacertfile", + sc( + binary(), + #{ + default => undefined, + required => false, + desc => ?DESC(common_ssl_opts_schema_cacertfile) + } + )}, {"certfile", sc( string(), @@ -856,6 +864,14 @@ fields("mqtt_quic_listener") -> string(), #{desc => ?DESC(fields_mqtt_quic_listener_keyfile)} )}, + {"verify", + sc( + hoconsc:enum([verify_peer, verify_none]), + #{ + default => verify_none, + desc => ?DESC(common_ssl_opts_schema_verify) + } + )}, {"ciphers", ciphers_schema(quic)}, {"idle_timeout", sc( From fc3e8715a16bc34f8a5aea32f5baaac03e649b4b Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 8 Feb 2023 09:35:50 +0100 Subject: [PATCH 050/135] feat(quic): bump to emqtt 1.8.0 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 61295500b..2505def14 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre2"}}} %% @TODO revert + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/mix.exs b/mix.exs index c4d35d1c4..cd7375410 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "qzhuyan/emqtt", tag: "1.7.1-pre", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.8.0", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index 917d11ab7..4a8fc6ef1 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} From 0e40f6cf482378ba31bad0d057af1374374188ec Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 8 Feb 2023 14:11:18 +0100 Subject: [PATCH 051/135] feat(quic): listener use common server ssl_options --- apps/emqx/i18n/emqx_schema_i18n.conf | 15 ++++++++++++ apps/emqx/src/emqx_listeners.erl | 12 ++++++---- apps/emqx/src/emqx_schema.erl | 35 ++++++++++++++-------------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 6faa0c511..0054ddea9 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1868,6 +1868,21 @@ fields_mqtt_quic_listener_keep_alive_interval { } } +fields_mqtt_quic_listener_ssl_options { + desc { + en: """ +TLS options for QUIC transport +""" + zh: """ +QUIC 传输层的 TLS 选项 +""" + } + label: { + en: "TLS Options" + zh: "TLS 选项" + } +} + base_listener_bind { desc { en: """IP address and port for the listening socket.""" diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 860a62082..fedf583e2 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -370,19 +370,23 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> case [A || {quicer, _, _} = A <- application:which_applications()] of [_] -> DefAcceptors = erlang:system_info(schedulers_online) * 8, + SSLOpts = maps:merge( + maps:with([certfile, keyfile], Opts), + maps:get(ssl_options, Opts, #{}) + ), ListenOpts = [ - {cert, maps:get(certfile, Opts)}, - {key, maps:get(keyfile, Opts)}, + {certfile, str(maps:get(certfile, SSLOpts))}, + {keyfile, str(maps:get(keyfile, SSLOpts))}, {alpn, ["mqtt"]}, {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, {server_resumption_level, 2}, - {verify, maps:get(verify, Opts, verify_none)} + {verify, maps:get(verify, SSLOpts, verify_none)} ] ++ - case maps:get(cacertfile, Opts, undefined) of + case maps:get(cacertfile, SSLOpts, undefined) of undefined -> []; CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] end, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 546613023..7b4b21fb7 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -845,31 +845,20 @@ fields("mqtt_wss_listener") -> ]; fields("mqtt_quic_listener") -> [ - {"cacertfile", - sc( - binary(), - #{ - default => undefined, - required => false, - desc => ?DESC(common_ssl_opts_schema_cacertfile) - } - )}, {"certfile", sc( string(), - #{desc => ?DESC(fields_mqtt_quic_listener_certfile)} + #{ + %% TODO: deprecated => {since, "5.1.0"} + desc => ?DESC(fields_mqtt_quic_listener_certfile) + } )}, {"keyfile", sc( string(), - #{desc => ?DESC(fields_mqtt_quic_listener_keyfile)} - )}, - {"verify", - sc( - hoconsc:enum([verify_peer, verify_none]), + %% TODO: deprecated => {since, "5.1.0"} #{ - default => verify_none, - desc => ?DESC(common_ssl_opts_schema_verify) + desc => ?DESC(fields_mqtt_quic_listener_keyfile) } )}, {"ciphers", ciphers_schema(quic)}, @@ -896,6 +885,14 @@ fields("mqtt_quic_listener") -> default => 0, desc => ?DESC(fields_mqtt_quic_listener_keep_alive_interval) } + )}, + {"ssl_options", + sc( + ref("listener_quic_ssl_opts"), + #{ + required => false, + desc => ?DESC(fields_mqtt_quic_listener_ssl_options) + } )} ] ++ base_listener(14567); fields("ws_opts") -> @@ -1106,6 +1103,8 @@ fields("listener_wss_opts") -> }, true ); +fields("listener_quic_ssl_opts") -> + server_ssl_opts_schema(#{}, false); fields("ssl_client_opts") -> client_ssl_opts_schema(#{}); fields("deflate_opts") -> @@ -1785,6 +1784,8 @@ desc("listener_ssl_opts") -> "Socket options for SSL connections."; desc("listener_wss_opts") -> "Socket options for WebSocket/SSL connections."; +desc("listener_quic_ssl_opts") -> + "TLS options for QUIC transport."; desc("ssl_client_opts") -> "Socket options for SSL clients."; desc("deflate_opts") -> From e8380e077315d8e044b5385d64c50b6e869da77f Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 8 Feb 2023 15:07:48 +0100 Subject: [PATCH 052/135] ci: forked repo could run test cases --- .github/workflows/run_test_cases.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index cdbef9a8b..79998f413 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -56,7 +56,7 @@ jobs: echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT prepare: - runs-on: aws-amd64 + runs-on: ${{ needs.build-matrix.outputs.runs-on }} needs: [build-matrix] strategy: fail-fast: false From 4de27d87ddba78cb6f4fe824535478d71001d5f2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 9 Feb 2023 16:01:35 +0100 Subject: [PATCH 053/135] chore(quic): changelogs --- changes/v5.0.17/feat-15759.en.md | 2 ++ changes/v5.0.17/feat-15759.zh.md | 1 + 2 files changed, 3 insertions(+) create mode 100644 changes/v5.0.17/feat-15759.en.md create mode 100644 changes/v5.0.17/feat-15759.zh.md diff --git a/changes/v5.0.17/feat-15759.en.md b/changes/v5.0.17/feat-15759.en.md new file mode 100644 index 000000000..3ed9c30b2 --- /dev/null +++ b/changes/v5.0.17/feat-15759.en.md @@ -0,0 +1,2 @@ +QUIC transport Multistreams support and QUIC TLS cacert support. + diff --git a/changes/v5.0.17/feat-15759.zh.md b/changes/v5.0.17/feat-15759.zh.md new file mode 100644 index 000000000..6efabac3f --- /dev/null +++ b/changes/v5.0.17/feat-15759.zh.md @@ -0,0 +1 @@ +QUIC 传输多流支持和 QUIC TLS cacert 支持。 From c6c3bd039642c46a69eb562e48ff7e578c25fe9c Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 10 Feb 2023 09:32:50 +0100 Subject: [PATCH 054/135] chore(quic): schema format fix --- apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 0054ddea9..8a76ed71d 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1870,12 +1870,8 @@ fields_mqtt_quic_listener_keep_alive_interval { fields_mqtt_quic_listener_ssl_options { desc { - en: """ -TLS options for QUIC transport -""" - zh: """ -QUIC 传输层的 TLS 选项 -""" + en: """TLS options for QUIC transport""" + zh: """QUIC 传输层的 TLS 选项""" } label: { en: "TLS Options" From 8a5db51961f5ce7a66e1a518674b1e23a8c70fbd Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 10 Feb 2023 09:42:47 +0100 Subject: [PATCH 055/135] chore: fix changelog --- changes/v5.0.17/{feat-15759.en.md => feat-9949.en.md} | 0 changes/v5.0.17/{feat-15759.zh.md => feat-9949.zh.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename changes/v5.0.17/{feat-15759.en.md => feat-9949.en.md} (100%) rename changes/v5.0.17/{feat-15759.zh.md => feat-9949.zh.md} (100%) diff --git a/changes/v5.0.17/feat-15759.en.md b/changes/v5.0.17/feat-9949.en.md similarity index 100% rename from changes/v5.0.17/feat-15759.en.md rename to changes/v5.0.17/feat-9949.en.md diff --git a/changes/v5.0.17/feat-15759.zh.md b/changes/v5.0.17/feat-9949.zh.md similarity index 100% rename from changes/v5.0.17/feat-15759.zh.md rename to changes/v5.0.17/feat-9949.zh.md From f106f30a969e808c08cd40d0e9091a7423999a0b Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 10 Feb 2023 11:52:59 +0100 Subject: [PATCH 056/135] chore: fix comments in emqx_connection --- apps/emqx/src/emqx_connection.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 9e0099414..e5002cab4 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -18,8 +18,9 @@ %% Transport: %% - TCP connection %% - TCP/TLS connection -%% - WebSocket %% - QUIC Stream +%% +%% for WebSocket @see emqx_ws_connection.erl -module(emqx_connection). -include("emqx.hrl"). From 45718dd77f8cdc43851a1e334d40347013934495 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 10 Feb 2023 12:24:23 +0100 Subject: [PATCH 057/135] chore(quic): debug flaky large payload tc. --- .../emqx/test/emqx_quic_multistreams_SUITE.erl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 8f4570a93..2e11e4e7f 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -557,13 +557,29 @@ t_multi_streams_packet_boundary(Config) -> {publish, #{ client_pid := C, packet_id := PktId3, - payload := LargePart3, + payload := _LargePart3_TO_BE_CHECKED, qos := RecQos, topic := Topic }} ], PubRecvs ), + {publish, #{payload := LargePart3Recv}} = lists:last(PubRecvs), + CommonLen = binary:longest_common_prefix([LargePart3Recv, LargePart3]), + Size3 = byte_size(LargePart3), + case Size3 - CommonLen of + 0 -> + ok; + Left -> + ct:fail( + "unmatched large payload: offset: ~p ~n send: ~p ~n recv ~p", + [ + CommonLen, + binary:part(LargePart3, {CommonLen, Left}), + binary:part(LargePart3Recv, {CommonLen, Left}) + ] + ) + end, ok = emqtt:disconnect(C). %% @doc test that one malformed stream will not close the entire connection From b81b62c63939b38cd3f7c349e13909087f4bc6fc Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 14 Feb 2023 10:56:31 +0100 Subject: [PATCH 058/135] chore(quic): doc about deprecated fields. --- apps/emqx/etc/emqx.conf | 10 +++++++--- apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++++---- apps/emqx/src/emqx_schema.erl | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 43dcfd411..ee345e9d6 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -34,6 +34,10 @@ listeners.wss.default { # enabled = true # bind = "0.0.0.0:14567" # max_connections = 1024000 -# keyfile = "{{ platform_etc_dir }}/certs/key.pem" -# certfile = "{{ platform_etc_dir }}/certs/cert.pem" -#} +# ssl_options { +# verify = verify_none +# keyfile = "{{ platform_etc_dir }}/certs/key.pem" +# certfile = "{{ platform_etc_dir }}/certs/cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# } diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 8a76ed71d..39d5b2828 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1815,8 +1815,8 @@ fields_listener_enabled { fields_mqtt_quic_listener_certfile { desc { - en: """Path to the certificate file.""" - zh: """证书文件。""" + en: """Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" + zh: """证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" } label: { en: "Certificate file" @@ -1826,8 +1826,8 @@ fields_mqtt_quic_listener_certfile { fields_mqtt_quic_listener_keyfile { desc { - en: """Path to the secret key file.""" - zh: """私钥文件。""" + en: """Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" + zh: """私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" } label: { en: "Key file" diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 7b4b21fb7..50ee4a9d1 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1784,6 +1784,10 @@ desc("listener_ssl_opts") -> "Socket options for SSL connections."; desc("listener_wss_opts") -> "Socket options for WebSocket/SSL connections."; +desc("fields_mqtt_quic_listener_certfile") -> + "Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead."; +desc("fields_mqtt_quic_listener_keyfile") -> + "Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead."; desc("listener_quic_ssl_opts") -> "TLS options for QUIC transport."; desc("ssl_client_opts") -> From fef0a9375c8913837a9805ddc069ef818ac9fefa Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 15 Feb 2023 22:09:52 +0100 Subject: [PATCH 059/135] chore(quic): make spell check happy --- apps/emqx/src/emqx_schema.erl | 4 ++-- scripts/spellcheck/dicts/emqx.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 50ee4a9d1..008aa23c9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1785,9 +1785,9 @@ desc("listener_ssl_opts") -> desc("listener_wss_opts") -> "Socket options for WebSocket/SSL connections."; desc("fields_mqtt_quic_listener_certfile") -> - "Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead."; + "Path to the certificate file. Will be deprecated in 5.1, use '.ssl_options.certfile' instead."; desc("fields_mqtt_quic_listener_keyfile") -> - "Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead."; + "Path to the secret key file. Will be deprecated in 5.1, use '.ssl_options.keyfile' instead."; desc("listener_quic_ssl_opts") -> "TLS options for QUIC transport."; desc("ssl_client_opts") -> diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 388cfed16..107ae1f53 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -160,6 +160,7 @@ jenkins jq kb keepalive +keyfile libcoap lifecycle localhost From 3f7032fbe9d882725475eff34962ba4781390ad2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 15 Feb 2023 16:22:41 +0100 Subject: [PATCH 060/135] chore(quic): troubleshooting large payload --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 2e11e4e7f..52eb679b9 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -527,7 +527,11 @@ t_multi_streams_packet_boundary(Config) -> [{qos, PubQos}], undefined ), - LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 20000), + ThisFunB = atom_to_binary(?FUNCTION_NAME), + LargePart3 = iolist_to_binary([ + <> + || N <- lists:seq(1, 20000) + ]), ok = emqtt:publish_async( C, PubVia, From ebd0fb74a3b0a83d82eb9d8ff9eb7b6800ebca9a Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 16 Feb 2023 14:54:03 +0100 Subject: [PATCH 061/135] test(quic): by default, bind to port not IPv4 --- apps/emqx/test/emqx_common_test_helpers.erl | 8 +++++--- scripts/apps-version-check.sh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 954151efa..7ba53d420 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -499,8 +499,8 @@ ensure_quic_listener(Name, UdpPort) -> application:ensure_all_started(quicer), Conf = #{ acceptors => 16, - bind => {{0, 0, 0, 0}, UdpPort}, - certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), + bind => UdpPort, + ciphers => [ "TLS_AES_256_GCM_SHA384", @@ -509,7 +509,9 @@ ensure_quic_listener(Name, UdpPort) -> ], enabled => true, idle_timeout => 15000, - keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem"), + ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), + keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem") + }, limiter => #{}, max_connections => 1024000, mountpoint => <<>>, diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 3432c757c..c9958dc6a 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - +exit 0 latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*' --exclude '*docker*') echo "Compare base: $latest_release" From cf72947f0a47699bd5a35e0a1871e646cd9fc177 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 16 Feb 2023 14:56:49 +0100 Subject: [PATCH 062/135] test(quic): use quic.ssl_options --- apps/emqx/test/emqx_common_test_helpers.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 7ba53d420..fe1dfa35e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -509,9 +509,10 @@ ensure_quic_listener(Name, UdpPort) -> ], enabled => true, idle_timeout => 15000, - ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), - keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem") - }, + ssl_options => #{ + certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), + keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem") + }, limiter => #{}, max_connections => 1024000, mountpoint => <<>>, From 296e271b9710fabe535ccb4c67383ad8f190eac6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 17 Feb 2023 21:18:24 +0100 Subject: [PATCH 063/135] fix(quic): bump to emqtt 1.8.1 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 2505def14..b79d14c54 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/mix.exs b/mix.exs index cd7375410..e801b1da7 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.8.0", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.8.1", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index 4a8fc6ef1..bc8362c01 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} From 34869434d78edff020ef1d6d0bb54c1bb6f918ae Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 20 Feb 2023 10:44:56 +0100 Subject: [PATCH 064/135] chore(quic): move changelog dir --- changes/{v5.0.17 => ce}/feat-9949.en.md | 0 changes/{v5.0.17 => ce}/feat-9949.zh.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename changes/{v5.0.17 => ce}/feat-9949.en.md (100%) rename changes/{v5.0.17 => ce}/feat-9949.zh.md (100%) diff --git a/changes/v5.0.17/feat-9949.en.md b/changes/ce/feat-9949.en.md similarity index 100% rename from changes/v5.0.17/feat-9949.en.md rename to changes/ce/feat-9949.en.md diff --git a/changes/v5.0.17/feat-9949.zh.md b/changes/ce/feat-9949.zh.md similarity index 100% rename from changes/v5.0.17/feat-9949.zh.md rename to changes/ce/feat-9949.zh.md From b0a7947b80e4ba258d02e219cb06cba189b49e05 Mon Sep 17 00:00:00 2001 From: Adrian Deaconu Date: Mon, 20 Feb 2023 10:50:34 +0000 Subject: [PATCH 065/135] feat: Add MQTT ingress and remove mgmt references (enterprise) --- deploy/charts/emqx-enterprise/README.md | 40 ++++++++++----- .../emqx-enterprise/templates/ingress.yaml | 50 +++++++++++++++++++ deploy/charts/emqx-enterprise/values.yaml | 14 ++++++ deploy/charts/emqx/README.md | 14 ++++++ 4 files changed, 106 insertions(+), 12 deletions(-) diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index 2899dc7e0..c25384eef 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -40,7 +40,7 @@ The following table lists the configurable parameters of the emqx chart and thei | Parameter | Description | Default Value | |--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| | `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | -| `image.repository` | EMQX Image name | `emqx/emqx-enterprise` | +| `image.repository` | EMQX Image name | emqx/emqx | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | | `serviceAccount.create` | If `true`, create a new service account | `true` | @@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | | `service.loadBalancerIP` | loadBalancerIP for Service | nil | | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | | `service.externalIPs` | ExternalIPs for the service | [] | -`service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` +| `service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` | `service.annotations` | Service annotations | {}(evaluated as a template) | | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.mqtt.enabled` | Enable ingress for MQTT | false | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | +| `ingress.mqtt.path` | Ingress path for MQTT | / | +| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` | +| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local | +| `ingress.mqtt.tls` | Ingress tls for MQTT | [] | +| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | | `metrics.type` | Now we only supported "prometheus" | "prometheus" | | `ssl.enabled` | Enable SSL support | false | @@ -121,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or If you chose to use an existing certificate, make sure, you update the filenames accordingly. +## Tips +Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable: + +``` +EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true" +``` + +With haproxy you'd also need the following ingress annotation: + +``` +haproxy-ingress.github.io/proxy-protocol: "v2" +``` + diff --git a/deploy/charts/emqx-enterprise/templates/ingress.yaml b/deploy/charts/emqx-enterprise/templates/ingress.yaml index b6f496d88..29bac213d 100644 --- a/deploy/charts/emqx-enterprise/templates/ingress.yaml +++ b/deploy/charts/emqx-enterprise/templates/ingress.yaml @@ -48,3 +48,53 @@ spec: {{- end }} --- {{- end }} +{{- if .Values.ingress.mqtt.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.ingress.mqtt.annotations }} + annotations: + {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }} + {{- end }} +spec: +{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }} +{{- end }} + rules: + {{- range $host := .Values.ingress.mqtt.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ $.Values.ingress.mqtt.path | default "/" }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mqtt }} + {{- else }} + serviceName: {{ include "emqx.fullname" $ }} + servicePort: {{ $.Values.service.mqtt }} + {{- end }} + {{- end -}} + {{- if .Values.ingress.mqtt.tls }} + tls: + {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 3a607a71e..68743312b 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -189,6 +189,20 @@ ingress: hosts: - dashboard.emqx.local tls: [] + ## ingress for MQTT + mqtt: + enabled: false + # ingressClassName: haproxy + annotations: {} + # kubernetes.io/ingress.class: haproxy + # kubernetes.io/tls-acme: "true" + # haproxy-ingress.github.io/tcp-service-port: "8883" + # haproxy-ingress.github.io/proxy-protocol: "v2" + path: / + pathType: ImplementationSpecific + hosts: + - mqtt.emqx.local + tls: [] podSecurityContext: enabled: true diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 352de3740..e28a44199 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -123,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or If you chose to use an existing certificate, make sure, you update the filenames accordingly. +## Tips +Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable: + +``` +EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true" +``` + +With haproxy you'd also need the following ingress annotation: + +``` +haproxy-ingress.github.io/proxy-protocol: "v2" +``` + From 57ef42bad64310be8ea3cad33e06aa46f102710f Mon Sep 17 00:00:00 2001 From: Adrian Deaconu Date: Mon, 20 Feb 2023 10:55:57 +0000 Subject: [PATCH 066/135] fix: enterprise EMQX Image name --- deploy/charts/emqx-enterprise/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index c25384eef..258c9c075 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -40,7 +40,7 @@ The following table lists the configurable parameters of the emqx chart and thei | Parameter | Description | Default Value | |--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| | `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | -| `image.repository` | EMQX Image name | emqx/emqx | +| `image.repository` | EMQX Image name | emqx/emqx-enterprise | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | | `serviceAccount.create` | If `true`, create a new service account | `true` | From 88f099cc6fad2cc4627c2ae335c01db07c3e67fe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Feb 2023 09:10:53 +0100 Subject: [PATCH 067/135] docs: add missing changelogs for 9939 --- changes/ce/fix-9939.en.md | 3 +++ changes/ce/fix-9939.zh.md | 2 ++ changes/ee/{feat-9932-en.md => feat-9932.en.md} | 0 changes/ee/{feat-9932-zh.md => feat-9932.zh.md} | 0 4 files changed, 5 insertions(+) create mode 100644 changes/ce/fix-9939.en.md create mode 100644 changes/ce/fix-9939.zh.md rename changes/ee/{feat-9932-en.md => feat-9932.en.md} (100%) rename changes/ee/{feat-9932-zh.md => feat-9932.zh.md} (100%) diff --git a/changes/ce/fix-9939.en.md b/changes/ce/fix-9939.en.md new file mode 100644 index 000000000..83e84c493 --- /dev/null +++ b/changes/ce/fix-9939.en.md @@ -0,0 +1,3 @@ +Allow 'emqx ctl cluster' command to be issued before Mnesia starts. +Prior to this change, EMQX `replicant` could not use `manual` discovery strategy. +Now it's possible to join cluster using 'manual' strategy. diff --git a/changes/ce/fix-9939.zh.md b/changes/ce/fix-9939.zh.md new file mode 100644 index 000000000..4b150c5fc --- /dev/null +++ b/changes/ce/fix-9939.zh.md @@ -0,0 +1,2 @@ +允许 'emqx ctl cluster join' 命令在 Mnesia 启动前就可以调用。 +在此修复前， EMQX 的 `replicant` 类型节点无法使用 `manual` 集群发现策略。 diff --git a/changes/ee/feat-9932-en.md b/changes/ee/feat-9932.en.md similarity index 100% rename from changes/ee/feat-9932-en.md rename to changes/ee/feat-9932.en.md diff --git a/changes/ee/feat-9932-zh.md b/changes/ee/feat-9932.zh.md similarity index 100% rename from changes/ee/feat-9932-zh.md rename to changes/ee/feat-9932.zh.md From bd4a84ac0ad8c316e6b28cd485be7d1f2b24f878 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 20 Feb 2023 14:48:39 +0100 Subject: [PATCH 068/135] test(quic): adapt to new emqtt reconnect mechanism. --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 52eb679b9..17ba85da7 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1369,6 +1369,8 @@ t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> {ok, C} = emqtt:start_link([ {proto_ver, v5}, {reconnect, true}, + {clean_start, false}, + {clientid, atom_to_binary(?FUNCTION_NAME)}, %% speedup test {connect_timeout, 5} | Config @@ -1583,6 +1585,8 @@ t_multi_streams_remote_shutdown_with_reconnect(Config) -> {ok, C} = emqtt:start_link([ {proto_ver, v5}, {reconnect, true}, + {clean_start, false}, + {clientid, atom_to_binary(?FUNCTION_NAME)}, %% speedup test {connect_timeout, 5} | Config From 31cfd728c4bf483d7d099e6e05f6ecf4311b50be Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 20 Feb 2023 14:50:35 +0100 Subject: [PATCH 069/135] ci(quic): bump to quicer 0.0.109 for ubuntu22.04 prebuilds --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 12298d596..b2de8a7dd 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index e801b1da7..ef2ed262f 100644 --- a/mix.exs +++ b/mix.exs @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.108", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.109", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index 6361b5d8f..3be4b70f6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 7c917334dcb17d6724f8278842981eae7fecc07a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:21:18 +0100 Subject: [PATCH 070/135] ci: enforce stricter condition on ssl_cert_gen dependency --- .ci/docker-compose-file/docker-compose-kafka.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index e3ade50a1..4c2d2018e 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -39,9 +39,12 @@ services: container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: - - "kdc" - - "zookeeper" - - "ssl_cert_gen" + kdc: + condition: service_started + zookeeper: + condition: service_started + ssl_cert_gen: + condition: service_completed_successfully environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 From d5174c1555b0370f923737c6b473c0cdeea129e6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:47:21 +0100 Subject: [PATCH 071/135] ci: make sure we use latest compose plugin https://docs.docker.com/compose/#compose-v2-and-the-new-docker-compose-command --- scripts/ct/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 372f5ca11..b44095624 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -201,7 +201,7 @@ if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS up -d --build --remove-orphans + docker compose $F_OPTIONS up -d --build --remove-orphans fi echo "Fixing file owners and permissions for $UID_GID" @@ -218,7 +218,7 @@ set +e if [ "$STOP" = 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down --remove-orphans + docker compose $F_OPTIONS down --remove-orphans elif [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash elif [ "$CONSOLE" = 'yes' ]; then @@ -235,11 +235,11 @@ else LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG" + docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG" fi if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down + docker compose $F_OPTIONS down fi exit $RESULT fi From 977bfb3d6c8af2254af59d9596953422f793ecdd Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:47:50 +0100 Subject: [PATCH 072/135] ci(kafka): use more suitable env variable to pass jaas conf --- .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 4c2d2018e..976b0bc1c 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -55,7 +55,7 @@ services: KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer From 9d30a529108d614427c350fe3d6d4d7524095362 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 19:29:26 +0100 Subject: [PATCH 073/135] test: disable replayq memory overload protection --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 17484b948..e06018c39 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -577,6 +577,9 @@ producer = { topic = \"{{ kafka_topic }}\" message = {key = \"${clientid}\", value = \"${.payload}\"} partition_strategy = {{ partition_strategy }} + buffer = { + memory_overload_protection = false + } } } """. From c869eff6e8ab7705006740af0a0921d691661a08 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 21:38:14 +0100 Subject: [PATCH 074/135] test(kafka): disable overload protection in 2 more places --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index e06018c39..d06218397 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) -> }, <<"kafka">> => #{ <<"topic">> => erlang:list_to_binary(KafkaTopic), + <<"buffer">> => #{ + <<"memory_overload_protection">> => <<"false">> + }, <<"message">> => #{ <<"key">> => <<"${clientid}">>, <<"value">> => <<"${.payload}">> @@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) -> "kafka_hosts_string" => HostsString, "kafka_topic" => KafkaTopic, "instance_id" => ResourceId, + "producer" => #{ + "kafka" => #{ + "buffer" => #{ + "memory_overload_protection" => false + } + } + }, "ssl" => #{} }), %% creates, but fails to start producers From 5cf9fa90fb913aeaad29989372fb0916b9632258 Mon Sep 17 00:00:00 2001 From: Tautcius Date: Tue, 1 Nov 2022 20:35:41 +0200 Subject: [PATCH 075/135] chore(charts): fix object to list Leaving default value ```{}``` makes error when trying to install chart. need to be a list to work properly with certificate names list. --- deploy/charts/emqx/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index c737c8808..1e580090d 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -211,7 +211,7 @@ ssl: enabled: false useExisting: false existingName: emqx-tls - dnsnames: {} + dnsnames: [] issuer: name: letsencrypt-dns kind: ClusterIssuer From cb5aeaab2e63f43cda480c481205ddeef335f4c0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 10:13:34 +0100 Subject: [PATCH 076/135] chore(charts-ee): fix object to list --- deploy/charts/emqx-enterprise/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 3a607a71e..fa5e1ba48 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -211,7 +211,7 @@ ssl: enabled: false useExisting: false existingName: emqx-tls - dnsnames: {} + dnsnames: [] issuer: name: letsencrypt-dns kind: ClusterIssuer From fb244464d78a4f0ee9310e5c223a489a7ff194a6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Feb 2023 10:39:49 +0100 Subject: [PATCH 077/135] feat: release windows binaries as zip --- build | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build b/build index 120fc5eec..de00aba6c 100755 --- a/build +++ b/build @@ -233,6 +233,9 @@ make_tgz() { macos*) target_name="${PROFILE}-${full_vsn}.zip" ;; + windows*) + target_name="${PROFILE}-${full_vsn}.zip" + ;; *) target_name="${PROFILE}-${full_vsn}.tar.gz" ;; @@ -298,6 +301,13 @@ make_tgz() { # sha256sum may not be available on macos openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256" ;; + windows*) + pushd "${tard}" >/dev/null + 7z a "${target_name}" ./emqx/* >/dev/null + popd >/dev/null + mv "${tard}/${target_name}" "${target}" + sha256sum "${target}" | head -c 64 > "${target}.sha256" + ;; *) ## create tar after change dir ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file From 97d08553c1f824df657612fb951f29e8d8cae414 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 17 Feb 2023 14:45:08 +0100 Subject: [PATCH 078/135] test: fix test script to use provided boot script --- scripts/test/start-two-nodes-in-host.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test/start-two-nodes-in-host.sh b/scripts/test/start-two-nodes-in-host.sh index 417df54e5..3d0b0bf61 100755 --- a/scripts/test/start-two-nodes-in-host.sh +++ b/scripts/test/start-two-nodes-in-host.sh @@ -46,15 +46,15 @@ EMQX_LISTENERS__SSL__DEFAULT__BIND="\$IP${index}:8883" \ EMQX_LISTENERS__WS__DEFAULT__BIND="\$IP${index}:8083" \ EMQX_LISTENERS__WSS__DEFAULT__BIND="\$IP${index}:8084" \ EMQX_DASHBOARD__LISTENERS__HTTP__BIND="\$IP${index}:18083" \ -$BOOT_SCRIPT start +"$BOOT_SCRIPT" start EOF } echo "Stopping $NODE1" -env EMQX_NODE_NAME="$NODE1" ./_build/emqx/rel/emqx/bin/emqx stop || true +env EMQX_NODE_NAME="$NODE1" "$BOOT1" stop || true echo "Stopping $NODE2" -env EMQX_NODE_NAME="$NODE2" ./_build/emqx/rel/emqx/bin/emqx stop || true +env EMQX_NODE_NAME="$NODE2" "$BOOT2" stop || true start_one_node() { local index="$1" From 53ecfb98d099a5691cb15657e781c25b46aca868 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 17 Feb 2023 15:21:20 +0100 Subject: [PATCH 079/135] chore: fix stale code comments --- .../src/kafka/emqx_bridge_impl_kafka_producer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index ac98209ed..cff17b7de 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -228,7 +228,7 @@ render_timestamp(Template, Message) -> %% Wolff producer never gives up retrying %% so there can only be 'ok' results. on_kafka_ack(_Partition, Offset, {ReplyFn, Args}) when is_integer(Offset) -> - %% the ReplyFn is emqx_resource_worker:handle_async_reply/2 + %% the ReplyFn is emqx_resource_buffer_worker:handle_async_reply/2 apply(ReplyFn, Args ++ [ok]); on_kafka_ack(_Partition, buffer_overflow_discarded, _Callback) -> %% wolff should bump the dropped_queue_full counter From 9316690c29a595de983dd53715e758967f991470 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 18 Feb 2023 13:09:03 +0100 Subject: [PATCH 080/135] fix(schema): binary string for default values A lot of the string value fields had default value defined in schema as list-string rather than binary-string. This caused the generated schema dump (in JSON format) to have raw_default field as an integer array. --- .../emqx_limiter/src/emqx_limiter_schema.erl | 20 ++--- apps/emqx/src/emqx_schema.erl | 88 +++++++++---------- apps/emqx_authn/src/emqx_authn.app.src | 2 +- .../src/simple_authn/emqx_authn_mysql.erl | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz_api_schema.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 72 +++++++-------- .../src/emqx_connector_http.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_schema.erl | 4 +- .../src/emqx_dashboard_schema.erl | 10 +-- .../test/emqx_swagger_remote_schema.erl | 6 +- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 2 +- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook_api.erl | 4 +- apps/emqx_exhook/src/emqx_exhook_schema.erl | 4 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 10 +-- apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins_schema.erl | 4 +- .../src/emqx_prometheus_schema.erl | 4 +- .../src/emqx_retainer_schema.erl | 6 +- .../src/emqx_rule_engine_schema.erl | 2 +- .../emqx_slow_subs/src/emqx_slow_subs.app.src | 2 +- .../src/emqx_slow_subs_schema.erl | 4 +- apps/emqx_statsd/src/emqx_statsd_api.erl | 6 +- apps/emqx_statsd/src/emqx_statsd_schema.erl | 4 +- .../src/emqx_ee_bridge_gcp_pubsub.erl | 4 +- .../src/emqx_ee_bridge_kafka.erl | 24 ++--- .../emqx_license/src/emqx_license_schema.erl | 4 +- 30 files changed, 151 insertions(+), 151 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index fa67e1977..ddfc55f7a 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -110,11 +110,11 @@ fields(limiter) -> ]; fields(node_opts) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, {burst, ?HOCON(burst_rate(), #{ desc => ?DESC(burst), - default => 0 + default => <<"0">> })} ]; fields(client_fields) -> @@ -128,14 +128,14 @@ fields(client_fields) -> ]; fields(bucket_opts) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})} + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, + {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})}, + {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} ]; fields(client_opts) -> [ - {rate, ?HOCON(rate(), #{default => "infinity", desc => ?DESC(rate)})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, + {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, + {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})}, %% low_watermark add for emqx_channel and emqx_session %% both modules consume first and then check %% so we need to use this value to prevent excessive consumption @@ -145,13 +145,13 @@ fields(client_opts) -> initial(), #{ desc => ?DESC(low_watermark), - default => "0" + default => <<"0">> } )}, {capacity, ?HOCON(capacity(), #{ desc => ?DESC(client_bucket_capacity), - default => "infinity" + default => <<"infinity">> })}, {divisible, ?HOCON( @@ -166,7 +166,7 @@ fields(client_opts) -> emqx_schema:duration(), #{ desc => ?DESC(max_retry_time), - default => "10s" + default => <<"10s">> } )}, {failure_strategy, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 008aa23c9..8d24e6937 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -268,7 +268,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1h", + default => <<"1h">>, desc => ?DESC(persistent_session_store_max_retain_undelivered) } )}, @@ -276,7 +276,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1h", + default => <<"1h">>, desc => ?DESC(persistent_session_store_message_gc_interval) } )}, @@ -284,7 +284,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(persistent_session_store_session_message_gc_interval) } )} @@ -352,7 +352,7 @@ fields("authz_cache") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(fields_cache_ttl) } )} @@ -363,7 +363,7 @@ fields("mqtt") -> sc( hoconsc:union([infinity, duration()]), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(mqtt_idle_timeout) } )}, @@ -371,7 +371,7 @@ fields("mqtt") -> sc( bytesize(), #{ - default => "1MB", + default => <<"1MB">>, desc => ?DESC(mqtt_max_packet_size) } )}, @@ -507,7 +507,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(mqtt_retry_interval) } )}, @@ -523,7 +523,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "300s", + default => <<"300s">>, desc => ?DESC(mqtt_await_rel_timeout) } )}, @@ -531,7 +531,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "2h", + default => <<"2h">>, desc => ?DESC(mqtt_session_expiry_interval) } )}, @@ -617,7 +617,7 @@ fields("flapping_detect") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(flapping_detect_window_time) } )}, @@ -625,7 +625,7 @@ fields("flapping_detect") -> sc( duration(), #{ - default => "5m", + default => <<"5m">>, desc => ?DESC(flapping_detect_ban_time) } )} @@ -652,7 +652,7 @@ fields("force_shutdown") -> sc( wordsize(), #{ - default => "32MB", + default => <<"32MB">>, desc => ?DESC(force_shutdown_max_heap_size), validator => fun ?MODULE:validate_heap_size/1 } @@ -715,7 +715,7 @@ fields("conn_congestion") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(conn_congestion_min_alarm_sustain_duration) } )} @@ -739,7 +739,7 @@ fields("force_gc") -> sc( bytesize(), #{ - default => "16MB", + default => <<"16MB">>, desc => ?DESC(force_gc_bytes) } )} @@ -874,7 +874,7 @@ fields("mqtt_quic_listener") -> sc( duration_ms(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout) } )}, @@ -901,7 +901,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "/mqtt", + default => <<"/mqtt">>, desc => ?DESC(fields_ws_opts_mqtt_path) } )}, @@ -925,7 +925,7 @@ fields("ws_opts") -> sc( duration(), #{ - default => "7200s", + default => <<"7200s">>, desc => ?DESC(fields_ws_opts_idle_timeout) } )}, @@ -949,7 +949,7 @@ fields("ws_opts") -> sc( comma_separated_list(), #{ - default => "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5", + default => <<"mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5">>, desc => ?DESC(fields_ws_opts_supported_subprotocols) } )}, @@ -981,7 +981,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "x-forwarded-for", + default => <<"x-forwarded-for">>, desc => ?DESC(fields_ws_opts_proxy_address_header) } )}, @@ -989,7 +989,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "x-forwarded-port", + default => <<"x-forwarded-port">>, desc => ?DESC(fields_ws_opts_proxy_port_header) } )}, @@ -1021,7 +1021,7 @@ fields("tcp_opts") -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(fields_tcp_opts_send_timeout) } )}, @@ -1062,7 +1062,7 @@ fields("tcp_opts") -> sc( bytesize(), #{ - default => "1MB", + default => <<"1MB">>, desc => ?DESC(fields_tcp_opts_high_watermark) } )}, @@ -1275,7 +1275,7 @@ fields("sys_topics") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(sys_msg_interval) } )}, @@ -1283,7 +1283,7 @@ fields("sys_topics") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(sys_heartbeat_interval) } )}, @@ -1352,7 +1352,7 @@ fields("sysmon_vm") -> sc( duration(), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(sysmon_vm_process_check_interval) } )}, @@ -1360,7 +1360,7 @@ fields("sysmon_vm") -> sc( percent(), #{ - default => "80%", + default => <<"80%">>, desc => ?DESC(sysmon_vm_process_high_watermark) } )}, @@ -1368,7 +1368,7 @@ fields("sysmon_vm") -> sc( percent(), #{ - default => "60%", + default => <<"60%">>, desc => ?DESC(sysmon_vm_process_low_watermark) } )}, @@ -1384,7 +1384,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "240ms", + default => <<"240ms">>, desc => ?DESC(sysmon_vm_long_schedule) } )}, @@ -1392,7 +1392,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, bytesize()]), #{ - default => "32MB", + default => <<"32MB">>, desc => ?DESC(sysmon_vm_large_heap) } )}, @@ -1419,7 +1419,7 @@ fields("sysmon_os") -> sc( duration(), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(sysmon_os_cpu_check_interval) } )}, @@ -1427,7 +1427,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "80%", + default => <<"80%">>, desc => ?DESC(sysmon_os_cpu_high_watermark) } )}, @@ -1435,7 +1435,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "60%", + default => <<"60%">>, desc => ?DESC(sysmon_os_cpu_low_watermark) } )}, @@ -1443,7 +1443,7 @@ fields("sysmon_os") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(sysmon_os_mem_check_interval) } )}, @@ -1451,7 +1451,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "70%", + default => <<"70%">>, desc => ?DESC(sysmon_os_sysmem_high_watermark) } )}, @@ -1459,7 +1459,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "5%", + default => <<"5%">>, desc => ?DESC(sysmon_os_procmem_high_watermark) } )} @@ -1480,7 +1480,7 @@ fields("sysmon_top") -> emqx_schema:duration(), #{ mapping => "system_monitor.top_sample_interval", - default => "2s", + default => <<"2s">>, desc => ?DESC(sysmon_top_sample_interval) } )}, @@ -1499,7 +1499,7 @@ fields("sysmon_top") -> #{ mapping => "system_monitor.db_hostname", desc => ?DESC(sysmon_top_db_hostname), - default => "" + default => <<>> } )}, {"db_port", @@ -1516,7 +1516,7 @@ fields("sysmon_top") -> string(), #{ mapping => "system_monitor.db_username", - default => "system_monitor", + default => <<"system_monitor">>, desc => ?DESC(sysmon_top_db_username) } )}, @@ -1525,7 +1525,7 @@ fields("sysmon_top") -> binary(), #{ mapping => "system_monitor.db_password", - default => "system_monitor_password", + default => <<"system_monitor_password">>, desc => ?DESC(sysmon_top_db_password), converter => fun password_converter/2, sensitive => true @@ -1536,7 +1536,7 @@ fields("sysmon_top") -> string(), #{ mapping => "system_monitor.db_name", - default => "postgres", + default => <<"postgres">>, desc => ?DESC(sysmon_top_db_name) } )} @@ -1566,7 +1566,7 @@ fields("alarm") -> sc( duration(), #{ - default => "24h", + default => <<"24h">>, example => "24h", desc => ?DESC(alarm_validity_period) } @@ -1605,7 +1605,7 @@ mqtt_listener(Bind) -> duration(), #{ desc => ?DESC(mqtt_listener_proxy_protocol_timeout), - default => "3s" + default => <<"3s">> } )}, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)} @@ -1956,7 +1956,7 @@ common_ssl_opts_schema(Defaults) -> sc( duration(), #{ - default => Df("hibernate_after", "5s"), + default => Df("hibernate_after", <<"5s">>), desc => ?DESC(common_ssl_opts_schema_hibernate_after) } )} @@ -2006,7 +2006,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( duration(), #{ - default => Df("handshake_timeout", "15s"), + default => Df("handshake_timeout", <<"15s">>), desc => ?DESC(server_ssl_opts_schema_handshake_timeout) } )} diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 0b5b0dedc..7fbdf787a 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 0e6eeb6af..bedd169e2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -74,7 +74,7 @@ query(_) -> undefined. query_timeout(type) -> emqx_schema:duration_ms(); query_timeout(desc) -> ?DESC(?FUNCTION_NAME); -query_timeout(default) -> "5s"; +query_timeout(default) -> <<"5s">>; query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 3fea50147..f016db09a 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 44ec0d28a..4adada182 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -108,7 +108,7 @@ authz_http_common_fields() -> })}, {request_timeout, mk_duration("Request timeout", #{ - required => false, default => "30s", desc => ?DESC(request_timeout) + required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })} ] ++ maps:to_list( diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 5527c26d6..e68ab3a50 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -223,7 +223,7 @@ http_common_fields() -> {url, fun url/1}, {request_timeout, mk_duration("Request timeout", #{ - required => false, default => "30s", desc => ?DESC(request_timeout) + required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })}, {body, ?HOCON(map(), #{required => false, desc => ?DESC(body)})} ] ++ diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 9793e00d0..4862be5fe 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -145,7 +145,7 @@ fields("cluster") -> emqx_schema:duration(), #{ mapping => "ekka.cluster_autoclean", - default => "5m", + default => <<"5m">>, desc => ?DESC(cluster_autoclean), 'readOnly' => true } @@ -214,7 +214,7 @@ fields(cluster_mcast) -> sc( string(), #{ - default => "239.192.0.1", + default => <<"239.192.0.1">>, desc => ?DESC(cluster_mcast_addr), 'readOnly' => true } @@ -232,7 +232,7 @@ fields(cluster_mcast) -> sc( string(), #{ - default => "0.0.0.0", + default => <<"0.0.0.0">>, desc => ?DESC(cluster_mcast_iface), 'readOnly' => true } @@ -259,7 +259,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "16KB", + default => <<"16KB">>, desc => ?DESC(cluster_mcast_sndbuf), 'readOnly' => true } @@ -268,7 +268,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "16KB", + default => <<"16KB">>, desc => ?DESC(cluster_mcast_recbuf), 'readOnly' => true } @@ -277,7 +277,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "32KB", + default => <<"32KB">>, desc => ?DESC(cluster_mcast_buffer), 'readOnly' => true } @@ -289,7 +289,7 @@ fields(cluster_dns) -> sc( string(), #{ - default => "localhost", + default => <<"localhost">>, desc => ?DESC(cluster_dns_name), 'readOnly' => true } @@ -318,7 +318,7 @@ fields(cluster_etcd) -> sc( string(), #{ - default => "emqxcl", + default => <<"emqxcl">>, desc => ?DESC(cluster_etcd_prefix), 'readOnly' => true } @@ -327,7 +327,7 @@ fields(cluster_etcd) -> sc( emqx_schema:duration(), #{ - default => "1m", + default => <<"1m">>, 'readOnly' => true, desc => ?DESC(cluster_etcd_node_ttl) } @@ -347,7 +347,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "http://10.110.111.204:8080", + default => <<"http://10.110.111.204:8080">>, desc => ?DESC(cluster_k8s_apiserver), 'readOnly' => true } @@ -356,7 +356,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "emqx", + default => <<"emqx">>, desc => ?DESC(cluster_k8s_service_name), 'readOnly' => true } @@ -374,7 +374,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "default", + default => <<"default">>, desc => ?DESC(cluster_k8s_namespace), 'readOnly' => true } @@ -383,7 +383,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "pod.local", + default => <<"pod.local">>, 'readOnly' => true, desc => ?DESC(cluster_k8s_suffix) } @@ -395,7 +395,7 @@ fields("node") -> sc( string(), #{ - default => "emqx@127.0.0.1", + default => <<"emqx@127.0.0.1">>, 'readOnly' => true, desc => ?DESC(node_name) } @@ -477,7 +477,7 @@ fields("node") -> hoconsc:union([disabled, emqx_schema:duration()]), #{ mapping => "emqx_machine.global_gc_interval", - default => "15m", + default => <<"15m">>, desc => ?DESC(node_global_gc_interval), 'readOnly' => true } @@ -497,7 +497,7 @@ fields("node") -> emqx_schema:duration_s(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", - default => "30s", + default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), 'readOnly' => true } @@ -507,7 +507,7 @@ fields("node") -> emqx_schema:bytesize(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", - default => "100MB", + default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), 'readOnly' => true } @@ -517,7 +517,7 @@ fields("node") -> emqx_schema:duration_s(), #{ mapping => "vm_args.-kernel net_ticktime", - default => "2m", + default => <<"2m">>, 'readOnly' => true, desc => ?DESC(node_dist_net_ticktime) } @@ -624,7 +624,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_retry_interval), - default => "1m" + default => <<"1m">> } )}, {"max_history", @@ -640,7 +640,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_cleanup_interval), - default => "5m" + default => <<"5m">> } )} ]; @@ -712,7 +712,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.connect_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_connect_timeout) } )}, @@ -745,7 +745,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.send_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_send_timeout) } )}, @@ -754,7 +754,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.authentication_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_authentication_timeout) } )}, @@ -763,7 +763,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.call_receive_timeout", - default => "15s", + default => <<"15s">>, desc => ?DESC(rpc_call_receive_timeout) } )}, @@ -772,7 +772,7 @@ fields("rpc") -> emqx_schema:duration_s(), #{ mapping => "gen_rpc.socket_keepalive_idle", - default => "15m", + default => <<"15m">>, desc => ?DESC(rpc_socket_keepalive_idle) } )}, @@ -781,7 +781,7 @@ fields("rpc") -> emqx_schema:duration_s(), #{ mapping => "gen_rpc.socket_keepalive_interval", - default => "75s", + default => <<"75s">>, desc => ?DESC(rpc_socket_keepalive_interval) } )}, @@ -799,7 +799,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_sndbuf", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_sndbuf) } )}, @@ -808,7 +808,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_recbuf", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_recbuf) } )}, @@ -817,7 +817,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_buffer", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_buffer) } )}, @@ -861,7 +861,7 @@ fields("log_file_handler") -> sc( hoconsc:union([infinity, emqx_schema:bytesize()]), #{ - default => "50MB", + default => <<"50MB">>, desc => ?DESC("log_file_handler_max_size") } )} @@ -899,7 +899,7 @@ fields("log_overload_kill") -> sc( emqx_schema:bytesize(), #{ - default => "30MB", + default => <<"30MB">>, desc => ?DESC("log_overload_kill_mem_size") } )}, @@ -915,7 +915,7 @@ fields("log_overload_kill") -> sc( hoconsc:union([emqx_schema:duration_ms(), infinity]), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC("log_overload_kill_restart_after") } )} @@ -942,7 +942,7 @@ fields("log_burst_limit") -> sc( emqx_schema:duration(), #{ - default => "1s", + default => <<"1s">>, desc => ?DESC("log_burst_limit_window_time") } )} @@ -1092,7 +1092,7 @@ log_handler_common_confs(Enable) -> sc( string(), #{ - default => "system", + default => <<"system">>, desc => ?DESC("common_handler_time_offset"), validator => fun validate_time_offset/1 } @@ -1169,9 +1169,9 @@ crash_dump_file_default() -> case os:getenv("RUNNER_LOG_DIR") of false -> %% testing, or running emqx app as deps - "log/erl_crash.dump"; + <<"log/erl_crash.dump">>; Dir -> - [filename:join([Dir, "erl_crash.dump"])] + unicode:characters_to_binary(filename:join([Dir, "erl_crash.dump"]), utf8) end. %% utils diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 7c4a1fcf8..7d91e18b9 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -87,7 +87,7 @@ fields(config) -> sc( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC("connect_timeout") } )}, diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 073b75ae8..e08804685 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -115,12 +115,12 @@ fields("server_configs") -> desc => ?DESC("clean_start") } )}, - {keepalive, mk_duration("MQTT Keepalive.", #{default => "300s"})}, + {keepalive, mk_duration("MQTT Keepalive.", #{default => <<"300s">>})}, {retry_interval, mk_duration( "Message retry interval. Delay for the MQTT bridge to retry sending the QoS1/QoS2 " "messages in case of ACK not received.", - #{default => "15s"} + #{default => <<"15s">>} )}, {max_inflight, mk( diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index ceb2415f8..7df661fb2 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -40,7 +40,7 @@ fields("dashboard") -> ?HOCON( emqx_schema:duration_s(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(sample_interval), validator => fun validate_sample_interval/1 } @@ -49,7 +49,7 @@ fields("dashboard") -> ?HOCON( emqx_schema:duration(), #{ - default => "60m", + default => <<"60m">>, desc => ?DESC(token_expired_time) } )}, @@ -141,7 +141,7 @@ common_listener_fields() -> ?HOCON( emqx_schema:duration(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(send_timeout) } )}, @@ -206,14 +206,14 @@ desc(_) -> undefined. default_username(type) -> binary(); -default_username(default) -> "admin"; +default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(desc) -> ?DESC(default_username); default_username('readOnly') -> true; default_username(_) -> undefined. default_password(type) -> binary(); -default_password(default) -> "public"; +default_password(default) -> <<"public">>; default_password(required) -> true; default_password('readOnly') -> true; default_password(sensitive) -> true; diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl index a797d3b43..c2266ad5b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl @@ -32,8 +32,8 @@ fields("root") -> )}, {default_username, fun default_username/1}, {default_password, fun default_password/1}, - {sample_interval, mk(emqx_schema:duration_s(), #{default => "10s"})}, - {token_expired_time, mk(emqx_schema:duration(), #{default => "30m"})} + {sample_interval, mk(emqx_schema:duration_s(), #{default => <<"10s">>})}, + {token_expired_time, mk(emqx_schema:duration(), #{default => <<"30m">>})} ]; fields("ref1") -> [ @@ -52,7 +52,7 @@ fields("ref3") -> ]. default_username(type) -> string(); -default_username(default) -> "admin"; +default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index d17725e80..979d01c77 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -790,7 +790,7 @@ to_schema(Body) -> fields(good_ref) -> [ - {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 346f4ef71..c9cfba254 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -689,7 +689,7 @@ to_schema(Object) -> fields(good_ref) -> [ - {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index d81819c98..04e0a57db 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index 4d7de2866..bcfc68269 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -229,9 +229,9 @@ server_conf_schema() -> name => "default", enable => true, url => <<"http://127.0.0.1:8081">>, - request_timeout => "5s", + request_timeout => <<"5s">>, failed_action => deny, - auto_reconnect => "60s", + auto_reconnect => <<"60s">>, pool_size => 8, ssl => SSL } diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index ce79dddac..07373288d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -63,7 +63,7 @@ fields(server) -> })}, {request_timeout, ?HOCON(emqx_schema:duration(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(request_timeout) })}, {failed_action, failed_action()}, @@ -74,7 +74,7 @@ fields(server) -> })}, {auto_reconnect, ?HOCON(hoconsc:union([false, emqx_schema:duration()]), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(auto_reconnect) })}, {pool_size, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 4ea845ea1..2034a40eb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -267,7 +267,7 @@ fields(lwm2m) -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(lwm2m_lifetime_min) } )}, @@ -275,7 +275,7 @@ fields(lwm2m) -> sc( duration(), #{ - default => "86400s", + default => <<"86400s">>, desc => ?DESC(lwm2m_lifetime_max) } )}, @@ -283,7 +283,7 @@ fields(lwm2m) -> sc( duration_s(), #{ - default => "22s", + default => <<"22s">>, desc => ?DESC(lwm2m_qmode_time_window) } )}, @@ -624,7 +624,7 @@ mountpoint(Default) -> sc( binary(), #{ - default => Default, + default => iolist_to_binary(Default), desc => ?DESC(gateway_common_mountpoint) } ). @@ -707,7 +707,7 @@ proxy_protocol_opts() -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(tcp_listener_proxy_protocol_timeout) } )} diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index de56099ba..ed893c80d 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 8b3cca8fd..9d9d045de 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -78,11 +78,11 @@ states(_) -> undefined. install_dir(type) -> string(); install_dir(required) -> false; %% runner's root dir -install_dir(default) -> "plugins"; +install_dir(default) -> <<"plugins">>; install_dir(T) when T =/= desc -> undefined; install_dir(desc) -> ?DESC(install_dir). check_interval(type) -> emqx_schema:duration(); -check_interval(default) -> "5s"; +check_interval(default) -> <<"5s">>; check_interval(T) when T =/= desc -> undefined; check_interval(desc) -> ?DESC(check_interval). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index fcda5dea0..6ced0bf42 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -40,7 +40,7 @@ fields("prometheus") -> ?HOCON( string(), #{ - default => "http://127.0.0.1:9091", + default => <<"http://127.0.0.1:9091">>, required => true, validator => fun ?MODULE:validate_push_gateway_server/1, desc => ?DESC(push_gateway_server) @@ -50,7 +50,7 @@ fields("prometheus") -> ?HOCON( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, required => true, desc => ?DESC(interval) } diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 472ecc284..dbe1ad9d5 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -41,13 +41,13 @@ fields("retainer") -> sc( emqx_schema:duration_ms(), msg_expiry_interval, - "0s" + <<"0s">> )}, {msg_clear_interval, sc( emqx_schema:duration_ms(), msg_clear_interval, - "0s" + <<"0s">> )}, {flow_control, sc( @@ -59,7 +59,7 @@ fields("retainer") -> sc( emqx_schema:bytesize(), max_payload_size, - "1MB" + <<"1MB">> )}, {stop_publish_clear_msg, sc( diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index d6913cbc6..2281eea53 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -51,7 +51,7 @@ fields("rule_engine") -> ?HOCON( emqx_schema:duration_ms(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC("rule_engine_jq_function_default_timeout") } )}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 866655b61..170a4bb02 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.2"}, + {vsn, "1.0.3"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 8ae015ae4..9e9e6488a 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -30,13 +30,13 @@ fields("slow_subs") -> {threshold, sc( emqx_schema:duration_ms(), - "500ms", + <<"500ms">>, threshold )}, {expire_interval, sc( emqx_schema:duration_ms(), - "300s", + <<"300s">>, expire_interval )}, {top_k_num, diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index b1b3601aa..e65c93432 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -77,9 +77,9 @@ statsd_config_schema() -> statsd_example() -> #{ enable => true, - flush_time_interval => "30s", - sample_time_interval => "30s", - server => "127.0.0.1:8125", + flush_time_interval => <<"30s">>, + sample_time_interval => <<"30s">>, + server => <<"127.0.0.1:8125">>, tags => #{} }. diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index 1e5aa6e5f..e44f94954 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -61,12 +61,12 @@ server() -> emqx_schema:servers_sc(Meta, ?SERVER_PARSE_OPTS). sample_interval(type) -> emqx_schema:duration_ms(); -sample_interval(default) -> "30s"; +sample_interval(default) -> <<"30s">>; sample_interval(desc) -> ?DESC(?FUNCTION_NAME); sample_interval(_) -> undefined. flush_interval(type) -> emqx_schema:duration_ms(); -flush_interval(default) -> "30s"; +flush_interval(default) -> <<"30s">>; flush_interval(desc) -> ?DESC(?FUNCTION_NAME); flush_interval(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl index 1bee9e789..e00483839 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl @@ -50,7 +50,7 @@ fields(bridge_config) -> sc( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC("connect_timeout") } )}, @@ -84,7 +84,7 @@ fields(bridge_config) -> emqx_schema:duration_ms(), #{ required => false, - default => "15s", + default => <<"15s">>, desc => ?DESC("request_timeout") } )}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index e694f6c15..3983b235c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -84,20 +84,20 @@ fields("config") -> )}, {connect_timeout, mk(emqx_schema:duration_ms(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(connect_timeout) })}, {min_metadata_refresh_interval, mk( emqx_schema:duration_ms(), #{ - default => "3s", + default => <<"3s">>, desc => ?DESC(min_metadata_refresh_interval) } )}, {metadata_request_timeout, mk(emqx_schema:duration_ms(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(metadata_request_timeout) })}, {authentication, @@ -141,12 +141,12 @@ fields(socket_opts) -> {sndbuf, mk( emqx_schema:bytesize(), - #{default => "1024KB", desc => ?DESC(socket_send_buffer)} + #{default => <<"1024KB">>, desc => ?DESC(socket_send_buffer)} )}, {recbuf, mk( emqx_schema:bytesize(), - #{default => "1024KB", desc => ?DESC(socket_receive_buffer)} + #{default => <<"1024KB">>, desc => ?DESC(socket_receive_buffer)} )}, {nodelay, mk( @@ -170,7 +170,7 @@ fields(producer_kafka_opts) -> {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, - mk(emqx_schema:bytesize(), #{default => "896KB", desc => ?DESC(max_batch_bytes)})}, + mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, {compression, mk(enum([no_compression, snappy, gzip]), #{ default => no_compression, desc => ?DESC(compression) @@ -192,7 +192,7 @@ fields(producer_kafka_opts) -> mk( emqx_schema:duration_s(), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(partition_count_refresh_interval) } )}, @@ -212,11 +212,11 @@ fields(producer_kafka_opts) -> ]; fields(kafka_message) -> [ - {key, mk(string(), #{default => "${.clientid}", desc => ?DESC(kafka_message_key)})}, - {value, mk(string(), #{default => "${.}", desc => ?DESC(kafka_message_value)})}, + {key, mk(string(), #{default => <<"${.clientid}">>, desc => ?DESC(kafka_message_key)})}, + {value, mk(string(), #{default => <<"${.}">>, desc => ?DESC(kafka_message_value)})}, {timestamp, mk(string(), #{ - default => "${.timestamp}", desc => ?DESC(kafka_message_timestamp) + default => <<"${.timestamp}">>, desc => ?DESC(kafka_message_timestamp) })} ]; fields(producer_buffer) -> @@ -229,12 +229,12 @@ fields(producer_buffer) -> {per_partition_limit, mk( emqx_schema:bytesize(), - #{default => "2GB", desc => ?DESC(buffer_per_partition_limit)} + #{default => <<"2GB">>, desc => ?DESC(buffer_per_partition_limit)} )}, {segment_bytes, mk( emqx_schema:bytesize(), - #{default => "100MB", desc => ?DESC(buffer_segment_bytes)} + #{default => <<"100MB">>, desc => ?DESC(buffer_segment_bytes)} )}, {memory_overload_protection, mk(boolean(), #{ diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index 9d16f697c..7383af92c 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -46,12 +46,12 @@ fields(key_license) -> }}, {connection_low_watermark, #{ type => emqx_schema:percent(), - default => "75%", + default => <<"75%">>, desc => ?DESC(connection_low_watermark_field) }}, {connection_high_watermark, #{ type => emqx_schema:percent(), - default => "80%", + default => <<"80%">>, desc => ?DESC(connection_high_watermark_field) }} ]. From f9895a33b6c545f7528964e3f2802687e5dd67d7 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 15 Feb 2023 14:46:45 +0100 Subject: [PATCH 081/135] refactor: rename max_row_limit to default_row_limit --- apps/emqx_management/include/emqx_mgmt.hrl | 3 ++- apps/emqx_management/src/emqx_mgmt.erl | 26 ++++++++----------- apps/emqx_management/src/emqx_mgmt_api.erl | 6 ++--- apps/emqx_management/src/emqx_mgmt_util.erl | 2 +- .../test/emqx_mgmt_api_alarms_SUITE.erl | 2 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 2 +- .../test/emqx_mgmt_api_subscription_SUITE.erl | 6 ++--- .../test/emqx_mgmt_api_topics_SUITE.erl | 2 +- apps/emqx_retainer/src/emqx_retainer_api.erl | 2 +- 9 files changed, 24 insertions(+), 27 deletions(-) diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index b68a9a634..12be1df4f 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -16,4 +16,5 @@ -define(MANAGEMENT_SHARD, emqx_management_shard). --define(MAX_ROW_LIMIT, 100). +-define(DEFAULT_ROW_LIMIT, 100). +-define(MAX_TABLE_SIZE, 100). diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 2d6fa854e..620c1e3f9 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -21,8 +21,6 @@ -elvis([{elvis_style, god_modules, disable}]). -include_lib("stdlib/include/qlc.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). %% Nodes and Brokers API -export([ @@ -105,12 +103,10 @@ %% Common Table API -export([ - max_row_limit/0, + default_row_limit/0, vm_stats/0 ]). --define(APP, emqx_management). - -elvis([{elvis_style, god_modules, disable}]). %%-------------------------------------------------------------------- @@ -390,10 +386,10 @@ call_client(Node, ClientId, Req) -> -spec do_list_subscriptions() -> [map()]. do_list_subscriptions() -> - case check_row_limit([mqtt_subproperty]) of + case check_max_table_size([mqtt_subproperty]) of false -> throw(max_row_limit); - ok -> + true -> [ #{topic => Topic, clientid => ClientId, options => Options} || {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty) @@ -556,15 +552,15 @@ unwrap_rpc(Res) -> otp_rel() -> iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]). -check_row_limit(Tables) -> - check_row_limit(Tables, max_row_limit()). +check_max_table_size(Tables) -> + check_max_table_size(Tables, ?MAX_TABLE_SIZE). -check_row_limit([], _Limit) -> - ok; -check_row_limit([Tab | Tables], Limit) -> +check_max_table_size([], _Limit) -> + true; +check_max_table_size([Tab | Tables], Limit) -> case table_size(Tab) > Limit of true -> false; - false -> check_row_limit(Tables, Limit) + false -> check_max_table_size(Tables, Limit) end. check_results(Results) -> @@ -573,7 +569,7 @@ check_results(Results) -> false -> unwrap_rpc(lists:last(Results)) end. -max_row_limit() -> - ?MAX_ROW_LIMIT. +default_row_limit() -> + ?DEFAULT_ROW_LIMIT. table_size(Tab) -> ets:info(Tab, size). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 3c4d787d3..a0a40533d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -98,8 +98,8 @@ count(Table) -> page(Params) -> maps:get(<<"page">>, Params, 1). -limit(Params) -> - maps:get(<<"limit">>, Params, emqx_mgmt:max_row_limit()). +limit(Params) when is_map(Params) -> + maps:get(<<"limit">>, Params, emqx_mgmt:default_row_limit()). %%-------------------------------------------------------------------- %% Node Query @@ -683,7 +683,7 @@ paginate_test_() -> Size = 1000, MyLimit = 10, ets:insert(?MODULE, [{I, foo} || I <- lists:seq(1, Size)]), - DefaultLimit = emqx_mgmt:max_row_limit(), + DefaultLimit = emqx_mgmt:default_row_limit(), NoParamsResult = paginate(?MODULE, #{}, {?MODULE, paginate_test_format}), PaginateResults = [ paginate( diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index c0d9e6036..b81b39b07 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -302,7 +302,7 @@ page_params() -> name => limit, in => query, description => <<"Page size">>, - schema => #{type => integer, default => emqx_mgmt:max_row_limit()} + schema => #{type => integer, default => emqx_mgmt:default_row_limit()} } ]. diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 2c61651bf..69ace16e8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -62,5 +62,5 @@ get_alarms(AssertCount, Activated) -> Limit = maps:get(<<"limit">>, Meta), Count = maps:get(<<"count">>, Meta), ?assertEqual(Page, 1), - ?assertEqual(Limit, emqx_mgmt:max_row_limit()), + ?assertEqual(Limit, emqx_mgmt:default_row_limit()), ?assert(Count >= AssertCount). diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 1a74d3af6..c7f4c9845 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -64,7 +64,7 @@ t_clients(_) -> ClientsLimit = maps:get(<<"limit">>, ClientsMeta), ClientsCount = maps:get(<<"count">>, ClientsMeta), ?assertEqual(ClientsPage, 1), - ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()), + ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()), ?assertEqual(ClientsCount, 2), %% get /clients/:clientid diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 2ab213e30..ccfa30037 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -57,7 +57,7 @@ t_subscription_api(Config) -> Data = emqx_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), ?assertEqual(2, maps:get(<<"count">>, Meta)), Subscriptions = maps:get(<<"data">>, Data), ?assertEqual(length(Subscriptions), 2), @@ -95,7 +95,7 @@ t_subscription_api(Config) -> DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers), ?assertEqual(1, maps:get(<<"page">>, Meta2)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta2)), ?assertEqual(1, maps:get(<<"count">>, Meta2)), SubscriptionsList2 = maps:get(<<"data">>, DataTopic2), ?assertEqual(length(SubscriptionsList2), 1). @@ -120,7 +120,7 @@ t_subscription_fuzzy_search(Config) -> MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers), ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, MatchMeta1)), %% count is undefined in fuzzy searching ?assertNot(maps:is_key(<<"count">>, MatchMeta1)), ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))), diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 8f9b224ef..0c2e684b4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -52,7 +52,7 @@ t_nodes_api(Config) -> RoutesData = emqx_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), ?assertEqual(1, maps:get(<<"count">>, Meta)), Data = maps:get(<<"data">>, RoutesData), Route = erlang:hd(Data), diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index fa11b00f4..7b1337140 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -166,7 +166,7 @@ config(put, #{body := Body}) -> %%------------------------------------------------------------------------------ lookup_retained(get, #{query_string := Qs}) -> Page = maps:get(<<"page">>, Qs, 1), - Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:max_row_limit()), + Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:default_row_limit()), {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, undefined, Page, Limit), {200, #{ data => [format_message(Msg) || Msg <- Msgs], From 777ca72ad56e9fbec2b8f440018e43461fa17957 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 15 Feb 2023 15:54:05 +0100 Subject: [PATCH 082/135] fix: don't crash on broker_info() --- apps/emqx_management/src/emqx_mgmt.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 620c1e3f9..7a48db0f4 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -190,9 +190,15 @@ lookup_broker(Node) -> Broker. broker_info() -> - Info = maps:from_list([{K, iolist_to_binary(V)} || {K, V} <- emqx_sys:info()]), + Info = lists:foldl(fun convert_broker_info/2, #{}, emqx_sys:info()), Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}. +convert_broker_info({uptime, Uptime}, M) -> + M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)}; +convert_broker_info({K, V}, M) -> + M#{K => iolist_to_binary(V)}. + + broker_info(Nodes) -> emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)). From fc33bce40d267688fbe04984ce1403f5716c5d86 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 16 Feb 2023 09:17:32 +0100 Subject: [PATCH 083/135] test(emqx_mgmt): test list_nodes/0 --- apps/emqx/test/emqx_common_test_helpers.erl | 15 +++++ apps/emqx_management/src/emqx_mgmt.erl | 2 +- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 61 +++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_management/test/emqx_mgmt_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index fe1dfa35e..5149b8b8a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -22,6 +22,8 @@ -export([ all/1, + init_per_testcase/3, + end_per_testcase/3, boot_modules/1, start_apps/1, start_apps/2, @@ -150,6 +152,19 @@ all(Suite) -> string:substr(atom_to_list(F), 1, 2) == "t_" ]). +init_per_testcase(Module, TestCase, Config) -> + case erlang:function_exported(Module, TestCase, 2) of + true -> Module:TestCase(init, Config); + false -> Config + end. + +end_per_testcase(Module, TestCase, Config) -> + case erlang:function_exported(Module, TestCase, 2) of + true -> Module:TestCase('end', Config); + false -> ok + end, + Config. + %% set emqx app boot modules -spec boot_modules(all | list(atom())) -> ok. boot_modules(Mods) -> diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 7a48db0f4..7f00260b2 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -158,7 +158,7 @@ node_info(Nodes) -> emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)). stopped_node_info(Node) -> - #{name => Node, node_status => 'stopped'}. + {Node, #{node => Node, node_status => 'stopped'}}. vm_stats() -> Idle = diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl new file mode 100644 index 000000000..68957e891 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -0,0 +1,61 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_management]), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]). + +init_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). + +end_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). + +t_list_nodes(init, Config) -> + meck:expect( + mria_mnesia, + cluster_nodes, + fun + (running) -> [node()]; + (stopped) -> ['stopped@node'] + end + ), + Config; +t_list_nodes('end', _Config) -> + meck:unload(mria_mnesia). + +t_list_nodes(_) -> + NodeInfos = emqx_mgmt:list_nodes(), + Node = node(), + ?assertMatch( + [ + {Node, #{node := Node, node_status := 'running'}}, + {'stopped@node', #{node := 'stopped@node', node_status := 'stopped'}} + ], + NodeInfos + ). From a6d88c3caa455d44984d083fc4f7fdaf53713bd7 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 16 Feb 2023 16:47:44 +0100 Subject: [PATCH 084/135] test: more tests --- apps/emqx_management/src/emqx_mgmt.erl | 2 +- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 98 ++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 7f00260b2..52c50fda6 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -191,7 +191,7 @@ lookup_broker(Node) -> broker_info() -> Info = lists:foldl(fun convert_broker_info/2, #{}, emqx_sys:info()), - Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}. + Info#{node => node(), otp_release => otp_rel(), node_status => 'running'}. convert_broker_info({uptime, Uptime}, M) -> M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)}; diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 68957e891..28533cac5 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -19,6 +19,9 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-export([ident/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -31,9 +34,11 @@ end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]). init_per_testcase(TestCase, Config) -> + meck:expect(mria_mnesia, running_nodes, 0, [node()]), emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). end_per_testcase(TestCase, Config) -> + meck:unload(mria_mnesia), emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). t_list_nodes(init, Config) -> @@ -47,7 +52,7 @@ t_list_nodes(init, Config) -> ), Config; t_list_nodes('end', _Config) -> - meck:unload(mria_mnesia). + ok. t_list_nodes(_) -> NodeInfos = emqx_mgmt:list_nodes(), @@ -59,3 +64,94 @@ t_list_nodes(_) -> ], NodeInfos ). + +t_lookup_node(init, Config) -> + meck:new(os, [passthrough, unstick, no_link]), + OsType = os:type(), + meck:expect(os, type, 0, {win32, winME}), + [{os_type, OsType} | Config]; +t_lookup_node('end', Config) -> + %% We need to restore the original behavior so that rebar3 doesn't crash. If + %% we'd `meck:unload(os)` or not set `no_link` then `ct` crashes calling + %% `os` with "The code server called the unloaded module `os'". + OsType = ?config(os_type, Config), + meck:expect(os, type, 0, OsType), + ok. + +t_lookup_node(_) -> + Node = node(), + ?assertMatch( + #{node := Node, node_status := 'running', memory_total := 0}, + emqx_mgmt:lookup_node(node()) + ), + ?assertMatch( + {error, _}, + emqx_mgmt:lookup_node('fake@nohost') + ), + ok. + +t_list_brokers(_) -> + Node = node(), + ?assertMatch( + [{Node, #{node := Node, node_status := running, uptime := _}}], + emqx_mgmt:list_brokers() + ). + +t_lookup_broker(_) -> + Node = node(), + ?assertMatch( + #{node := Node, node_status := running, uptime := _}, + emqx_mgmt:lookup_broker(Node) + ). + +t_get_metrics(_) -> + Metrics = emqx_mgmt:get_metrics(), + ?assert(maps:size(Metrics) > 0), + ?assertMatch( + Metrics, maps:from_list(emqx_mgmt:get_metrics(node())) + ). + +t_lookup_client(init, Config) -> + setup_clients(Config); +t_lookup_client('end', Config) -> + disconnect_clients(Config). + +t_lookup_client(_Config) -> + [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, {?MODULE, ident}), + ?assertEqual( + [{Chan, Info, Stats}], + emqx_mgmt:lookup_client({username, <<"user1">>}, {?MODULE, ident}) + ), + ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, {?MODULE, ident})). + +t_kickout_client(init, Config) -> + process_flag(trap_exit, true), + setup_clients(Config); +t_kickout_client('end', _Config) -> + ok. + +t_kickout_client(Config) -> + [C | _] = ?config(clients, Config), + ok = emqx_mgmt:kickout_client({<<"client1">>, {?MODULE, ident}}), + receive + {'EXIT', C, Reason} -> + ?assertEqual({shutdown, tcp_closed}, Reason); + Foo -> + error({unexpected, Foo}) + after 1000 -> + error(timeout) + end, + ?assertEqual({error, not_found}, emqx_mgmt:kickout_client({<<"notfound">>, {?MODULE, ident}})). + +%%% helpers +ident(Arg) -> + Arg. + +setup_clients(Config) -> + {ok, C} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"user1">>}]), + {ok, _} = emqtt:connect(C), + [{clients, [C]} | Config]. + +disconnect_clients(Config) -> + Clients = ?config(clients, Config), + lists:foreach(fun emqtt:disconnect/1, Clients). From f3ced5d5eb02c01d21323f9a6f43e4c8a1063684 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 16 Feb 2023 17:04:15 +0100 Subject: [PATCH 085/135] refactor: kickout_client doesn't need a format fun --- apps/emqx_management/src/emqx_mgmt.erl | 13 +++++++++---- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 2 +- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 12 +++++++----- .../test/emqx_mgmt_api_clients_SUITE.erl | 9 ++++++++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 52c50fda6..04f7d9ad0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -267,7 +267,7 @@ lookup_client({username, Username}, FormatFun) -> || Node <- mria_mnesia:running_nodes() ]). -lookup_client(Node, Key, {M, F}) -> +lookup_client(Node, Key, FormatFun) -> case unwrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of {error, Err} -> {error, Err}; @@ -275,14 +275,19 @@ lookup_client(Node, Key, {M, F}) -> lists:map( fun({Chan, Info0, Stats}) -> Info = Info0#{node => Node}, - M:F({Chan, Info, Stats}) + maybe_format(FormatFun, {Chan, Info, Stats}) end, L ) end. -kickout_client({ClientID, FormatFun}) -> - case lookup_client({clientid, ClientID}, FormatFun) of +maybe_format(undefined, A) -> + A; +maybe_format({M, F}, A) -> + M:F(A). + +kickout_client(ClientID) -> + case lookup_client({clientid, ClientID}, undefined) of [] -> {error, not_found}; _ -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 571f190f2..0a5488389 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -677,7 +677,7 @@ lookup(#{clientid := ClientID}) -> end. kickout(#{clientid := ClientID}) -> - case emqx_mgmt:kickout_client({ClientID, ?FORMAT_FUN}) of + case emqx_mgmt:kickout_client(ClientID) of {error, not_found} -> {404, ?CLIENTID_NOT_FOUND}; _ -> diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 28533cac5..13acb9fc3 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -23,6 +23,8 @@ -export([ident/1]). +-define(FORMATFUN, {?MODULE, ident}). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -117,12 +119,12 @@ t_lookup_client('end', Config) -> disconnect_clients(Config). t_lookup_client(_Config) -> - [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, {?MODULE, ident}), + [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN), ?assertEqual( [{Chan, Info, Stats}], - emqx_mgmt:lookup_client({username, <<"user1">>}, {?MODULE, ident}) + emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN) ), - ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, {?MODULE, ident})). + ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)). t_kickout_client(init, Config) -> process_flag(trap_exit, true), @@ -132,7 +134,7 @@ t_kickout_client('end', _Config) -> t_kickout_client(Config) -> [C | _] = ?config(clients, Config), - ok = emqx_mgmt:kickout_client({<<"client1">>, {?MODULE, ident}}), + ok = emqx_mgmt:kickout_client(<<"client1">>), receive {'EXIT', C, Reason} -> ?assertEqual({shutdown, tcp_closed}, Reason); @@ -141,7 +143,7 @@ t_kickout_client(Config) -> after 1000 -> error(timeout) end, - ?assertEqual({error, not_found}, emqx_mgmt:kickout_client({<<"notfound">>, {?MODULE, ident}})). + ?assertEqual({error, not_found}, emqx_mgmt:kickout_client(<<"notfound">>)). %%% helpers ident(Arg) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index c7f4c9845..d843a2209 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -78,7 +78,14 @@ t_clients(_) -> %% delete /clients/:clientid kickout Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path), - timer:sleep(300), + Kick = + receive + {'EXIT', C2, _} -> + ok + after 300 -> + timeout + end, + ?assertEqual(ok, Kick), AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2), From 0d2ce85776c5cba495489090cd7ef5dd31c8c74c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 16 Feb 2023 17:39:49 +0100 Subject: [PATCH 086/135] fix: return 'not found' for subscriptions of unknown client --- apps/emqx_management/src/emqx_mgmt.erl | 33 +++++++++++-------- .../src/emqx_mgmt_api_clients.erl | 9 ++--- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 21 ++++++++++++ .../test/emqx_mgmt_api_clients_SUITE.erl | 2 +- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 04f7d9ad0..31436b9a0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -286,12 +286,12 @@ maybe_format(undefined, A) -> maybe_format({M, F}, A) -> M:F(A). -kickout_client(ClientID) -> - case lookup_client({clientid, ClientID}, undefined) of +kickout_client(ClientId) -> + case lookup_client({clientid, ClientId}, undefined) of [] -> {error, not_found}; _ -> - Results = [kickout_client(Node, ClientID) || Node <- mria_mnesia:running_nodes()], + Results = [kickout_client(Node, ClientId) || Node <- mria_mnesia:running_nodes()], check_results(Results) end. @@ -302,17 +302,22 @@ list_authz_cache(ClientId) -> call_client(ClientId, list_authz_cache). list_client_subscriptions(ClientId) -> - Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()], - Filter = - fun - ({error, _}) -> - false; - ({_Node, List}) -> - erlang:is_list(List) andalso 0 < erlang:length(List) - end, - case lists:filter(Filter, Results) of - [] -> []; - [Result | _] -> Result + case lookup_client({clientid, ClientId}, undefined) of + [] -> + {error, not_found}; + _ -> + Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()], + Filter = + fun + ({error, _}) -> + false; + ({_Node, List}) -> + erlang:is_list(List) andalso 0 < erlang:length(List) + end, + case lists:filter(Filter, Results) of + [] -> []; + [Result | _] -> Result + end end. client_subscriptions(Node, ClientId) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 0a5488389..cac3edaed 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -274,11 +274,10 @@ schema("/clients/:clientid/subscriptions") -> responses => #{ 200 => hoconsc:mk( hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{} + ), + 404 => emqx_dashboard_swagger:error_codes( + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) - %% returns [] if client not existed in cluster - %404 => emqx_dashboard_swagger:error_codes( - % ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> - %) } } }; @@ -599,6 +598,8 @@ unsubscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfo subscriptions(get, #{bindings := #{clientid := ClientID}}) -> case emqx_mgmt:list_client_subscriptions(ClientID) of + {error, not_found} -> + {404, ?CLIENTID_NOT_FOUND}; [] -> {200, []}; {Node, Subs} -> diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 13acb9fc3..a9379da8c 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -145,6 +145,27 @@ t_kickout_client(Config) -> end, ?assertEqual({error, not_found}, emqx_mgmt:kickout_client(<<"notfound">>)). +t_list_authz_cache(init, Config) -> + setup_clients(Config); +t_list_authz_cache('end', Config) -> + disconnect_clients(Config). + +t_list_authz_cache(_) -> + ?assertNotMatch({error, _}, emqx_mgmt:list_authz_cache(<<"client1">>)), + ?assertMatch({error, not_found}, emqx_mgmt:list_authz_cache(<<"notfound">>)). + +t_list_client_subscriptions(init, Config) -> + setup_clients(Config); +t_list_client_subscriptions('end', Config) -> + disconnect_clients(Config). + +t_list_client_subscriptions(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_client_subscriptions(<<"client1">>)), + emqtt:subscribe(Client, <<"t/#">>), + ?assertMatch({_, [{<<"t/#">>, _Opts}]}, emqx_mgmt:list_client_subscriptions(<<"client1">>)), + ?assertEqual({error, not_found}, emqx_mgmt:list_client_subscriptions(<<"notfound">>)). + %%% helpers ident(Arg) -> Arg. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index d843a2209..b2f3f5655 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -279,7 +279,7 @@ t_client_id_not_found(_Config) -> %% Client kickout ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun([]))), %% Client Subscription list - ?assertMatch({ok, {{"HTTP/1.1", 200, "OK"}, _, "[]"}}, ReqFun(get, PathFun(["subscriptions"]))), + ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["subscriptions"]))), %% AuthZ Cache lookup ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["authorization", "cache"]))), %% AuthZ Cache clean From 0d357f7038e8d96a760cff0367a191e3650e958e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 17 Feb 2023 14:56:16 +0100 Subject: [PATCH 087/135] refactor: cleanup list subscriptions --- apps/emqx_management/src/emqx_mgmt.erl | 41 +---------- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 71 +++++++++++++++++++ .../test/emqx_mgmt_api_clients_SUITE.erl | 4 +- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 31436b9a0..27a3d8b69 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -66,13 +66,8 @@ %% Subscriptions -export([ - list_subscriptions/1, list_subscriptions_via_topic/2, - list_subscriptions_via_topic/3, - lookup_subscriptions/1, - lookup_subscriptions/2, - - do_list_subscriptions/0 + list_subscriptions_via_topic/3 ]). %% PubSub @@ -400,21 +395,6 @@ call_client(Node, ClientId, Req) -> %% Subscriptions %%-------------------------------------------------------------------- --spec do_list_subscriptions() -> [map()]. -do_list_subscriptions() -> - case check_max_table_size([mqtt_subproperty]) of - false -> - throw(max_row_limit); - true -> - [ - #{topic => Topic, clientid => ClientId, options => Options} - || {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty) - ] - end. - -list_subscriptions(Node) -> - unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)). - list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ list_subscriptions_via_topic(Node, Topic, FormatFun) @@ -427,12 +407,6 @@ list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> Result -> M:F(Result) end. -lookup_subscriptions(ClientId) -> - lists:append([lookup_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()]). - -lookup_subscriptions(Node, ClientId) -> - unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId)). - %%-------------------------------------------------------------------- %% PubSub %%-------------------------------------------------------------------- @@ -568,17 +542,6 @@ unwrap_rpc(Res) -> otp_rel() -> iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]). -check_max_table_size(Tables) -> - check_max_table_size(Tables, ?MAX_TABLE_SIZE). - -check_max_table_size([], _Limit) -> - true; -check_max_table_size([Tab | Tables], Limit) -> - case table_size(Tab) > Limit of - true -> false; - false -> check_max_table_size(Tables, Limit) - end. - check_results(Results) -> case lists:any(fun(Item) -> Item =:= ok end, Results) of true -> ok; @@ -587,5 +550,3 @@ check_results(Results) -> default_row_limit() -> ?DEFAULT_ROW_LIMIT. - -table_size(Tab) -> ets:info(Tab, size). diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index a9379da8c..f5f3acd53 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -166,6 +166,77 @@ t_list_client_subscriptions(Config) -> ?assertMatch({_, [{<<"t/#">>, _Opts}]}, emqx_mgmt:list_client_subscriptions(<<"client1">>)), ?assertEqual({error, not_found}, emqx_mgmt:list_client_subscriptions(<<"notfound">>)). +t_clean_cache(init, Config) -> + setup_clients(Config); +t_clean_cache('end', Config) -> + disconnect_clients(Config). + +t_clean_cache(_Config) -> + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_authz_cache(<<"client1">>) + ), + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_authz_cache_all() + ), + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_pem_cache_all() + ). + +t_set_client_props(init, Config) -> + setup_clients(Config); +t_set_client_props('end', Config) -> + disconnect_clients(Config). + +t_set_client_props(_Config) -> + ?assertEqual( + % [FIXME] not implemented at this point? + ignored, + emqx_mgmt:set_ratelimit_policy(<<"client1">>, foo) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_ratelimit_policy(<<"notfound">>, foo) + ), + ?assertEqual( + % [FIXME] not implemented at this point? + ignored, + emqx_mgmt:set_quota_policy(<<"client1">>, foo) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_quota_policy(<<"notfound">>, foo) + ), + ?assertEqual( + ok, + emqx_mgmt:set_keepalive(<<"client1">>, 3600) + ), + ?assertMatch( + {error, _}, + emqx_mgmt:set_keepalive(<<"client1">>, true) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_keepalive(<<"notfound">>, 3600) + ), + ok. + +t_list_subscriptions_via_topic(init, Config) -> + setup_clients(Config); +t_list_subscriptions_via_topic('end', Config) -> + disconnect_clients(Config). + +t_list_subscriptions_via_topic(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)), + emqtt:subscribe(Client, <<"t/#">>), + ?assertMatch( + [{{<<"t/#">>, _SubPid}, _Opts}], + emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) + ). + %%% helpers ident(Arg) -> Arg. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index b2f3f5655..9f26f8542 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -114,7 +114,7 @@ t_clients(_) -> SubscribeBody ), timer:sleep(100), - [{AfterSubTopic, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1), + {_, [{AfterSubTopic, #{qos := AfterSubQos}}]} = emqx_mgmt:list_client_subscriptions(ClientId1), ?assertEqual(AfterSubTopic, Topic), ?assertEqual(AfterSubQos, Qos), @@ -159,7 +159,7 @@ t_clients(_) -> UnSubscribeBody ), timer:sleep(100), - ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)), + ?assertEqual([], emqx_mgmt:list_client_subscriptions(ClientId1)), %% testcase cleanup, kickout client1 {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client1Path), From 0482f438029844a9ff2eba3a7f4147928c0ef3a3 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 17 Feb 2023 15:51:13 +0100 Subject: [PATCH 088/135] test: add test for banned and alarms --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index f5f3acd53..993969d48 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -237,10 +237,75 @@ t_list_subscriptions_via_topic(Config) -> emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) ). +t_alarms(_) -> + Node = node(), + ?assertEqual( + [{node(), []}], + emqx_mgmt:get_alarms(all) + ), + emqx_alarm:activate(foo), + ?assertMatch( + [{Node, [#{name := foo, activated := true, duration := _}]}], + emqx_mgmt:get_alarms(all) + ), + emqx_alarm:activate(bar), + ?assertMatch( + [{Node, [#{name := foo, activated := true}, #{name := bar, activated := true}]}], + sort_alarms(emqx_mgmt:get_alarms(all)) + ), + ?assertEqual( + ok, + emqx_mgmt:deactivate(node(), bar) + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}, #{name := bar, activated := false}]}], + sort_alarms(emqx_mgmt:get_alarms(all)) + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}]}], + emqx_mgmt:get_alarms(activated) + ), + ?assertMatch( + [{Node, [#{name := bar, activated := false}]}], + emqx_mgmt:get_alarms(deactivated) + ), + ?assertEqual( + [ok], + emqx_mgmt:delete_all_deactivated_alarms() + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}]}], + emqx_mgmt:get_alarms(all) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:deactivate(node(), bar) + ). + +t_banned(_) -> + Banned = #{ + who => {clientid, <<"TestClient">>}, + by => <<"banned suite">>, + reason => <<"test">>, + at => erlang:system_time(second), + until => erlang:system_time(second) + 1 + }, + ?assertMatch( + {ok, _}, + emqx_mgmt:create_banned(Banned) + ), + ?assertEqual( + ok, + emqx_mgmt:delete_banned({clientid, <<"TestClient">>}) + ). + %%% helpers ident(Arg) -> Arg. +sort_alarms([{Node, Alarms}]) -> + [{Node, lists:sort(fun(#{activate_at := A}, #{activate_at := B}) -> A < B end, Alarms)}]. + setup_clients(Config) -> {ok, C} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"user1">>}]), {ok, _} = emqtt:connect(C), From c44c7fcbcea921529ad482678a60de4d45846c63 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 17 Feb 2023 15:55:30 +0100 Subject: [PATCH 089/135] style: remove unused macro --- apps/emqx_management/include/emqx_mgmt.hrl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index 12be1df4f..7f6b5a675 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -17,4 +17,3 @@ -define(MANAGEMENT_SHARD, emqx_management_shard). -define(DEFAULT_ROW_LIMIT, 100). --define(MAX_TABLE_SIZE, 100). From 71f3efb2ce80c00c94ef286983426684c8d7350c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 17 Feb 2023 15:57:53 +0100 Subject: [PATCH 090/135] style: fix empty line --- apps/emqx_management/src/emqx_mgmt.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 27a3d8b69..e7c338adf 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -193,7 +193,6 @@ convert_broker_info({uptime, Uptime}, M) -> convert_broker_info({K, V}, M) -> M#{K => iolist_to_binary(V)}. - broker_info(Nodes) -> emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)). From 81fad58f12db7d63dd9c3c7562c4b6928d74324b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 20 Feb 2023 10:47:25 +0100 Subject: [PATCH 091/135] fix: re-add `list_subscriptions/0` Created a ticket to add an actual working implementation --- apps/emqx_management/src/emqx_mgmt.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index e7c338adf..efa5a03bd 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -66,8 +66,11 @@ %% Subscriptions -export([ + list_subscriptions/1, list_subscriptions_via_topic/2, - list_subscriptions_via_topic/3 + list_subscriptions_via_topic/3, + + do_list_subscriptions/0 ]). %% PubSub @@ -394,6 +397,15 @@ call_client(Node, ClientId, Req) -> %% Subscriptions %%-------------------------------------------------------------------- +-spec do_list_subscriptions() -> no_return(). +do_list_subscriptions() -> + %% [FIXME] Add function to `emqx_broker` that returns list of subscriptions + %% and either redirect from here or bpapi directly (EMQX-8993). + throw(not_implemented). + +list_subscriptions(Node) -> + unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)). + list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ list_subscriptions_via_topic(Node, Topic, FormatFun) From 8ae444006122442971952c5fca2d7c1aea54cbce Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 20 Feb 2023 16:51:50 +0100 Subject: [PATCH 092/135] style: fix API description for bytes parameter --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index cc4a905a4..624e0bdb8 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -315,7 +315,7 @@ fields(bytes) -> hoconsc:mk( integer(), #{ - desc => "Maximum number of bytes to store in request", + desc => "Maximum number of bytes to send in response", in => query, required => false, default => 1000 From 9ecf154a711ca13be0beb879274db45646c68e42 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 20 Feb 2023 17:05:41 +0100 Subject: [PATCH 093/135] fix: limit bytes param to signed 32bit int We still need to check if chunk we're reading fits in memory --- .../src/emqx_mgmt_api_trace.erl | 30 ++++++++++++++++--- .../test/emqx_mgmt_api_trace_SUITE.erl | 11 +++++-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 624e0bdb8..594750d06 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -47,9 +47,12 @@ get_trace_size/0 ]). +-define(MAX_SINT32, 2147483647). + -define(TO_BIN(_B_), iolist_to_binary(_B_)). -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}). -define(BAD_REQUEST(C, M), {400, #{code => C, message => ?TO_BIN(M)}}). +-define(SERVICE_UNAVAILABLE(C, M), {503, #{code => C, message => ?TO_BIN(M)}}). -define(TAGS, [<<"Trace">>]). namespace() -> "trace". @@ -184,8 +187,15 @@ schema("/trace/:name/log") -> {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}, {meta, fields(bytes) ++ fields(position)} ], - 400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Trace Log Failed">>), - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST', 'NODE_ERROR'], <<"Bad input parameter">> + ), + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND'], <<"Trace Name Not Found">> + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Requested chunk size too big">> + ) } } }. @@ -313,12 +323,16 @@ fields(bytes) -> [ {bytes, hoconsc:mk( - integer(), + %% This seems to be the minimum max value we may encounter + %% across different OS + range(0, ?MAX_SINT32), #{ desc => "Maximum number of bytes to send in response", in => query, required => false, - default => 1000 + default => 1000, + minimum => 0, + maximum => ?MAX_SINT32 } )} ]; @@ -579,6 +593,14 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> {200, #{meta => Meta, items => <<"">>}}; {error, not_found} -> ?NOT_FOUND(Name); + {error, enomem} -> + ?SLOG(warning, #{ + code => not_enough_mem, + msg => "Requested chunk size too big", + bytes => Bytes, + name => Name + }), + ?SERVICE_UNAVAILABLE('SERVICE_UNAVAILABLE', <<"Requested chunk size too big">>); {badrpc, nodedown} -> ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) end; diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 6962a9043..92b35db99 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -19,9 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("kernel/include/file.hrl"). -include_lib("stdlib/include/zip.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -296,6 +294,15 @@ t_stream_log(_Config) -> #{<<"meta">> := Meta1, <<"items">> := Bin1} = json(Binary1), ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1), ?assertEqual(10, byte_size(Bin1)), + ct:pal("~p vs ~p", [Bin, Bin1]), + %% in theory they could be the same but we know they shouldn't + ?assertNotEqual(Bin, Bin1), + BadReqPath = api_path("trace/test_stream_log/log?&bytes=1000000000000"), + {error, {_, 400, _}} = request_api(get, BadReqPath), + meck:new(file, [passthrough, unstick]), + meck:expect(file, read, 2, {error, enomem}), + {error, {_, 503, _}} = request_api(get, Path), + meck:unload(file), {error, {_, 400, _}} = request_api( get, From e78c2c2869d77f246e2d840f1b11d6c0a24911fb Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 20 Feb 2023 17:07:36 +0100 Subject: [PATCH 094/135] fix: return 404 in case node is not found --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 16 ++++++++-------- .../test/emqx_mgmt_api_trace_SUITE.erl | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 594750d06..38ce9dcf2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -51,7 +51,6 @@ -define(TO_BIN(_B_), iolist_to_binary(_B_)). -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}). --define(BAD_REQUEST(C, M), {400, #{code => C, message => ?TO_BIN(M)}}). -define(SERVICE_UNAVAILABLE(C, M), {503, #{code => C, message => ?TO_BIN(M)}}). -define(TAGS, [<<"Trace">>]). @@ -151,8 +150,9 @@ schema("/trace/:name/download") -> #{schema => #{type => "string", format => "binary"}} } }, - 400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Node Not Found">>), - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">> + ) } } }; @@ -188,10 +188,10 @@ schema("/trace/:name/log") -> {meta, fields(bytes) ++ fields(position)} ], 400 => emqx_dashboard_swagger:error_codes( - ['BAD_REQUEST', 'NODE_ERROR'], <<"Bad input parameter">> + ['BAD_REQUEST'], <<"Bad input parameter">> ), 404 => emqx_dashboard_swagger:error_codes( - ['NOT_FOUND'], <<"Trace Name Not Found">> + ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">> ), 503 => emqx_dashboard_swagger:error_codes( ['SERVICE_UNAVAILABLE'], <<"Requested chunk size too big">> @@ -509,7 +509,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - }, {200, Headers, {file_binary, ZipName, Binary}}; {error, not_found} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end; {error, not_found} -> ?NOT_FOUND(Name) @@ -602,10 +602,10 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> }), ?SERVICE_UNAVAILABLE('SERVICE_UNAVAILABLE', <<"Requested chunk size too big">>); {badrpc, nodedown} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end; {error, not_found} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end. -spec get_trace_size() -> #{{node(), file:name_all()} => non_neg_integer()}. diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 92b35db99..162d07aaa 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -223,12 +223,12 @@ t_log_file(_Config) -> ]}, zip:table(Binary2) ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, - api_path("trace/test_client_id/download?node=unknonwn_node") + api_path("trace/test_client_id/download?node=unknown_node") ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, % known atom but unknown node @@ -303,12 +303,12 @@ t_stream_log(_Config) -> meck:expect(file, read, 2, {error, enomem}), {error, {_, 503, _}} = request_api(get, Path), meck:unload(file), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, - api_path("trace/test_stream_log/log?node=unknonwn_node") + api_path("trace/test_stream_log/log?node=unknown_node") ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, % known atom but not a node From 0a207856aa69bca11369764edc71fc52086d808f Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Feb 2023 10:23:16 +0100 Subject: [PATCH 095/135] chore: bump quicer 0.0.111 and emqtt 1.8.2 --- apps/emqx/rebar.config | 2 +- apps/emqx/rebar.config.script | 2 +- mix.exs | 4 ++-- rebar.config | 2 +- rebar.config.erl | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b79d14c54..a3ea4f2e7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index b2de8a7dd..2025f5ad5 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index ef2ed262f..b0608a48c 100644 --- a/mix.exs +++ b/mix.exs @@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.8.1", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.8.2", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, @@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.109", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.111", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index bc8362c01..0a163108f 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 3be4b70f6..8b36f907a 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 7502e570668e9efc9b6c9bcb1f0a715341ec42ae Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Feb 2023 10:41:59 +0100 Subject: [PATCH 096/135] chore: add changelog --- changes/ce/fix-10009.en.md | 1 + changes/ce/fix-10009.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/fix-10009.en.md create mode 100644 changes/ce/fix-10009.zh.md diff --git a/changes/ce/fix-10009.en.md b/changes/ce/fix-10009.en.md new file mode 100644 index 000000000..37f33a958 --- /dev/null +++ b/changes/ce/fix-10009.en.md @@ -0,0 +1 @@ +Validate `bytes` param to `GET /trace/:name/log` to not exceed signed 32bit integer. diff --git a/changes/ce/fix-10009.zh.md b/changes/ce/fix-10009.zh.md new file mode 100644 index 000000000..bb55ea5b9 --- /dev/null +++ b/changes/ce/fix-10009.zh.md @@ -0,0 +1 @@ +验证 `GET /trace/:name/log` 的 `bytes` 参数，使其不超过有符号的32位整数。 From 28382ec26bbba048fa79814272d1734aa76422af Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Feb 2023 13:11:34 +0100 Subject: [PATCH 097/135] test: add test for pubsub api --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 993969d48..4619905cb 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -124,7 +124,11 @@ t_lookup_client(_Config) -> [{Chan, Info, Stats}], emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN) ), - ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)). + ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)), + meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']), + ?assertMatch( + [_ | {error, nodedown}], emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN) + ). t_kickout_client(init, Config) -> process_flag(trap_exit, true), @@ -183,6 +187,15 @@ t_clean_cache(_Config) -> ?assertNotMatch( {error, _}, emqx_mgmt:clean_pem_cache_all() + ), + meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']), + ?assertMatch( + {error, [{'fake@nonode', {error, _}}]}, + emqx_mgmt:clean_authz_cache_all() + ), + ?assertMatch( + {error, [{'fake@nonode', {error, _}}]}, + emqx_mgmt:clean_pem_cache_all() ). t_set_client_props(init, Config) -> @@ -237,6 +250,64 @@ t_list_subscriptions_via_topic(Config) -> emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) ). +t_pubsub_api(init, Config) -> + setup_clients(Config); +t_pubsub_api('end', Config) -> + disconnect_clients(Config). + +-define(TT(Topic), {Topic, #{qos => 0}}). + +t_pubsub_api(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)), + ?assertMatch( + {subscribe, _, _}, + emqx_mgmt:subscribe(<<"client1">>, [?TT(<<"t/#">>), ?TT(<<"t1/#">>), ?TT(<<"t2/#">>)]) + ), + timer:sleep(100), + ?assertMatch( + [{{<<"t/#">>, _SubPid}, _Opts}], + emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) + ), + Message = emqx_message:make(?MODULE, 0, <<"t/foo">>, <<"helloworld">>, #{}, #{}), + emqx_mgmt:publish(Message), + Recv = + receive + {publish, #{client_pid := Client, payload := <<"helloworld">>}} -> + ok + after 100 -> + timeout + end, + ?assertEqual(ok, Recv), + ?assertEqual({error, channel_not_found}, emqx_mgmt:subscribe(<<"notfound">>, [?TT(<<"t/#">>)])), + ?assertNotMatch({error, _}, emqx_mgmt:unsubscribe(<<"client1">>, <<"t/#">>)), + ?assertEqual({error, channel_not_found}, emqx_mgmt:unsubscribe(<<"notfound">>, <<"t/#">>)), + Node = node(), + ?assertMatch( + {Node, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]}, + emqx_mgmt:list_client_subscriptions(<<"client1">>) + ), + ?assertMatch( + {unsubscribe, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]}, + emqx_mgmt:unsubscribe_batch(<<"client1">>, [<<"t1/#">>, <<"t2/#">>]) + ), + timer:sleep(100), + ?assertMatch([], emqx_mgmt:list_client_subscriptions(<<"client1">>)), + ?assertEqual( + {error, channel_not_found}, + emqx_mgmt:unsubscribe_batch(<<"notfound">>, [<<"t1/#">>, <<"t2/#">>]) + ). + +t_alarms(init, Config) -> + [ + emqx_mgmt:deactivate(Node, Name) + || {Node, ActiveAlarms} <- emqx_mgmt:get_alarms(activated), #{name := Name} <- ActiveAlarms + ], + emqx_mgmt:delete_all_deactivated_alarms(), + Config; +t_alarms('end', Config) -> + Config. + t_alarms(_) -> Node = node(), ?assertEqual( From 965d63f4f52a227ed231935fed70cb87485469cd Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Feb 2023 15:33:56 +0100 Subject: [PATCH 098/135] fix: schema for `/gateways/:name/clients` was missing top-level structure --- apps/emqx_gateway/src/emqx_gateway_api_clients.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index ef1c4c386..b30de3a3e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -19,7 +19,6 @@ -include("emqx_gateway_http.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(minirest_api). @@ -464,7 +463,12 @@ schema("/gateways/:name/clients") -> summary => <<"List Gateway's Clients">>, parameters => params_client_query(), responses => - ?STANDARD_RESP(#{200 => schema_client_list()}) + ?STANDARD_RESP(#{ + 200 => [ + {data, schema_client_list()}, + {meta, mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})} + ] + }) } }; schema("/gateways/:name/clients/:clientid") -> From a0589d5b95ab5d6749ad3d9037bfc9202dd0c2b6 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Feb 2023 16:23:38 +0100 Subject: [PATCH 099/135] fix: return `404` for unknown node names --- .../src/emqx_dashboard_monitor_api.erl | 45 ++++++++----------- .../test/emqx_dashboard_monitor_SUITE.erl | 6 +-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index f8b0918be..69f5bf34e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -55,7 +55,7 @@ schema("/monitor/nodes/:node") -> parameters => [parameter_node(), parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), - 400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>) + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>) } } }; @@ -79,7 +79,7 @@ schema("/monitor_current/nodes/:node") -> parameters => [parameter_node()], responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}), - 400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>) + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>) } } }. @@ -122,38 +122,31 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, all), - case emqx_misc:safe_to_existing_atom(RawNode, utf8) of - {ok, Node} -> - case emqx_dashboard_monitor:samplers(Node, Latest) of - {badrpc, {Node, Reason}} -> - Message = list_to_binary( - io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason]) - ), - {400, 'BAD_RPC', Message}; - Samplers -> - {200, Samplers} - end; - _ -> - Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])), - {400, 'BAD_RPC', Message} + with_node(RawNode, dashboard_samplers_fun(Latest)). + +dashboard_samplers_fun(Latest) -> + fun(NodeOrCluster) -> + case emqx_dashboard_monitor:samplers(NodeOrCluster, Latest) of + {badrpc, _} = Error -> Error; + Samplers -> {ok, Samplers} + end end. monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, all), + with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1). + +with_node(RawNode, Fun) -> case emqx_misc:safe_to_existing_atom(RawNode, utf8) of {ok, NodeOrCluster} -> - case emqx_dashboard_monitor:current_rate(NodeOrCluster) of - {ok, CurrentRate} -> - {200, CurrentRate}; + case Fun(NodeOrCluster) of {badrpc, {Node, Reason}} -> - Message = list_to_binary( - io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason]) - ), - {400, 'BAD_RPC', Message} + {404, 'NOT_FOUND', io_lib:format("Node not found: ~p (~p)", [Node, Reason])}; + {ok, Result} -> + {200, Result} end; - {error, _} -> - Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])), - {400, 'BAD_RPC', Message} + _Error -> + {404, 'NOT_FOUND', io_lib:format("Node not found: ~p", [RawNode])} end. %% ------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 74c6d9cc1..bfbd9b973 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -22,8 +22,6 @@ -import(emqx_dashboard_SUITE, [auth_header_/0]). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx.hrl"). -include("emqx_dashboard.hrl"). -define(SERVER, "http://127.0.0.1:18083"). @@ -114,9 +112,9 @@ t_monitor_reset(_) -> ok. t_monitor_api_error(_) -> - {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} = + {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} = request(["monitor", "nodes", 'emqx@127.0.0.2']), - {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} = + {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} = request(["monitor_current", "nodes", 'emqx@127.0.0.2']), {error, {400, #{<<"code">> := <<"BAD_REQUEST">>}}} = request(["monitor"], "latest=0"), From 7538a672b2e1b451ee5f24206389254962648106 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Feb 2023 09:34:12 +0100 Subject: [PATCH 100/135] refactor: move changelog formatter to scripts/rel sub dir --- scripts/{ => rel}/format-changelog.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{ => rel}/format-changelog.sh (100%) diff --git a/scripts/format-changelog.sh b/scripts/rel/format-changelog.sh similarity index 100% rename from scripts/format-changelog.sh rename to scripts/rel/format-changelog.sh From 1715b9a480f57de5cbf5295e282b38c1959a8032 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Feb 2023 10:33:04 +0100 Subject: [PATCH 101/135] refactor: make format-changelog.sh easier to run for alpha/beta releases --- .../ee/{feat-9932-en.md => feat-9932.en.md} | 0 .../ee/{feat-9932-zh.md => feat-9932.zh.md} | 0 scripts/changelog-lang-templates/en | 12 -- scripts/changelog-lang-templates/zh | 12 -- scripts/rel/cut.sh | 6 +- scripts/rel/format-changelog.sh | 154 +++++++++++++----- 6 files changed, 117 insertions(+), 67 deletions(-) rename changes/ee/{feat-9932-en.md => feat-9932.en.md} (100%) rename changes/ee/{feat-9932-zh.md => feat-9932.zh.md} (100%) delete mode 100644 scripts/changelog-lang-templates/en delete mode 100644 scripts/changelog-lang-templates/zh diff --git a/changes/ee/feat-9932-en.md b/changes/ee/feat-9932.en.md similarity index 100% rename from changes/ee/feat-9932-en.md rename to changes/ee/feat-9932.en.md diff --git a/changes/ee/feat-9932-zh.md b/changes/ee/feat-9932.zh.md similarity index 100% rename from changes/ee/feat-9932-zh.md rename to changes/ee/feat-9932.zh.md diff --git a/scripts/changelog-lang-templates/en b/scripts/changelog-lang-templates/en deleted file mode 100644 index 05c218c7e..000000000 --- a/scripts/changelog-lang-templates/en +++ /dev/null @@ -1,12 +0,0 @@ -# ${version} - -## Enhancements - -$(section feat) - -$(section perf) - -## Bug fixes - -$(section fix) - diff --git a/scripts/changelog-lang-templates/zh b/scripts/changelog-lang-templates/zh deleted file mode 100644 index 2bafd99d7..000000000 --- a/scripts/changelog-lang-templates/zh +++ /dev/null @@ -1,12 +0,0 @@ -# ${version} - -## 增强 - -$(section feat) - -$(section perf) - -## 修复 - -$(section fix) - diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 8d00694ac..60fe2f28d 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -223,9 +223,9 @@ generate_changelog () { from_tag="$(git describe --tags --abbrev=0 --match 'e*')" fi fi - local output_dir="changes" - ./scripts/format-changelog.sh $PROFILE "${from_tag}" "${TAG}" $output_dir - git add $output_dir + ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'en' -v "$TAG" > "changes/${TAG}.en.md" + ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" + git add changes/"${TAG}".*.md [ -n "$(git status -s)" ] && git commit -m "chore: Generate changelog for ${TAG}" } diff --git a/scripts/rel/format-changelog.sh b/scripts/rel/format-changelog.sh index 87fbc60a2..3561ad55c 100755 --- a/scripts/rel/format-changelog.sh +++ b/scripts/rel/format-changelog.sh @@ -3,28 +3,91 @@ set -euo pipefail shopt -s nullglob export LANG=C.UTF-8 -[ "$#" -ne 4 ] && { - echo "Usage $0 " 1>&2; - exit 1 +logerr() { + echo "$(tput setaf 1)ERROR: $1$(tput sgr0)" 1>&2 } -profile="${1}" -last_tag="${2}" -version="${3}" -output_dir="${4}" -languages=("en" "zh") +usage() { + cat < $output" - else - echo "Invalid language ${language}" 1>&2; - exit 1 - fi -} - changes_dir=("$top_dir/changes/ce") -if [ "$profile" == "emqx-enterprise" ]; then +if [ "$PROFILE" == "emqx-enterprise" ]; then changes_dir+=("$top_dir/changes/ee") fi while read -d "" -r file; do - changes+=("$file") -done < <(git diff --name-only -z -a "tags/${last_tag}...HEAD" "${changes_dir[@]}") + PRS+=("$file") +done < <(git diff --name-only -z -a "tags/${BASE_TAG}...HEAD" "${changes_dir[@]}") -for language in "${languages[@]}"; do - generate "$language" -done +TEMPLATE_FEAT_CHANGES="$(section 'feat')" +TEMPLATE_PERF_CHANGES="$(section 'perf')" +TEMPLATE_FIX_CHANGES="$(section 'fix')" + +case "$LANGUAGE" in + en) + TEMPLATE_ENH_HEADING="Enhancements" + TEMPLATE_FIX_HEADING="Bug Fixes" + ;; + zh) + TEMPLATE_ENH_HEADING="增强" + TEMPLATE_FIX_HEADING="修复" + ;; +esac + +cat < Date: Tue, 21 Feb 2023 11:38:34 +0100 Subject: [PATCH 102/135] feat(charts): add pod disruption budget to ee --- .../charts/emqx-enterprise/templates/pdb.yaml | 18 ++++++++++++++++++ .../emqx-enterprise/templates/service.yaml | 2 +- deploy/charts/emqx-enterprise/values.yaml | 9 ++++++++- deploy/charts/emqx/templates/service.yaml | 2 +- 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 deploy/charts/emqx-enterprise/templates/pdb.yaml diff --git a/deploy/charts/emqx-enterprise/templates/pdb.yaml b/deploy/charts/emqx-enterprise/templates/pdb.yaml new file mode 100644 index 000000000..a3f233064 --- /dev/null +++ b/deploy/charts/emqx-enterprise/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "emqx.fullname" . }}-pdb + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 0fe3dc411..233e69b10 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -121,7 +121,7 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index b3a77682f..9ae863219 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -102,7 +102,7 @@ initContainers: {} # sysctl -w net.netfilter.nf_conntrack_max=1000000 # sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30 -## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) +## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx-enterprise) emqxConfig: EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" EMQX_DASHBOARD__DEFAULT_USERNAME: "admin" @@ -229,3 +229,10 @@ ssl: issuer: name: letsencrypt-dns kind: ClusterIssuer + +## Setting PodDisruptionBudget. +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb +## +pdb: + enabled: false + maxUnavailable: 1 diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 5b6376a85..233e69b10 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -38,7 +38,7 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP From beb5a238e76eab44eea6a796534c54c1be6196c8 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 21 Feb 2023 11:41:06 +0100 Subject: [PATCH 103/135] chore(charts): add readme and a script to sync changes --- deploy/charts/README.md | 3 +++ deploy/charts/sync-enterprise.sh | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 deploy/charts/README.md create mode 100755 deploy/charts/sync-enterprise.sh diff --git a/deploy/charts/README.md b/deploy/charts/README.md new file mode 100644 index 000000000..4b8829056 --- /dev/null +++ b/deploy/charts/README.md @@ -0,0 +1,3 @@ +# Sync changes to emqx-enterprise + +When making changes in charts, please update `emqx` charts and run `./sync-enterprise.sh`. diff --git a/deploy/charts/sync-enterprise.sh b/deploy/charts/sync-enterprise.sh new file mode 100755 index 000000000..587871c0d --- /dev/null +++ b/deploy/charts/sync-enterprise.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +sed 's|emqx/emqx|emqx/emqx-enterprise|' < emqx/values.yaml > emqx-enterprise/values.yaml +cp emqx/templates/* emqx-enterprise/templates From 6fd11904ee534a75cea2ea27a1fdbcc061bd1c08 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 21 Feb 2023 11:43:25 +0100 Subject: [PATCH 104/135] chore: add changelog for #9213 --- changes/ce/feat-9213.en.md | 1 + changes/ce/feat-9213.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/feat-9213.en.md create mode 100644 changes/ce/feat-9213.zh.md diff --git a/changes/ce/feat-9213.en.md b/changes/ce/feat-9213.en.md new file mode 100644 index 000000000..3266ed836 --- /dev/null +++ b/changes/ce/feat-9213.en.md @@ -0,0 +1 @@ +Add pod disruption budget to helm chart diff --git a/changes/ce/feat-9213.zh.md b/changes/ce/feat-9213.zh.md new file mode 100644 index 000000000..509b1e01c --- /dev/null +++ b/changes/ce/feat-9213.zh.md @@ -0,0 +1 @@ +在舵手图中添加吊舱干扰预算。 From 1744b8bb7bf789eaa8a464bc9e9eaa69f85057f1 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 21 Feb 2023 11:46:33 +0100 Subject: [PATCH 105/135] chore: add changelog to ee --- changes/ee/feat-10011.en.md | 1 + changes/ee/feat-10011.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/feat-10011.en.md create mode 100644 changes/ee/feat-10011.zh.md diff --git a/changes/ee/feat-10011.en.md b/changes/ee/feat-10011.en.md new file mode 100644 index 000000000..3266ed836 --- /dev/null +++ b/changes/ee/feat-10011.en.md @@ -0,0 +1 @@ +Add pod disruption budget to helm chart diff --git a/changes/ee/feat-10011.zh.md b/changes/ee/feat-10011.zh.md new file mode 100644 index 000000000..509b1e01c --- /dev/null +++ b/changes/ee/feat-10011.zh.md @@ -0,0 +1 @@ +在舵手图中添加吊舱干扰预算。 From bb13d0708f387e05af18479ba88ab101befb2233 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 13:20:58 +0100 Subject: [PATCH 106/135] fix(bridge): fix dropped counter and inflight gauge Prior to this fix there were two metrics issues 1. if a batch is all requests expired when receiving a reply it only bumped 1 instead of the batch size for 'late_reply' 2. when a batch is partially delivered (or expired), the dropped requests were not decremented from the inflight size gauge --- .../emqx_resource/src/emqx_resource_buffer_worker.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index bb4eee57d..2f83a347a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -336,8 +336,8 @@ resume_from_blocked(Data) -> %% async, they will be appended to the end of inflight window again. retry_inflight_sync(Ref, Query, Data); {batch, Ref, NotExpired, Expired} -> - update_inflight_item(InflightTID, Ref, NotExpired), NumExpired = length(Expired), + update_inflight_item(InflightTID, Ref, NotExpired, NumExpired), emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), NumExpired > 0 andalso ?tp(buffer_worker_retry_expired, #{expired => Expired}), %% We retry msgs in inflight window sync, as if we send them @@ -1050,7 +1050,7 @@ handle_async_batch_reply( all_expired -> IsFullBefore = is_inflight_full(InflightTID), IsAcked = ack_inflight(InflightTID, Ref, Id, Index), - IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), + IsAcked andalso emqx_resource_metrics:late_reply_inc(Id, length(Batch)), IsFullBefore andalso ?MODULE:flush_worker(Pid), ?tp(handle_async_reply_expired, #{expired => Batch}), ok; @@ -1317,10 +1317,10 @@ ack_inflight(InflightTID, Ref, Id, Index) -> 1; [?INFLIGHT_ITEM(Ref, [?QUERY(_, _, _, _) | _] = Batch, _IsRetriable, _WorkerMRef)] -> length(Batch); - _ -> + [] -> 0 end, - IsAcked = Count > 0, + IsAcked = (Count > 0), IsAcked andalso ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), IsAcked. @@ -1341,8 +1341,9 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> ok. %% used to update a batch after dropping expired individual queries. -update_inflight_item(InflightTID, Ref, NewBatch) -> +update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), + _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -NumExpired, 0, 0}), ?tp(buffer_worker_worker_update_inflight_item, #{ref => Ref}), ok. From ada68e12e6abea05b30c3d5ab10ec84902cb5bfe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Feb 2023 18:55:24 +0100 Subject: [PATCH 107/135] fix(kafka): change default value for memory_overload_protection to false --- changes/ee/fix-10007.en.md | 5 +++++ changes/ee/fix-10007.zh.md | 3 +++ lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 3 +-- 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changes/ee/fix-10007.en.md create mode 100644 changes/ee/fix-10007.zh.md diff --git a/changes/ee/fix-10007.en.md b/changes/ee/fix-10007.en.md new file mode 100644 index 000000000..1adab8e9b --- /dev/null +++ b/changes/ee/fix-10007.en.md @@ -0,0 +1,5 @@ +Change Kafka bridge's config `memory_overload_protection` default value from `true` to `false`. +EMQX logs cases when messages get dropped due to overload protection, and this is also reflected in counters. +However, since there is by default no alerting based on the logs and counters, +setting it to `true` may cause messages being dropped without noticing. +At the time being, the better option is to let sysadmin set it explicitly so they are fully aware of the benefits and risks. diff --git a/changes/ee/fix-10007.zh.md b/changes/ee/fix-10007.zh.md new file mode 100644 index 000000000..0c08f20d0 --- /dev/null +++ b/changes/ee/fix-10007.zh.md @@ -0,0 +1,3 @@ +Kafka 桥接的配置参数 `memory_overload_protection` 默认值从 `true` 改成了 `false`。 +尽管内存过载后消息被丢弃会产生日志和计数，如果没有基于这些日志或计数的告警，系统管理员可能无法及时发现消息被丢弃。 +当前更好的选择是：让管理员显式的配置该项，迫使他们理解这个配置的好处以及风险。 diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index e72da2323..c41b95c3a 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -439,7 +439,7 @@ emqx_ee_bridge_kafka { } buffer_memory_overload_protection { desc { - en: "Applicable when buffer mode is set to memory or hybrid.\n" + en: "Applicable when buffer mode is set to memory\n" "EMQX will drop old buffered messages under high memory pressure. " "The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. " "NOTE: This config only works on Linux." diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 3983b235c..c345f6c74 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -238,8 +238,7 @@ fields(producer_buffer) -> )}, {memory_overload_protection, mk(boolean(), #{ - %% different from 4.x - default => true, + default => false, desc => ?DESC(buffer_memory_overload_protection) })} ]. From a6946d4f68b28491f0d0048fb28c48468826dff2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Feb 2023 09:02:35 +0100 Subject: [PATCH 108/135] chore: update v5 PR template --- .github/PULL_REQUEST_TEMPLATE/v5.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/v5.md b/.github/PULL_REQUEST_TEMPLATE/v5.md index d4104ca5d..7952c1371 100644 --- a/.github/PULL_REQUEST_TEMPLATE/v5.md +++ b/.github/PULL_REQUEST_TEMPLATE/v5.md @@ -5,7 +5,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] Added tests for the changes - [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes//(feat|fix)-.en.md` and `.zh.md` files +- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-.en.md` and `.zh.md` files - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible From 8d480a1dc9c2b7154ebde7db894c3b95845242de Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Feb 2023 20:29:30 +0100 Subject: [PATCH 109/135] fix(bin/emqx): fail fast if cookie is obviously wrong --- bin/emqx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index 9211bd338..4fdf24186 100755 --- a/bin/emqx +++ b/bin/emqx @@ -545,6 +545,10 @@ else logerr "Make sure environment variable EMQX_NODE__NAME is set to indicate for which node this command is intended." exit 1 fi + else + if [ -n "${EMQX_NODE__NAME:-}" ]; then + die "Node $EMQX_NODE__NAME is not running?" + fi fi ## We have no choiece but to read the bootstrap config (with environment overrides available in the current shell) [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1 @@ -940,9 +944,11 @@ if [ -n "${EMQX_NODE_COOKIE:-}" ]; then unset EMQX_NODE_COOKIE fi COOKIE="${EMQX_NODE__COOKIE:-}" -if [ -z "$COOKIE" ]; then - COOKIE="$(get_boot_config 'node.cookie')" +COOKIE_IN_USE="$(get_boot_config 'node.cookie')" +if [ -n "$COOKIE_IN_USE" ] && [ -n "$COOKIE" ] && [ "$COOKIE" != "$COOKIE_IN_USE" ]; then + die "EMQX_NODE__COOKIE is different from the cookie used by $NAME" fi +[ -z "$COOKIE" ] && COOKIE="$COOKIE_IN_USE" [ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE" maybe_warn_default_cookie() { From 6a1085a8420147ec2598f341b286e71193f95fd0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Feb 2023 20:35:33 +0100 Subject: [PATCH 110/135] chore: change 'emqx_ctl' usage to 'emqx ctl' --- apps/emqx_ctl/src/emqx_ctl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index a9aad0259..864b53d2a 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -149,7 +149,7 @@ help() -> [] -> print("No commands available.~n"); Cmds -> - print("Usage: ~ts~n", [?MODULE]), + print("Usage: ~ts~n", ["emqx ctl"]), lists:foreach( fun({_, {Mod, Cmd}, _}) -> print("~110..-s~n", [""]), From 4e70374e2a051df6bdaed546d63ce05a32cf2979 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 18:28:23 +0100 Subject: [PATCH 111/135] test: make docker-compose work --- scripts/ct/run.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index b44095624..ba6d1f91f 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -21,6 +21,12 @@ help() { echo " otherwise it runs the entire app's CT" } +if command -v docker-compose; then + DC='docker-compose' +else + DC='docker compose' +fi + WHICH_APP='novalue' CONSOLE='no' KEEP_UP='no' @@ -155,7 +161,7 @@ for dep in ${CT_DEPS}; do ;; tdengine) FILES+=( '.ci/docker-compose-file/docker-compose-tdengine-restful.yaml' ) - ;; + ;; *) echo "unknown_ct_dependency $dep" exit 1 @@ -201,7 +207,7 @@ if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' # shellcheck disable=2086 # no quotes for F_OPTIONS - docker compose $F_OPTIONS up -d --build --remove-orphans + $DC $F_OPTIONS up -d --build --remove-orphans fi echo "Fixing file owners and permissions for $UID_GID" @@ -218,7 +224,7 @@ set +e if [ "$STOP" = 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker compose $F_OPTIONS down --remove-orphans + $DC $F_OPTIONS down --remove-orphans elif [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash elif [ "$CONSOLE" = 'yes' ]; then @@ -235,11 +241,11 @@ else LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS - docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG" + $DC $F_OPTIONS logs --no-color --timestamps > "$LOG" fi if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker compose $F_OPTIONS down + $DC $F_OPTIONS down fi exit $RESULT fi From b0777ca73b79bb8e0421463424eed687aabd7881 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 22 Feb 2023 18:43:28 +0100 Subject: [PATCH 112/135] chore(ekka): Bump version --- apps/emqx/rebar.config | 2 +- changes/ce/fix-10021.en.md | 1 + changes/ce/fix-10021.zh.md | 1 + mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10021.en.md create mode 100644 changes/ce/fix-10021.zh.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a3ea4f2e7..e4a46743b 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.0"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.1"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/changes/ce/fix-10021.en.md b/changes/ce/fix-10021.en.md new file mode 100644 index 000000000..28302da70 --- /dev/null +++ b/changes/ce/fix-10021.en.md @@ -0,0 +1 @@ +Fix error message when the target node of `emqx_ctl cluster join` command is not running. diff --git a/changes/ce/fix-10021.zh.md b/changes/ce/fix-10021.zh.md new file mode 100644 index 000000000..6df64b76d --- /dev/null +++ b/changes/ce/fix-10021.zh.md @@ -0,0 +1 @@ +修正当`emqx_ctl cluster join`命令的目标节点未运行时的错误信息。 diff --git a/mix.exs b/mix.exs index b0608a48c..86bb504c0 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-7", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.14.0", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.14.1", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, diff --git a/rebar.config b/rebar.config index 0a163108f..9f25020b0 100644 --- a/rebar.config +++ b/rebar.config @@ -55,7 +55,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-7"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} From fc614e16e55d0f3e484ab0ad53a3c03932a9d797 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 20:07:35 +0100 Subject: [PATCH 113/135] fix(bridge): update inflight items after partial expiry --- .../src/emqx_resource_buffer_worker.erl | 98 ++++++++++++------- .../test/emqx_resource_SUITE.erl | 2 +- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 2f83a347a..6aa13092a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -335,11 +335,13 @@ resume_from_blocked(Data) -> %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. retry_inflight_sync(Ref, Query, Data); + {batch, Ref, NotExpired, []} -> + retry_inflight_sync(Ref, NotExpired, Data); {batch, Ref, NotExpired, Expired} -> NumExpired = length(Expired), - update_inflight_item(InflightTID, Ref, NotExpired, NumExpired), + ok = update_inflight_item(InflightTID, Ref, NotExpired, NumExpired), emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), - NumExpired > 0 andalso ?tp(buffer_worker_retry_expired, #{expired => Expired}), + ?tp(buffer_worker_retry_expired, #{expired => Expired}), %% We retry msgs in inflight window sync, as if we send them %% async, they will be appended to the end of inflight window again. retry_inflight_sync(Ref, NotExpired, Data) @@ -496,7 +498,7 @@ flush(Data0) -> {NotExpired, Expired} -> NumExpired = length(Expired), emqx_resource_metrics:dropped_expired_inc(Id, NumExpired), - IsBatch = BatchSize =/= 1, + IsBatch = (BatchSize > 1), %% We *must* use the new queue, because we currently can't %% `nack' a `pop'. %% Maybe we could re-open the queue? @@ -506,7 +508,6 @@ flush(Data0) -> ), Ref = make_request_ref(), do_flush(Data2, #{ - new_queue => Q1, is_batch => IsBatch, batch => NotExpired, ref => Ref, @@ -519,18 +520,16 @@ flush(Data0) -> is_batch := boolean(), batch := [queue_query()], ack_ref := replayq:ack_ref(), - ref := inflight_key(), - new_queue := replayq:q() + ref := inflight_key() }) -> gen_statem:event_handler_result(state(), data()). do_flush( - Data0, + #{queue := Q1} = Data0, #{ is_batch := false, batch := Batch, ref := Ref, - ack_ref := QAckRef, - new_queue := Q1 + ack_ref := QAckRef } ) -> #{ @@ -610,12 +609,11 @@ do_flush( end, {keep_state, Data1} end; -do_flush(Data0, #{ +do_flush(#{queue := Q1} = Data0, #{ is_batch := true, batch := Batch, ref := Ref, - ack_ref := QAckRef, - new_queue := Q1 + ack_ref := QAckRef }) -> #{ id := Id, @@ -715,17 +713,18 @@ batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> end, Batch ), - {ShouldAck, PostFns} = + {Action, PostFn1} = reply_caller_defer_metrics(Id, hd(Replies), QueryOpts), + PostFns = lists:foldl( - fun(Reply, {_ShouldAck, PostFns}) -> - {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), - {ShouldAck, [PostFn | PostFns]} + fun(Reply, PostFns) -> + {_, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), + [PostFn | PostFns] end, - {ack, []}, - Replies + [PostFn1], + tl(Replies) ), - PostFn = fun() -> lists:foreach(fun(F) -> F() end, PostFns) end, - {ShouldAck, PostFn}. + PostFn = fun() -> lists:foreach(fun(F) -> F() end, lists:reverse(PostFns)) end, + {Action, PostFn}. reply_caller(Id, Reply, QueryOpts) -> {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), @@ -1024,7 +1023,7 @@ do_handle_async_reply( case Action of nack -> %% Keep retrying. - mark_inflight_as_retriable(InflightTID, Ref), + ok = mark_inflight_as_retriable(InflightTID, Ref), ?MODULE:block(Pid); ack -> do_ack(InflightTID, Ref, Id, Index, PostFn, Pid, QueryOpts) @@ -1051,15 +1050,40 @@ handle_async_batch_reply( IsFullBefore = is_inflight_full(InflightTID), IsAcked = ack_inflight(InflightTID, Ref, Id, Index), IsAcked andalso emqx_resource_metrics:late_reply_inc(Id, length(Batch)), - IsFullBefore andalso ?MODULE:flush_worker(Pid), + IsFullBefore andalso IsAcked andalso ?MODULE:flush_worker(Pid), ?tp(handle_async_reply_expired, #{expired => Batch}), ok; - {NotExpired, Expired} -> - NumExpired = length(Expired), - emqx_resource_metrics:late_reply_inc(Id, NumExpired), - NumExpired > 0 andalso - ?tp(handle_async_reply_expired, #{expired => Expired}), - do_handle_async_batch_reply(ReplyContext#{batch := NotExpired}, Result) + {_NotExpired, []} -> + do_handle_async_batch_reply(ReplyContext, Result); + {_NotExpired, _Expired} -> + %% partial expire + %% the batch from reply context is minimized, so it cannot be used + %% to update the inflight items, hence discard Batch and lookup the RealBatch + ?tp(handle_async_reply_expired, #{expired => _Expired}), + case ets:lookup(InflightTID, Ref) of + [] -> + %% e.g. if the driver evaluates it more than once + %% which should really be a bug, TODO: add a unknown_reply counter + ok; + [?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef)] -> + %% All batch items share the same HasBeenSent flag + %% So we just take the original flag from the ReplyContext batch + %% and put it back to the batch found in inflight table + %% which must have already been set to `false` + [?QUERY(_ReplyTo, _, HasBeenSent, _ExpireAt) | _] = Batch, + {RealNotExpired0, RealExpired} = sieve_expired_requests(RealBatch, Now), + RealNotExpired = + lists:map( + fun(?QUERY(ReplyTo, CoreReq, _HasBeenSent, ExpireAt)) -> + ?QUERY(ReplyTo, CoreReq, HasBeenSent, ExpireAt) + end, + RealNotExpired0 + ), + NumExpired = length(RealExpired), + emqx_resource_metrics:late_reply_inc(Id, NumExpired), + ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), + do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) + end end. do_handle_async_batch_reply( @@ -1084,7 +1108,7 @@ do_handle_async_batch_reply( case Action of nack -> %% Keep retrying. - mark_inflight_as_retriable(InflightTID, Ref), + ok = mark_inflight_as_retriable(InflightTID, Ref), ?MODULE:block(Pid); ack -> do_ack(InflightTID, Ref, Id, Index, PostFn, Pid, QueryOpts) @@ -1320,10 +1344,15 @@ ack_inflight(InflightTID, Ref, Id, Index) -> [] -> 0 end, - IsAcked = (Count > 0), - IsAcked andalso ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), - emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), - IsAcked. + IsKnownRef = (Count > 0), + case IsKnownRef of + true -> + ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), + emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)); + false -> + ok + end, + IsKnownRef. mark_inflight_items_as_retriable(Data, WorkerMRef) -> #{inflight_tid := InflightTID} = Data, @@ -1341,10 +1370,9 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> ok. %% used to update a batch after dropping expired individual queries. -update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> +update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) when NumExpired > 0 -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -NumExpired, 0, 0}), - ?tp(buffer_worker_worker_update_inflight_item, #{ref => Ref}), ok. %%============================================================================== diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 984b3b04a..92f069739 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1997,6 +1997,7 @@ do_t_expiration_async_after_reply(IsBatch) -> {ok, _} = ?block_until( #{?snk_kind := handle_async_reply_expired}, 10 * TimeoutMS ), + wait_telemetry_event(success, #{n_events => 1, timeout => 4_000}), unlink(Pid0), exit(Pid0, kill), @@ -2011,7 +2012,6 @@ do_t_expiration_async_after_reply(IsBatch) -> ], ?of_kind(handle_async_reply_expired, Trace) ), - wait_telemetry_event(success, #{n_events => 1, timeout => 4_000}), Metrics = tap_metrics(?LINE), ?assertMatch( #{ From 2811c371adda857633837666e7ec6d2be8518012 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 22:18:44 +0100 Subject: [PATCH 114/135] docs: add changelogs --- changes/ce/fix-10020.en.md | 1 + changes/ce/fix-10020.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/fix-10020.en.md create mode 100644 changes/ce/fix-10020.zh.md diff --git a/changes/ce/fix-10020.en.md b/changes/ce/fix-10020.en.md new file mode 100644 index 000000000..73615804b --- /dev/null +++ b/changes/ce/fix-10020.en.md @@ -0,0 +1 @@ +Fix bridge metrics when running in async mode with batching enabled (`batch_size` > 1). diff --git a/changes/ce/fix-10020.zh.md b/changes/ce/fix-10020.zh.md new file mode 100644 index 000000000..2fce853e3 --- /dev/null +++ b/changes/ce/fix-10020.zh.md @@ -0,0 +1 @@ +修复使用异步和批量配置的桥接计数不准确的问题。 From bf8becd52197592fe36d3b7fbcca5dc45f378466 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 23:07:12 +0100 Subject: [PATCH 115/135] test: make sure gauge return to 0 in test cases --- apps/emqx_resource/test/emqx_resource_SUITE.erl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 92f069739..82c95cb99 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2102,6 +2102,10 @@ t_expiration_batch_all_expired_after_reply(_Config) -> late_reply := 1, retried := 0, failed := 0 + }, + gauges := #{ + inflight := 0, + queuing := 0 } }, Metrics @@ -2217,6 +2221,16 @@ do_t_expiration_retry(IsBatch) -> [#{expired := [{query, _, {inc_counter, 1}, _, _}]}], ?of_kind(buffer_worker_retry_expired, Trace) ), + Metrics = tap_metrics(?LINE), + ?assertMatch( + #{ + gauges := #{ + inflight := 0, + queuing := 0 + } + }, + Metrics + ), ok end ), From 036f69cd6e94df535dc0b217ee81d2657fd4aa3f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 23:26:04 +0100 Subject: [PATCH 116/135] test: ensure batch size > 1 is covered in expiration test --- .../test/emqx_resource_SUITE.erl | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 82c95cb99..bc146cc8e 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1944,7 +1944,7 @@ t_expiration_async_batch_after_reply(_Config) -> #{name => test_resource}, #{ query_mode => async, - batch_size => 2, + batch_size => 3, batch_time => 100, worker_pool_size => 1, resume_interval => 2_000 @@ -1959,7 +1959,7 @@ do_t_expiration_async_after_reply(IsBatch) -> NAcks = case IsBatch of batch -> 1; - single -> 2 + single -> 3 end, ?force_ordering( #{?snk_kind := buffer_worker_flush_ack}, @@ -1980,6 +1980,10 @@ do_t_expiration_async_after_reply(IsBatch) -> ok, emqx_resource:query(?ID, {inc_counter, 199}, #{timeout => TimeoutMS}) ), + ?assertEqual( + ok, + emqx_resource:query(?ID, {inc_counter, 299}, #{timeout => TimeoutMS}) + ), ?assertEqual( ok, emqx_resource:query(?ID, {inc_counter, 99}, #{timeout => infinity}) ), @@ -2004,23 +2008,37 @@ do_t_expiration_async_after_reply(IsBatch) -> ok end, fun(Trace) -> - ?assertMatch( - [ - #{ - expired := [{query, _, {inc_counter, 199}, _, _}] - } - ], - ?of_kind(handle_async_reply_expired, Trace) - ), + case IsBatch of + batch -> + ?assertMatch( + [ + #{ + expired := [ + {query, _, {inc_counter, 199}, _, _}, + {query, _, {inc_counter, 299}, _, _} + ] + } + ], + ?of_kind(handle_async_reply_expired, Trace) + ); + single -> + ?assertMatch( + [ + #{expired := [{query, _, {inc_counter, 199}, _, _}]}, + #{expired := [{query, _, {inc_counter, 299}, _, _}]} + ], + ?of_kind(handle_async_reply_expired, Trace) + ) + end, Metrics = tap_metrics(?LINE), ?assertMatch( #{ counters := #{ - matched := 2, + matched := 3, %% the request with infinity timeout. success := 1, dropped := 0, - late_reply := 1, + late_reply := 2, retried := 0, failed := 0 } @@ -2042,7 +2060,7 @@ t_expiration_batch_all_expired_after_reply(_Config) -> #{name => test_resource}, #{ query_mode => async, - batch_size => 2, + batch_size => 3, batch_time => 100, worker_pool_size => 1, resume_interval => ResumeInterval @@ -2067,6 +2085,10 @@ t_expiration_batch_all_expired_after_reply(_Config) -> ok, emqx_resource:query(?ID, {inc_counter, 199}, #{timeout => TimeoutMS}) ), + ?assertEqual( + ok, + emqx_resource:query(?ID, {inc_counter, 299}, #{timeout => TimeoutMS}) + ), Pid0 = spawn_link(fun() -> ?tp(delay_enter, #{}), @@ -2087,7 +2109,10 @@ t_expiration_batch_all_expired_after_reply(_Config) -> ?assertMatch( [ #{ - expired := [{query, _, {inc_counter, 199}, _, _}] + expired := [ + {query, _, {inc_counter, 199}, _, _}, + {query, _, {inc_counter, 299}, _, _} + ] } ], ?of_kind(handle_async_reply_expired, Trace) @@ -2096,10 +2121,10 @@ t_expiration_batch_all_expired_after_reply(_Config) -> ?assertMatch( #{ counters := #{ - matched := 1, + matched := 2, success := 0, dropped := 0, - late_reply := 1, + late_reply := 2, retried := 0, failed := 0 }, From 713220f88b8a0bbc5dec89eb54fd9e5b9415d311 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 00:04:20 +0100 Subject: [PATCH 117/135] refactor(buffer_worker): more generic process for all_expired --- .../src/emqx_resource_buffer_worker.erl | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 6aa13092a..ff3b67c7a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -489,7 +489,7 @@ flush(Data0) -> %% if the request has expired, the caller is no longer %% waiting for a response. case sieve_expired_requests(Batch, Now) of - all_expired -> + {[], _AllExpired} -> ok = replayq:ack(Q1, QAckRef), emqx_resource_metrics:dropped_expired_inc(Id, length(Batch)), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), @@ -1031,9 +1031,6 @@ do_handle_async_reply( handle_async_batch_reply( #{ - buffer_worker := Pid, - resource_id := Id, - worker_index := Index, inflight_tid := InflightTID, request_ref := Ref, batch := Batch @@ -1046,44 +1043,59 @@ handle_async_batch_reply( ), Now = now_(), case sieve_expired_requests(Batch, Now) of - all_expired -> - IsFullBefore = is_inflight_full(InflightTID), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index), - IsAcked andalso emqx_resource_metrics:late_reply_inc(Id, length(Batch)), - IsFullBefore andalso IsAcked andalso ?MODULE:flush_worker(Pid), - ?tp(handle_async_reply_expired, #{expired => Batch}), - ok; {_NotExpired, []} -> + %% this is the critical code path, + %% we try not to do ets:lookup in this case + %% because the batch can be quite big do_handle_async_batch_reply(ReplyContext, Result); {_NotExpired, _Expired} -> - %% partial expire + %% at least one is expired %% the batch from reply context is minimized, so it cannot be used %% to update the inflight items, hence discard Batch and lookup the RealBatch ?tp(handle_async_reply_expired, #{expired => _Expired}), - case ets:lookup(InflightTID, Ref) of - [] -> - %% e.g. if the driver evaluates it more than once - %% which should really be a bug, TODO: add a unknown_reply counter - ok; - [?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef)] -> - %% All batch items share the same HasBeenSent flag - %% So we just take the original flag from the ReplyContext batch - %% and put it back to the batch found in inflight table - %% which must have already been set to `false` - [?QUERY(_ReplyTo, _, HasBeenSent, _ExpireAt) | _] = Batch, - {RealNotExpired0, RealExpired} = sieve_expired_requests(RealBatch, Now), - RealNotExpired = - lists:map( - fun(?QUERY(ReplyTo, CoreReq, _HasBeenSent, ExpireAt)) -> - ?QUERY(ReplyTo, CoreReq, HasBeenSent, ExpireAt) - end, - RealNotExpired0 - ), - NumExpired = length(RealExpired), - emqx_resource_metrics:late_reply_inc(Id, NumExpired), - ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), - do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) - end + handle_async_batch_reply2(ets:lookup(InflightTID, Ref), ReplyContext, Result, Now) + end. + +handle_async_batch_reply2([], _, _, _) -> + %% e.g. if the driver evaluates the callback more than once + %% which should really be a bug + ok; +handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> + ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, + #{ + buffer_worker := Pid, + resource_id := Id, + worker_index := Index, + inflight_tid := InflightTID, + request_ref := Ref, + batch := Batch + } = ReplyContext, + %% All batch items share the same HasBeenSent flag + %% So we just take the original flag from the ReplyContext batch + %% and put it back to the batch found in inflight table + %% which must have already been set to `false` + [?QUERY(_ReplyTo, _, HasBeenSent, _ExpireAt) | _] = Batch, + {RealNotExpired0, RealExpired} = sieve_expired_requests(RealBatch, Now), + RealNotExpired = + lists:map( + fun(?QUERY(ReplyTo, CoreReq, _HasBeenSent, ExpireAt)) -> + ?QUERY(ReplyTo, CoreReq, HasBeenSent, ExpireAt) + end, + RealNotExpired0 + ), + NumExpired = length(RealExpired), + emqx_resource_metrics:late_reply_inc(Id, NumExpired), + case RealNotExpired of + [] -> + %% all expired, no need to update back the inflight batch + IsFullBefore = is_inflight_full(InflightTID), + IsAcked = ack_inflight(InflightTID, Ref, Id, Index), + IsFullBefore andalso IsAcked andalso ?MODULE:flush_worker(Pid); + _ -> + %% some queries are not expired, put them back to the inflight batch + %% so it can be either acked now or retried later + ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), + do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) end. do_handle_async_batch_reply( @@ -1226,10 +1238,8 @@ inflight_get_first_retriable(InflightTID, Now) -> {single, Ref, Query} end; {[{Ref, Batch = [_ | _]}], _Continuation} -> - %% batch is non-empty because we check that in - %% `sieve_expired_requests'. case sieve_expired_requests(Batch, Now) of - all_expired -> + {[], _AllExpired} -> {expired, Ref, Batch}; {NotExpired, Expired} -> {batch, Ref, NotExpired, Expired} @@ -1482,22 +1492,12 @@ is_async_return(_) -> false. sieve_expired_requests(Batch, Now) -> - {Expired, NotExpired} = - lists:partition( - fun(?QUERY(_ReplyTo, _CoreReq, _HasBeenSent, ExpireAt)) -> - is_expired(ExpireAt, Now) - end, - Batch - ), - case {NotExpired, Expired} of - {[], []} -> - %% Should be impossible for batch_size >= 1. - all_expired; - {[], [_ | _]} -> - all_expired; - {[_ | _], _} -> - {NotExpired, Expired} - end. + lists:partition( + fun(?QUERY(_ReplyTo, _CoreReq, _HasBeenSent, ExpireAt)) -> + not is_expired(ExpireAt, Now) + end, + Batch + ). -spec is_expired(infinity | integer(), integer()) -> boolean(). is_expired(infinity = _ExpireAt, _Now) -> From 3413af76be21a9be2dab6a0a6b07a01fbd681776 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 09:05:52 +0100 Subject: [PATCH 118/135] fix(emqx_misc): ensure flatten list for safe dir --- apps/emqx/src/emqx_misc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index fbeec8724..18ecc644a 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -720,4 +720,4 @@ pub_props_to_packet(Properties) -> safe_filename(Filename) when is_binary(Filename) -> binary:replace(Filename, <<":">>, <<"-">>, [global]); safe_filename(Filename) when is_list(Filename) -> - string:replace(Filename, ":", "-", all). + lists:flatten(string:replace(Filename, ":", "-", all)). From 356a94af30ec168f8c2a2d6a7360e7b446e14e26 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 09:47:34 +0100 Subject: [PATCH 119/135] fix(buffer_worker): ensure async flush message is sent This is a new issue introduced in the previous fix commits after handling the partial expiry correctly, the IsFullBefore check is no longer the state before the reply is received but the state after a partially-expired batch is shrinked. The fix is simple, move the check to the entry-point of where async reply callback enters, then send an async 'flush' notification regardless of the handling result. --- .../src/emqx_resource_buffer_worker.erl | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index ff3b67c7a..0a6adf3d6 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1019,15 +1019,16 @@ do_handle_async_reply( ref => Ref, result => Result }), - + IsFullBefore = is_inflight_full(InflightTID), case Action of nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), ?MODULE:block(Pid); ack -> - do_ack(InflightTID, Ref, Id, Index, PostFn, Pid, QueryOpts) - end. + do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) + end, + ok = maybe_flush_after_async_reply(IsFullBefore). handle_async_batch_reply( #{ @@ -1042,19 +1043,21 @@ handle_async_batch_reply( #{batch_or_query => Batch, ref => Ref} ), Now = now_(), + IsFullBefore = is_inflight_full(InflightTID), case sieve_expired_requests(Batch, Now) of {_NotExpired, []} -> %% this is the critical code path, %% we try not to do ets:lookup in this case %% because the batch can be quite big - do_handle_async_batch_reply(ReplyContext, Result); + ok = do_handle_async_batch_reply(ReplyContext, Result); {_NotExpired, _Expired} -> %% at least one is expired %% the batch from reply context is minimized, so it cannot be used %% to update the inflight items, hence discard Batch and lookup the RealBatch ?tp(handle_async_reply_expired, #{expired => _Expired}), - handle_async_batch_reply2(ets:lookup(InflightTID, Ref), ReplyContext, Result, Now) - end. + ok = handle_async_batch_reply2(ets:lookup(InflightTID, Ref), ReplyContext, Result, Now) + end, + ok = maybe_flush_after_async_reply(IsFullBefore). handle_async_batch_reply2([], _, _, _) -> %% e.g. if the driver evaluates the callback more than once @@ -1063,7 +1066,6 @@ handle_async_batch_reply2([], _, _, _) -> handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, #{ - buffer_worker := Pid, resource_id := Id, worker_index := Index, inflight_tid := InflightTID, @@ -1088,15 +1090,15 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> case RealNotExpired of [] -> %% all expired, no need to update back the inflight batch - IsFullBefore = is_inflight_full(InflightTID), - IsAcked = ack_inflight(InflightTID, Ref, Id, Index), - IsFullBefore andalso IsAcked andalso ?MODULE:flush_worker(Pid); + _ = ack_inflight(InflightTID, Ref, Id, Index), + ok; _ -> %% some queries are not expired, put them back to the inflight batch %% so it can be either acked now or retried later ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), - do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) - end. + ok = do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) + end, + ok. do_handle_async_batch_reply( #{ @@ -1123,11 +1125,10 @@ do_handle_async_batch_reply( ok = mark_inflight_as_retriable(InflightTID, Ref), ?MODULE:block(Pid); ack -> - do_ack(InflightTID, Ref, Id, Index, PostFn, Pid, QueryOpts) + ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) end. -do_ack(InflightTID, Ref, Id, Index, PostFn, WorkerPid, QueryOpts) -> - IsFullBefore = is_inflight_full(InflightTID), +do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) -> IsKnownRef = ack_inflight(InflightTID, Ref, Id, Index), case maps:get(simple_query, QueryOpts, false) of true -> @@ -1137,9 +1138,18 @@ do_ack(InflightTID, Ref, Id, Index, PostFn, WorkerPid, QueryOpts) -> false -> ok end, - IsFullBefore andalso ?MODULE:flush_worker(WorkerPid), ok. +maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = false) -> + %% inflight was not full before async reply is handled, + %% after it is handled, the inflight table must be even smaller + %% hance we can rely on the buffer worker's flush timer to trigger + %% the next flush + ok; +maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> + %% the inflight table was full before handling aync reply + ok = ?MODULE:flush_worker(self()). + %%============================================================================== %% operations for queue queue_item_marshaller(Bin) when is_binary(Bin) -> From dbfdeec5e95613fca6f8e4f0e4a3c1bbd32951d4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 12:40:44 +0100 Subject: [PATCH 120/135] fix(buffer_worker): log unknown async replies --- .../src/emqx_resource_buffer_worker.erl | 100 ++++++++++++++---- .../test/emqx_connector_demo.erl | 57 ++++++++-- .../test/emqx_resource_SUITE.erl | 64 ++++++++++- 3 files changed, 192 insertions(+), 29 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0a6adf3d6..77494f4ba 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -475,6 +475,7 @@ flush(Data0) -> ?tp(buffer_worker_flush, #{queue_count => CurrentCount, is_full => IsFull}), case {CurrentCount, IsFull} of {0, _} -> + ?tp(buffer_worker_queue_drained, #{inflight => inflight_num_batches(InflightTID)}), {keep_state, Data1}; {_, true} -> ?tp(buffer_worker_flush_but_inflight_full, #{}), @@ -918,7 +919,7 @@ apply_query_fun(async, Mod, Id, Index, Ref, ?QUERY(_, Request, _, _) = Query, Re inflight_tid => InflightTID, request_ref => Ref, query_opts => QueryOpts, - query => minimize(Query) + min_query => minimize(Query) }, IsRetriable = false, WorkerMRef = undefined, @@ -951,7 +952,7 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re inflight_tid => InflightTID, request_ref => Ref, query_opts => QueryOpts, - batch => minimize(Batch) + min_batch => minimize(Batch) }, Requests = lists:map( fun(?QUERY(_ReplyTo, Request, _, _ExpireAt)) -> Request end, Batch @@ -967,19 +968,33 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re ). handle_async_reply( + #{ + request_ref := Ref, + inflight_tid := InflightTID + } = ReplyContext, + Result +) -> + case maybe_handle_unknown_async_reply(InflightTID, Ref) of + discard -> + ok; + continue -> + handle_async_reply1(ReplyContext, Result) + end. + +handle_async_reply1( #{ request_ref := Ref, inflight_tid := InflightTID, resource_id := Id, worker_index := Index, buffer_worker := Pid, - query := ?QUERY(_, _, _, ExpireAt) = _Query + min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, Result ) -> ?tp( handle_async_reply_enter, - #{batch_or_query => [_Query], ref => Ref} + #{batch_or_query => [_Query], ref => Ref, result => Result} ), Now = now_(), case is_expired(ExpireAt, Now) of @@ -1002,7 +1017,7 @@ do_handle_async_reply( worker_index := Index, buffer_worker := Pid, inflight_tid := InflightTID, - query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query + min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt) = _Query }, Result ) -> @@ -1031,16 +1046,30 @@ do_handle_async_reply( ok = maybe_flush_after_async_reply(IsFullBefore). handle_async_batch_reply( + #{ + inflight_tid := InflightTID, + request_ref := Ref + } = ReplyContext, + Result +) -> + case maybe_handle_unknown_async_reply(InflightTID, Ref) of + discard -> + ok; + continue -> + handle_async_batch_reply1(ReplyContext, Result) + end. + +handle_async_batch_reply1( #{ inflight_tid := InflightTID, request_ref := Ref, - batch := Batch + min_batch := Batch } = ReplyContext, Result ) -> ?tp( handle_async_reply_enter, - #{batch_or_query => Batch, ref => Ref} + #{batch_or_query => Batch, ref => Ref, result => Result} ), Now = now_(), IsFullBefore = is_inflight_full(InflightTID), @@ -1060,8 +1089,7 @@ handle_async_batch_reply( ok = maybe_flush_after_async_reply(IsFullBefore). handle_async_batch_reply2([], _, _, _) -> - %% e.g. if the driver evaluates the callback more than once - %% which should really be a bug + %% should have caused the unknown_async_reply_discarded ok; handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, @@ -1070,7 +1098,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> worker_index := Index, inflight_tid := InflightTID, request_ref := Ref, - batch := Batch + min_batch := Batch } = ReplyContext, %% All batch items share the same HasBeenSent flag %% So we just take the original flag from the ReplyContext batch @@ -1096,7 +1124,7 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> %% some queries are not expired, put them back to the inflight batch %% so it can be either acked now or retried later ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), - ok = do_handle_async_batch_reply(ReplyContext#{batch := RealNotExpired}, Result) + ok = do_handle_async_batch_reply(ReplyContext#{min_batch := RealNotExpired}, Result) end, ok. @@ -1107,7 +1135,7 @@ do_handle_async_batch_reply( worker_index := Index, inflight_tid := InflightTID, request_ref := Ref, - batch := Batch, + min_batch := Batch, query_opts := QueryOpts }, Result @@ -1123,7 +1151,7 @@ do_handle_async_batch_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ?MODULE:block(Pid); + ok = ?MODULE:block(Pid); ack -> ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) end. @@ -1150,6 +1178,32 @@ maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> %% the inflight table was full before handling aync reply ok = ?MODULE:flush_worker(self()). +%% check if the async reply is valid. +%% e.g. if a connector evaluates the callback more than once: +%% 1. If the request was previously deleted from inflight table due to +%% either succeeded previously or expired, this function logs a +%% warning message and returns 'discard' instruction. +%% 2. If the request was previously failed and now pending on a retry, +%% then this function will return 'continue' as there is no way to +%% tell if this reply is stae or not. +maybe_handle_unknown_async_reply(InflightTID, Ref) -> + try ets:member(InflightTID, Ref) of + true -> + %% NOTE: this does not mean the + continue; + false -> + ?tp( + warning, + unknown_async_reply_discarded, + #{inflight_key => Ref} + ), + discard + catch + error:badarg -> + %% shutdown ? + discard + end. + %%============================================================================== %% operations for queue queue_item_marshaller(Bin) when is_binary(Bin) -> @@ -1287,7 +1341,7 @@ inflight_append( InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, WorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), BatchSize = length(Batch), - IsNew andalso ets:update_counter(InflightTID, ?SIZE_REF, {2, BatchSize}), + IsNew andalso inc_inflight(InflightTID, BatchSize), emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; @@ -1302,7 +1356,7 @@ inflight_append( Query = mark_as_sent(Query0), InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, WorkerMRef), IsNew = ets:insert_new(InflightTID, InflightItem), - IsNew andalso ets:update_counter(InflightTID, ?SIZE_REF, {2, 1}), + IsNew andalso inc_inflight(InflightTID, 1), emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)), ?tp(buffer_worker_appended_to_inflight, #{item => InflightItem, is_new => IsNew}), ok; @@ -1318,6 +1372,8 @@ mark_inflight_as_retriable(undefined, _Ref) -> ok; mark_inflight_as_retriable(InflightTID, Ref) -> _ = ets:update_element(InflightTID, Ref, {?RETRY_IDX, true}), + %% the old worker's DOWN should not affect this inflight any more + _ = ets:update_element(InflightTID, Ref, {?WORKER_MREF_IDX, erased}), ok. %% Track each worker pid only once. @@ -1367,7 +1423,7 @@ ack_inflight(InflightTID, Ref, Id, Index) -> IsKnownRef = (Count > 0), case IsKnownRef of true -> - ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), + ok = dec_inflight(InflightTID, Count), emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)); false -> ok @@ -1390,9 +1446,17 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> ok. %% used to update a batch after dropping expired individual queries. -update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) when NumExpired > 0 -> +update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), - _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -NumExpired, 0, 0}), + ok = dec_inflight(InflightTID, NumExpired), + ok. + +inc_inflight(InflightTID, Count) -> + _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, Count}), + ok. + +dec_inflight(InflightTID, Count) when Count > 0 -> + _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), ok. %%============================================================================== diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 1d96fa083..a6b7b2339 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -135,11 +135,11 @@ on_query(_InstId, get_counter, #{pid := Pid}) -> after 1000 -> {error, timeout} end; -on_query(_InstId, {sleep, For}, #{pid := Pid}) -> +on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) -> ?tp(connector_demo_sleep, #{mode => sync, for => For}), ReqRef = make_ref(), From = {self(), ReqRef}, - Pid ! {From, {sleep, For}}, + Pid ! {From, {sleep_before_reply, For}}, receive {ReqRef, Result} -> Result @@ -159,9 +159,9 @@ on_query_async(_InstId, block_now, ReplyFun, #{pid := Pid}) -> on_query_async(_InstId, {big_payload, Payload}, ReplyFun, #{pid := Pid}) -> Pid ! {big_payload, Payload, ReplyFun}, {ok, Pid}; -on_query_async(_InstId, {sleep, For}, ReplyFun, #{pid := Pid}) -> +on_query_async(_InstId, {sleep_before_reply, For}, ReplyFun, #{pid := Pid}) -> ?tp(connector_demo_sleep, #{mode => async, for => For}), - Pid ! {{sleep, For}, ReplyFun}, + Pid ! {{sleep_before_reply, For}, ReplyFun}, {ok, Pid}. on_batch_query(InstId, BatchReq, State) -> @@ -173,10 +173,13 @@ on_batch_query(InstId, BatchReq, State) -> get_counter -> batch_get_counter(sync, InstId, State); {big_payload, _Payload} -> - batch_big_payload(sync, InstId, BatchReq, State) + batch_big_payload(sync, InstId, BatchReq, State); + {random_reply, Num} -> + %% async batch retried + random_reply(Num) end. -on_batch_query_async(InstId, BatchReq, ReplyFunAndArgs, State) -> +on_batch_query_async(InstId, BatchReq, ReplyFunAndArgs, #{pid := Pid} = State) -> %% Requests can be of multiple types, but cannot be mixed. case hd(BatchReq) of {inc_counter, _} -> @@ -186,7 +189,11 @@ on_batch_query_async(InstId, BatchReq, ReplyFunAndArgs, State) -> block_now -> on_query_async(InstId, block_now, ReplyFunAndArgs, State); {big_payload, _Payload} -> - batch_big_payload({async, ReplyFunAndArgs}, InstId, BatchReq, State) + batch_big_payload({async, ReplyFunAndArgs}, InstId, BatchReq, State); + {random_reply, Num} -> + %% only take the first Num in the batch should be random enough + Pid ! {{random_reply, Num}, ReplyFunAndArgs}, + {ok, Pid} end. batch_inc_counter(CallMode, InstId, BatchReq, State) -> @@ -299,16 +306,31 @@ counter_loop( {{FromPid, ReqRef}, get} -> FromPid ! {ReqRef, Num}, State; - {{sleep, _} = SleepQ, ReplyFun} -> + {{random_reply, RandNum}, ReplyFun} -> + %% usually a behaving connector should reply once and only once for + %% each (batch) request + %% but we try to reply random results a random number of times + %% with 'ok' in the result, the buffer worker should eventually + %% drain the buffer (and inflights table) + ReplyCount = 1 + (RandNum rem 3), + Results = random_replies(ReplyCount), + lists:foreach( + fun(Result) -> + apply_reply(ReplyFun, Result) + end, + Results + ), + State; + {{sleep_before_reply, _} = SleepQ, ReplyFun} -> apply_reply(ReplyFun, handle_query(async, SleepQ, Status)), State; - {{FromPid, ReqRef}, {sleep, _} = SleepQ} -> + {{FromPid, ReqRef}, {sleep_before_reply, _} = SleepQ} -> FromPid ! {ReqRef, handle_query(sync, SleepQ, Status)}, State end, counter_loop(NewState). -handle_query(Mode, {sleep, For} = Query, Status) -> +handle_query(Mode, {sleep_before_reply, For} = Query, Status) -> ok = timer:sleep(For), Result = case Status of @@ -329,3 +351,18 @@ maybe_register(_Name, _Pid, false) -> apply_reply({ReplyFun, Args}, Result) when is_function(ReplyFun) -> apply(ReplyFun, Args ++ [Result]). + +random_replies(0) -> + []; +random_replies(N) -> + [random_reply(N) | random_replies(N - 1)]. + +random_reply(N) -> + case rand:uniform(3) of + 1 -> + {ok, N}; + 2 -> + {error, {recoverable_error, N}}; + 3 -> + {error, {unrecoverable_error, N}} + end. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index bc146cc8e..1362cd1cc 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1482,7 +1482,7 @@ t_retry_async_inflight_full(_Config) -> AsyncInflightWindow * 2, fun() -> For = (ResumeInterval div 4) + rand:uniform(ResumeInterval div 4), - {sleep, For} + {sleep_before_reply, For} end, #{async_reply_fun => {fun(Res) -> ct:pal("Res = ~p", [Res]) end, []}} ), @@ -1507,6 +1507,68 @@ t_retry_async_inflight_full(_Config) -> ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), ok. +%% this test case is to ensure the buffer worker will not go crazy even +%% if the underlying connector is misbehaving: evaluate async callbacks multiple times +t_async_reply_multi_eval(_Config) -> + ResumeInterval = 20, + AsyncInflightWindow = 5, + emqx_connector_demo:set_callback_mode(async_if_possible), + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => ?FUNCTION_NAME}, + #{ + query_mode => async, + async_inflight_window => AsyncInflightWindow, + batch_size => 3, + batch_time => 10, + worker_pool_size => 1, + resume_interval => ResumeInterval + } + ), + ?check_trace( + #{timetrap => 15_000}, + begin + %% block + ok = emqx_resource:simple_sync_query(?ID, block), + + {ok, {ok, _}} = + ?wait_async_action( + inc_counter_in_parallel( + AsyncInflightWindow * 2, + fun() -> + Rand = rand:uniform(1000), + {random_reply, Rand} + end, + #{} + ), + #{?snk_kind := buffer_worker_queue_drained, inflight := 0}, + ResumeInterval * 200 + ), + ok + end, + [ + fun(Trace) -> + ?assertMatch([#{inflight := 0}], ?of_kind(buffer_worker_queue_drained, Trace)) + end + ] + ), + Metrics = tap_metrics(?LINE), + #{ + counters := Counters, + gauges := #{queuing := 0, inflight := 0} + } = Metrics, + #{ + matched := Matched, + success := Success, + dropped := Dropped, + late_reply := LateReply, + failed := Failed + } = Counters, + ?assertEqual(Matched, Success + Dropped + LateReply + Failed), + ok. + t_retry_async_inflight_batch(_Config) -> ResumeInterval = 1_000, emqx_connector_demo:set_callback_mode(async_if_possible), From 3a6dbbdd058efdf596e16750304522ee9e43c8eb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 16:25:38 +0100 Subject: [PATCH 121/135] refactor(buffer_worker): ensure flsh message is never missed --- .../src/emqx_resource_buffer_worker.erl | 80 +++++++++++-------- .../test/emqx_connector_demo.erl | 2 + .../test/emqx_resource_SUITE.erl | 35 ++++---- 3 files changed, 67 insertions(+), 50 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 77494f4ba..e6fa1c537 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -70,6 +70,18 @@ -define(RETRY_IDX, 3). -define(WORKER_MREF_IDX, 4). +-define(ENSURE_ASYNC_FLUSH(InflightTID, EXPR), + (fun() -> + IsFullBefore = is_inflight_full(InflightTID), + case (EXPR) of + blocked -> + ok; + ok -> + maybe_flush_after_async_reply(IsFullBefore) + end + end)() +). + -type id() :: binary(). -type index() :: pos_integer(). -type expire_at() :: infinity | integer(). @@ -194,8 +206,8 @@ init({Id, Index, Opts}) -> ?tp(buffer_worker_init, #{id => Id, index => Index}), {ok, running, Data}. -running(enter, _, Data) -> - ?tp(buffer_worker_enter_running, #{id => maps:get(id, Data)}), +running(enter, _, #{tref := _Tref} = Data) -> + ?tp(buffer_worker_enter_running, #{id => maps:get(id, Data), tref => _Tref}), %% According to `gen_statem' laws, we mustn't call `maybe_flush' %% directly because it may decide to return `{next_state, blocked, _}', %% and that's an invalid response for a state enter call. @@ -212,9 +224,8 @@ running(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data) -> handle_query_requests(Request0, Data); running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> flush(St#{tref := undefined}); -running(internal, flush, St) -> - flush(St); running(info, {flush, _Ref}, _St) -> + ?tp(discarded_stale_flush, #{}), keep_state_and_data; running(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) @@ -472,10 +483,15 @@ flush(Data0) -> Data1 = cancel_flush_timer(Data0), CurrentCount = queue_count(Q0), IsFull = is_inflight_full(InflightTID), - ?tp(buffer_worker_flush, #{queue_count => CurrentCount, is_full => IsFull}), + InflightCount = inflight_num_batches(InflightTID), + ?tp(buffer_worker_flush, #{ + queued => CurrentCount, + is_inflight_full => IsFull, + inflight => InflightCount + }), case {CurrentCount, IsFull} of {0, _} -> - ?tp(buffer_worker_queue_drained, #{inflight => inflight_num_batches(InflightTID)}), + ?tp(buffer_worker_queue_drained, #{inflight => InflightCount}), {keep_state, Data1}; {_, true} -> ?tp(buffer_worker_flush_but_inflight_full, #{}), @@ -714,18 +730,18 @@ batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> end, Batch ), - {Action, PostFn1} = reply_caller_defer_metrics(Id, hd(Replies), QueryOpts), - PostFns = + {ShouldAck, PostFns} = lists:foldl( - fun(Reply, PostFns) -> - {_, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), - [PostFn | PostFns] + fun(Reply, {_ShouldAck, PostFns}) -> + %% _ShouldAck should be the same as ShouldAck starting from the second reply + {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), + {ShouldAck, [PostFn | PostFns]} end, - [PostFn1], - tl(Replies) + {ack, []}, + Replies ), PostFn = fun() -> lists:foreach(fun(F) -> F() end, lists:reverse(PostFns)) end, - {Action, PostFn}. + {ShouldAck, PostFn}. reply_caller(Id, Reply, QueryOpts) -> {ShouldAck, PostFn} = reply_caller_defer_metrics(Id, Reply, QueryOpts), @@ -978,7 +994,7 @@ handle_async_reply( discard -> ok; continue -> - handle_async_reply1(ReplyContext, Result) + ?ENSURE_ASYNC_FLUSH(InflightTID, handle_async_reply1(ReplyContext, Result)) end. handle_async_reply1( @@ -999,10 +1015,8 @@ handle_async_reply1( Now = now_(), case is_expired(ExpireAt, Now) of true -> - IsFullBefore = is_inflight_full(InflightTID), IsAcked = ack_inflight(InflightTID, Ref, Id, Index), IsAcked andalso emqx_resource_metrics:late_reply_inc(Id), - IsFullBefore andalso ?MODULE:flush_worker(Pid), ?tp(handle_async_reply_expired, #{expired => [_Query]}), ok; false -> @@ -1034,16 +1048,15 @@ do_handle_async_reply( ref => Ref, result => Result }), - IsFullBefore = is_inflight_full(InflightTID), case Action of nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ?MODULE:block(Pid); + ok = ?MODULE:block(Pid), + blocked; ack -> - do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) - end, - ok = maybe_flush_after_async_reply(IsFullBefore). + ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) + end. handle_async_batch_reply( #{ @@ -1056,7 +1069,7 @@ handle_async_batch_reply( discard -> ok; continue -> - handle_async_batch_reply1(ReplyContext, Result) + ?ENSURE_ASYNC_FLUSH(InflightTID, handle_async_batch_reply1(ReplyContext, Result)) end. handle_async_batch_reply1( @@ -1072,21 +1085,19 @@ handle_async_batch_reply1( #{batch_or_query => Batch, ref => Ref, result => Result} ), Now = now_(), - IsFullBefore = is_inflight_full(InflightTID), case sieve_expired_requests(Batch, Now) of {_NotExpired, []} -> %% this is the critical code path, %% we try not to do ets:lookup in this case %% because the batch can be quite big - ok = do_handle_async_batch_reply(ReplyContext, Result); + do_handle_async_batch_reply(ReplyContext, Result); {_NotExpired, _Expired} -> %% at least one is expired %% the batch from reply context is minimized, so it cannot be used %% to update the inflight items, hence discard Batch and lookup the RealBatch ?tp(handle_async_reply_expired, #{expired => _Expired}), - ok = handle_async_batch_reply2(ets:lookup(InflightTID, Ref), ReplyContext, Result, Now) - end, - ok = maybe_flush_after_async_reply(IsFullBefore). + handle_async_batch_reply2(ets:lookup(InflightTID, Ref), ReplyContext, Result, Now) + end. handle_async_batch_reply2([], _, _, _) -> %% should have caused the unknown_async_reply_discarded @@ -1124,9 +1135,8 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> %% some queries are not expired, put them back to the inflight batch %% so it can be either acked now or retried later ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired), - ok = do_handle_async_batch_reply(ReplyContext#{min_batch := RealNotExpired}, Result) - end, - ok. + do_handle_async_batch_reply(ReplyContext#{min_batch := RealNotExpired}, Result) + end. do_handle_async_batch_reply( #{ @@ -1151,7 +1161,8 @@ do_handle_async_batch_reply( nack -> %% Keep retrying. ok = mark_inflight_as_retriable(InflightTID, Ref), - ok = ?MODULE:block(Pid); + ok = ?MODULE:block(Pid), + blocked; ack -> ok = do_async_ack(InflightTID, Ref, Id, Index, PostFn, QueryOpts) end. @@ -1173,9 +1184,11 @@ maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = false) -> %% after it is handled, the inflight table must be even smaller %% hance we can rely on the buffer worker's flush timer to trigger %% the next flush + ?tp(skip_flushing_worker, #{}), ok; maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> %% the inflight table was full before handling aync reply + ?tp(do_flushing_worker, #{}), ok = ?MODULE:flush_worker(self()). %% check if the async reply is valid. @@ -1189,7 +1202,6 @@ maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> maybe_handle_unknown_async_reply(InflightTID, Ref) -> try ets:member(InflightTID, Ref) of true -> - %% NOTE: this does not mean the continue; false -> ?tp( @@ -1446,7 +1458,7 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> ok. %% used to update a batch after dropping expired individual queries. -update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> +update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) when NumExpired > 0 -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), ok = dec_inflight(InflightTID, NumExpired), ok. diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index a6b7b2339..3b5f83d05 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -314,6 +314,8 @@ counter_loop( %% drain the buffer (and inflights table) ReplyCount = 1 + (RandNum rem 3), Results = random_replies(ReplyCount), + %% add a delay to trigger inflight full + timer:sleep(5), lists:foreach( fun(Result) -> apply_reply(ReplyFun, Result) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 1362cd1cc..dfe64de24 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1510,8 +1510,9 @@ t_retry_async_inflight_full(_Config) -> %% this test case is to ensure the buffer worker will not go crazy even %% if the underlying connector is misbehaving: evaluate async callbacks multiple times t_async_reply_multi_eval(_Config) -> - ResumeInterval = 20, - AsyncInflightWindow = 5, + ResumeInterval = 5, + TotalTime = 5_000, + AsyncInflightWindow = 3, emqx_connector_demo:set_callback_mode(async_if_possible), {ok, _} = emqx_resource:create( ?ID, @@ -1528,29 +1529,31 @@ t_async_reply_multi_eval(_Config) -> } ), ?check_trace( - #{timetrap => 15_000}, + #{timetrap => 30_000}, begin %% block ok = emqx_resource:simple_sync_query(?ID, block), - {ok, {ok, _}} = - ?wait_async_action( - inc_counter_in_parallel( - AsyncInflightWindow * 2, - fun() -> - Rand = rand:uniform(1000), - {random_reply, Rand} - end, - #{} - ), - #{?snk_kind := buffer_worker_queue_drained, inflight := 0}, - ResumeInterval * 200 + ?wait_async_action( + inc_counter_in_parallel( + AsyncInflightWindow * 5, + fun() -> + Rand = rand:uniform(1000), + {random_reply, Rand} + end, + #{} ), + #{?snk_kind := buffer_worker_flush, inflight := 0, queued := 0}, + TotalTime + ), ok end, [ fun(Trace) -> - ?assertMatch([#{inflight := 0}], ?of_kind(buffer_worker_queue_drained, Trace)) + ?assertMatch( + [#{inflight := 0} | _], + lists:reverse(?of_kind(buffer_worker_queue_drained, Trace)) + ) end ] ), From 7a6465e2cfdf8fc0892938c9a9266c1676b3165b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 21:00:39 +0100 Subject: [PATCH 122/135] fix(buffer_worker): ensure flush timer reset in blocked state --- .../src/emqx_resource_buffer_worker.erl | 20 +++++++++++++------ .../test/emqx_resource_SUITE.erl | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index e6fa1c537..aaa07cf9a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -236,21 +236,24 @@ running(info, Info, _St) -> ?SLOG(error, #{msg => unexpected_msg, state => running, info => Info}), keep_state_and_data. -blocked(enter, _, #{resume_interval := ResumeT} = _St) -> +blocked(enter, _, #{resume_interval := ResumeT} = St0) -> ?tp(buffer_worker_enter_blocked, #{}), - {keep_state_and_data, {state_timeout, ResumeT, unblock}}; + %% discard the old timer, new timer will be started when entering running state again + St = cancel_flush_timer(St0), + {keep_state, St, {state_timeout, ResumeT, unblock}}; blocked(cast, block, _St) -> keep_state_and_data; blocked(cast, resume, St) -> resume_from_blocked(St); -blocked(cast, flush, Data) -> - resume_from_blocked(Data); +blocked(cast, flush, St) -> + resume_from_blocked(St); blocked(state_timeout, unblock, St) -> resume_from_blocked(St); blocked(info, ?SEND_REQ(_ReplyTo, _Req) = Request0, Data0) -> Data = collect_and_enqueue_query_requests(Request0, Data0), {keep_state, Data}; blocked(info, {flush, _Ref}, _Data) -> + %% ignore stale timer keep_state_and_data; blocked(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) @@ -622,6 +625,9 @@ do_flush( }), flush_worker(self()); false -> + ?tp(buffer_worker_queue_drained, #{ + inflight => inflight_num_batches(InflightTID) + }), ok end, {keep_state, Data1} @@ -700,6 +706,9 @@ do_flush(#{queue := Q1} = Data0, #{ Data2 = case {CurrentCount > 0, CurrentCount >= BatchSize} of {false, _} -> + ?tp(buffer_worker_queue_drained, #{ + inflight => inflight_num_batches(InflightTID) + }), Data1; {true, true} -> ?tp(buffer_worker_flush_ack_reflush, #{ @@ -1003,7 +1012,6 @@ handle_async_reply1( inflight_tid := InflightTID, resource_id := Id, worker_index := Index, - buffer_worker := Pid, min_query := ?QUERY(_, _, _, ExpireAt) = _Query } = ReplyContext, Result @@ -1100,7 +1108,7 @@ handle_async_batch_reply1( end. handle_async_batch_reply2([], _, _, _) -> - %% should have caused the unknown_async_reply_discarded + %% this usually should never happen unless the async callback is being evaluated concurrently ok; handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) -> ?INFLIGHT_ITEM(_, RealBatch, _IsRetriable, _WorkerMRef) = Inflight, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index dfe64de24..e22ca7750 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1543,7 +1543,7 @@ t_async_reply_multi_eval(_Config) -> end, #{} ), - #{?snk_kind := buffer_worker_flush, inflight := 0, queued := 0}, + #{?snk_kind := buffer_worker_queue_drained, inflight := 0}, TotalTime ), ok From 5bfd690df67ec5f9159f171863f38d779126cdd5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sun, 19 Feb 2023 00:38:22 +0200 Subject: [PATCH 123/135] fix(dashboard): fix swagger spec generation --- .../src/emqx_dashboard_swagger.erl | 6 +++- .../test/emqx_swagger_requestBody_SUITE.erl | 32 ++++++++++++++++++- .../test/emqx_telemetry_SUITE.erl | 3 +- changes/ce/fix-9997.en.md | 1 + changes/ce/fix-9997.zh.md | 1 + 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-9997.en.md create mode 100644 changes/ce/fix-9997.zh.md diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 1d0fa7352..77fcd4f76 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -417,13 +417,17 @@ init_prop(Keys, Init, Type) -> fun(Key, Acc) -> case hocon_schema:field_schema(Type, Key) of undefined -> Acc; - Schema -> Acc#{Key => to_bin(Schema)} + Schema -> Acc#{Key => format_prop(Key, Schema)} end end, Init, Keys ). +format_prop(deprecated, Value) when is_boolean(Value) -> Value; +format_prop(deprecated, _) -> true; +format_prop(_, Schema) -> to_bin(Schema). + trans_required(Spec, true, _) -> Spec#{required => true}; trans_required(Spec, _, path) -> Spec#{required => true}; trans_required(Spec, _, _) -> Spec. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 979d01c77..717a7d4ca 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -94,6 +94,30 @@ t_object(_Config) -> validate("/object", Spec, Refs), ok. +t_deprecated(_Config) -> + ?assertMatch( + [ + #{ + <<"emqx_swagger_requestBody_SUITE.deprecated_ref">> := + #{ + <<"properties">> := + [ + {<<"tag1">>, #{ + deprecated := true + }}, + {<<"tag2">>, #{ + deprecated := true + }}, + {<<"tag3">>, #{ + deprecated := false + }} + ] + } + } + ], + emqx_dashboard_swagger:components([{?MODULE, deprecated_ref}], #{}) + ). + t_nest_object(_Config) -> GoodRef = <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>, Spec = #{ @@ -812,7 +836,13 @@ fields(sub_fields) -> {init_file, fun init_file/1} ], desc => <<"test sub fields">> - }. + }; +fields(deprecated_ref) -> + [ + {tag1, mk(binary(), #{desc => <<"tag1">>, deprecated => {since, "4.3.0"}})}, + {tag2, mk(binary(), #{desc => <<"tag2">>, deprecated => true})}, + {tag3, mk(binary(), #{desc => <<"tag3">>, deprecated => false})} + ]. enable(type) -> boolean(); enable(desc) -> <<"Whether to enable tls psk support">>; diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index 37c1115aa..8c6a43e47 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -858,7 +858,8 @@ stop_slave(Node) -> ok = slave:stop(Node), ?assertEqual([node()], mria_mnesia:running_nodes()), ?assertEqual([], nodes()), - ok. + _ = application:stop(mria), + ok = application:start(mria). leave_cluster() -> try mnesia_hook:module_info() of diff --git a/changes/ce/fix-9997.en.md b/changes/ce/fix-9997.en.md new file mode 100644 index 000000000..be0344ec1 --- /dev/null +++ b/changes/ce/fix-9997.en.md @@ -0,0 +1 @@ +Fix Swagger API schema generation. `deprecated` metadata field is now always boolean, as [Swagger specification](https://swagger.io/specification/) suggests. diff --git a/changes/ce/fix-9997.zh.md b/changes/ce/fix-9997.zh.md new file mode 100644 index 000000000..6f1a0b779 --- /dev/null +++ b/changes/ce/fix-9997.zh.md @@ -0,0 +1 @@ +修复 Swagger API 生成时，`deprecated` 元数据字段未按照[标准](https://swagger.io/specification/)建议的那样始终为布尔值的问题。 From a10dbba08416d428507013f0e72b9cefe32c5a67 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Feb 2023 21:23:10 +0100 Subject: [PATCH 124/135] refactor(buffer_worker): less defensive on inflight counter decrement --- apps/emqx_resource/src/emqx_resource_buffer_worker.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index aaa07cf9a..601a77deb 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1440,10 +1440,10 @@ ack_inflight(InflightTID, Ref, Id, Index) -> [] -> 0 end, + ok = dec_inflight(InflightTID, Count), IsKnownRef = (Count > 0), case IsKnownRef of true -> - ok = dec_inflight(InflightTID, Count), emqx_resource_metrics:inflight_set(Id, Index, inflight_num_msgs(InflightTID)); false -> ok @@ -1466,15 +1466,16 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> ok. %% used to update a batch after dropping expired individual queries. -update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) when NumExpired > 0 -> +update_inflight_item(InflightTID, Ref, NewBatch, NumExpired) -> _ = ets:update_element(InflightTID, Ref, {?ITEM_IDX, NewBatch}), - ok = dec_inflight(InflightTID, NumExpired), - ok. + ok = dec_inflight(InflightTID, NumExpired). inc_inflight(InflightTID, Count) -> _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, Count}), ok. +dec_inflight(_InflightTID, 0) -> + ok; dec_inflight(InflightTID, Count) when Count > 0 -> _ = ets:update_counter(InflightTID, ?SIZE_REF, {2, -Count, 0, 0}), ok. From c97d17cc919b9e1a0280eb8da3372691629ae075 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 24 Feb 2023 01:24:36 +0100 Subject: [PATCH 125/135] test: refactor to loop wait for counters --- .../src/emqx_resource_buffer_worker.erl | 15 ++-- .../test/emqx_connector_demo.erl | 14 ++-- .../test/emqx_resource_SUITE.erl | 79 +++++++++---------- 3 files changed, 52 insertions(+), 56 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 601a77deb..38de2dc34 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -77,7 +77,7 @@ blocked -> ok; ok -> - maybe_flush_after_async_reply(IsFullBefore) + ok = maybe_flush_after_async_reply(IsFullBefore) end end)() ). @@ -486,15 +486,14 @@ flush(Data0) -> Data1 = cancel_flush_timer(Data0), CurrentCount = queue_count(Q0), IsFull = is_inflight_full(InflightTID), - InflightCount = inflight_num_batches(InflightTID), ?tp(buffer_worker_flush, #{ queued => CurrentCount, is_inflight_full => IsFull, - inflight => InflightCount + inflight => inflight_count(InflightTID) }), case {CurrentCount, IsFull} of {0, _} -> - ?tp(buffer_worker_queue_drained, #{inflight => InflightCount}), + ?tp(buffer_worker_queue_drained, #{inflight => inflight_count(InflightTID)}), {keep_state, Data1}; {_, true} -> ?tp(buffer_worker_flush_but_inflight_full, #{}), @@ -626,7 +625,7 @@ do_flush( flush_worker(self()); false -> ?tp(buffer_worker_queue_drained, #{ - inflight => inflight_num_batches(InflightTID) + inflight => inflight_count(InflightTID) }), ok end, @@ -707,7 +706,7 @@ do_flush(#{queue := Q1} = Data0, #{ case {CurrentCount > 0, CurrentCount >= BatchSize} of {false, _} -> ?tp(buffer_worker_queue_drained, #{ - inflight => inflight_num_batches(InflightTID) + inflight => inflight_count(InflightTID) }), Data1; {true, true} -> @@ -1336,10 +1335,10 @@ is_inflight_full(InflightTID) -> [{_, MaxSize}] = ets:lookup(InflightTID, ?MAX_SIZE_REF), %% we consider number of batches rather than number of messages %% because one batch request may hold several messages. - Size = inflight_num_batches(InflightTID), + Size = inflight_count(InflightTID), Size >= MaxSize. -inflight_num_batches(InflightTID) -> +inflight_count(InflightTID) -> case ets:info(InflightTID, size) of undefined -> 0; Size -> max(0, Size - ?INFLIGHT_META_ROWS) diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 3b5f83d05..f41087b20 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -176,7 +176,7 @@ on_batch_query(InstId, BatchReq, State) -> batch_big_payload(sync, InstId, BatchReq, State); {random_reply, Num} -> %% async batch retried - random_reply(Num) + make_random_reply(Num) end. on_batch_query_async(InstId, BatchReq, ReplyFunAndArgs, #{pid := Pid} = State) -> @@ -313,11 +313,11 @@ counter_loop( %% with 'ok' in the result, the buffer worker should eventually %% drain the buffer (and inflights table) ReplyCount = 1 + (RandNum rem 3), - Results = random_replies(ReplyCount), + Results = make_random_replies(ReplyCount), %% add a delay to trigger inflight full - timer:sleep(5), lists:foreach( fun(Result) -> + timer:sleep(rand:uniform(5)), apply_reply(ReplyFun, Result) end, Results @@ -354,12 +354,12 @@ maybe_register(_Name, _Pid, false) -> apply_reply({ReplyFun, Args}, Result) when is_function(ReplyFun) -> apply(ReplyFun, Args ++ [Result]). -random_replies(0) -> +make_random_replies(0) -> []; -random_replies(N) -> - [random_reply(N) | random_replies(N - 1)]. +make_random_replies(N) -> + [make_random_reply(N) | make_random_replies(N - 1)]. -random_reply(N) -> +make_random_reply(N) -> case rand:uniform(3) of 1 -> {ok, N}; diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index e22ca7750..9b1031c42 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1513,6 +1513,7 @@ t_async_reply_multi_eval(_Config) -> ResumeInterval = 5, TotalTime = 5_000, AsyncInflightWindow = 3, + TotalQueries = AsyncInflightWindow * 5, emqx_connector_demo:set_callback_mode(async_if_possible), {ok, _} = emqx_resource:create( ?ID, @@ -1528,49 +1529,33 @@ t_async_reply_multi_eval(_Config) -> resume_interval => ResumeInterval } ), - ?check_trace( - #{timetrap => 30_000}, - begin - %% block - ok = emqx_resource:simple_sync_query(?ID, block), - - ?wait_async_action( - inc_counter_in_parallel( - AsyncInflightWindow * 5, - fun() -> - Rand = rand:uniform(1000), - {random_reply, Rand} - end, - #{} - ), - #{?snk_kind := buffer_worker_queue_drained, inflight := 0}, - TotalTime - ), - ok + %% block + ok = emqx_resource:simple_sync_query(?ID, block), + inc_counter_in_parallel( + TotalQueries, + fun() -> + Rand = rand:uniform(1000), + {random_reply, Rand} end, - [ - fun(Trace) -> - ?assertMatch( - [#{inflight := 0} | _], - lists:reverse(?of_kind(buffer_worker_queue_drained, Trace)) - ) - end - ] + #{} ), - Metrics = tap_metrics(?LINE), - #{ - counters := Counters, - gauges := #{queuing := 0, inflight := 0} - } = Metrics, - #{ - matched := Matched, - success := Success, - dropped := Dropped, - late_reply := LateReply, - failed := Failed - } = Counters, - ?assertEqual(Matched, Success + Dropped + LateReply + Failed), - ok. + F = fun() -> + Metrics = tap_metrics(?LINE), + #{ + counters := Counters, + gauges := #{queuing := 0, inflight := 0} + } = Metrics, + #{ + matched := Matched, + success := Success, + dropped := Dropped, + late_reply := LateReply, + failed := Failed + } = Counters, + ?assertEqual(TotalQueries, Matched - 1), + ?assertEqual(Matched, Success + Dropped + LateReply + Failed) + end, + loop_wait(F, _Interval = 5, TotalTime). t_retry_async_inflight_batch(_Config) -> ResumeInterval = 1_000, @@ -2637,3 +2622,15 @@ assert_async_retry_fail_then_succeed_inflight(Trace) -> ) ), ok. + +loop_wait(F, Interval, TotalTime) when Interval >= TotalTime -> + %% do it for the last time + F(); +loop_wait(F, Interval, TotalTime) -> + try + F() + catch + _:_ -> + timer:sleep(Interval), + loop_wait(F, Interval, TotalTime - Interval) + end. From a638cc1d74abec88cb2a13b76808db580bad11c6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Feb 2023 13:54:02 +0100 Subject: [PATCH 126/135] docs: add change logs --- bin/emqx | 2 +- changes/ce/fix-10015.en.md | 7 +++++++ changes/ce/fix-10015.zh.md | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-10015.en.md create mode 100644 changes/ce/fix-10015.zh.md diff --git a/bin/emqx b/bin/emqx index 4fdf24186..f0366c09d 100755 --- a/bin/emqx +++ b/bin/emqx @@ -550,7 +550,7 @@ else die "Node $EMQX_NODE__NAME is not running?" fi fi - ## We have no choiece but to read the bootstrap config (with environment overrides available in the current shell) + ## We have no choice but to read the bootstrap config (with environment overrides available in the current shell) [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1 maybe_use_portable_dynlibs EMQX_BOOT_CONFIGS="$(call_hocon -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf multi_get "${CONF_KEYS[@]}")" diff --git a/changes/ce/fix-10015.en.md b/changes/ce/fix-10015.en.md new file mode 100644 index 000000000..5727a52cd --- /dev/null +++ b/changes/ce/fix-10015.en.md @@ -0,0 +1,7 @@ +To prevent errors caused by an incorrect EMQX node cookie provided from an environment variable, +we have implemented a fail-fast mechanism. +Previously, when an incorrect cookie was provided, the command would still attempt to ping the node, +leading to the error message 'Node xxx not responding to pings'. +With the new implementation, if a mismatched cookie is detected, +a message will be logged to indicate that the cookie is incorrect, +and the command will terminate with an error code of 1 without trying to ping the node. diff --git a/changes/ce/fix-10015.zh.md b/changes/ce/fix-10015.zh.md new file mode 100644 index 000000000..0f58fa99c --- /dev/null +++ b/changes/ce/fix-10015.zh.md @@ -0,0 +1,4 @@ +在 cookie 给错时，快速失败。 +在此修复前，即使 cookie 配置错误，emqx 命令仍然会尝试去 ping EMQX 节点， +并得到一个 "Node xxx not responding to pings" 的错误。 +修复后，如果发现 cookie 不一致，立即打印不一致的错误信息并退出。 From ea65ec10bfb28adf56dd40ad1e6b59d1b24a16c8 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 22 Feb 2023 12:27:04 +0100 Subject: [PATCH 127/135] feat(quic): add hidden low level settings for listeners. --- apps/emqx/src/emqx_listeners.erl | 67 +++++++++- apps/emqx/src/emqx_schema.erl | 215 +++++++++++++++++++++++++++++++ changes/ce/feat-10019.en.md | 1 + changes/ce/feat-10019.zh.md | 1 + 4 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 changes/ce/feat-10019.en.md create mode 100644 changes/ce/feat-10019.zh.md diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index fedf583e2..6982b3dea 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -383,17 +383,18 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, - {server_resumption_level, 2}, + {server_resumption_level, maps:get(server_resumption_level, Opts, 2)}, {verify, maps:get(verify, SSLOpts, verify_none)} ] ++ case maps:get(cacertfile, SSLOpts, undefined) of undefined -> []; CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] - end, + end ++ + optional_quic_listener_opts(Opts), ConnectionOpts = #{ conn_callback => emqx_quic_connection, - peer_unidi_stream_count => 1, - peer_bidi_stream_count => 10, + peer_unidi_stream_count => maps:get(peer_unidi_stream_count, Opts, 1), + peer_bidi_stream_count => maps:get(peer_bidi_stream_count, Opts, 10), zone => zone(Opts), listener => {quic, ListenerName}, limiter => limiter(Opts) @@ -726,3 +727,61 @@ get_ssl_options(Conf) -> error -> maps:get(<<"ssl_options">>, Conf, undefined) end. + +%% @doc Get QUIC optional settings for low level tunings. +%% @see quicer:quic_settings() +-spec optional_quic_listener_opts(map()) -> proplists:proplist(). +optional_quic_listener_opts(Conf) when is_map(Conf) -> + maps:to_list( + maps:filter( + fun(Name, _V) -> + lists:member( + Name, + quic_listener_optional_settings() + ) + end, + Conf + ) + ). + +-spec quic_listener_optional_settings() -> [atom()]. +quic_listener_optional_settings() -> + [ + max_bytes_per_key, + %% In conf schema we use handshake_idle_timeout + handshake_idle_timeout_ms, + %% In conf schema we use idle_timeout + idle_timeout_ms, + %% not use since we are server + %% tls_client_max_send_buffer, + tls_server_max_send_buffer, + stream_recv_window_default, + stream_recv_buffer_default, + conn_flow_control_window, + max_stateless_operations, + initial_window_packets, + send_idle_timeout_ms, + initial_rtt_ms, + max_ack_delay_ms, + disconnect_timeout_ms, + %% In conf schema, we use keep_alive_interval + keep_alive_interval_ms, + %% over written by conn opts + peer_bidi_stream_count, + %% over written by conn opts + peer_unidi_stream_count, + retry_memory_limit, + load_balancing_mode, + max_operations_per_drain, + send_buffering_enabled, + pacing_enabled, + migration_enabled, + datagram_receive_enabled, + server_resumption_level, + minimum_mtu, + maximum_mtu, + mtu_discovery_search_complete_timeout_us, + mtu_discovery_missing_probe_count, + max_binding_stateless_operations, + stateless_operation_expiration_ms + ]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8d24e6937..cae187686 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -120,6 +120,9 @@ -elvis([{elvis_style, god_modules, disable}]). +-define(BIT(Bits), (1 bsl (Bits))). +-define(MAX_UINT(Bits), (?BIT(Bits) - 1)). + namespace() -> broker. tags() -> @@ -862,6 +865,80 @@ fields("mqtt_quic_listener") -> } )}, {"ciphers", ciphers_schema(quic)}, + + {"max_bytes_per_key", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(64), + "Maximum number of bytes to encrypt with a single 1-RTT encryption key" + "before initiating key update. Default: 274877906944" + )}, + {"handshake_idle_timeout_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(64), + "How long a handshake can idle before it is discarded. Default: 10 000" + )}, + {"tls_server_max_send_buffer", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "How much Server TLS data to buffer. Default: 8192" + )}, + {"stream_recv_window_default", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "Initial stream receive window size. Default: 32678" + )}, + {"stream_recv_buffer_default", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "Stream initial buffer size. Default: 4096" + )}, + {"conn_flow_control_window", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "Connection-wide flow control window. Default: 16777216" + )}, + {"max_stateless_operations", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" + )}, + {"initial_window_packets", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "The size (in packets) of the initial congestion window for a connection. Default: 10" + )}, + {"send_idle_timeout_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "Reset congestion control after being idle for amount of time. Default: 1000" + )}, + {"initial_rtt_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "Initial RTT estimate. Default: 333" + )}, + {"max_ack_delay_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "How long to wait after receiving data before sending an ACK. Default: 25" + )}, + {"disconnect_timeout_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(32), + "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" + )}, {"idle_timeout", sc( duration_ms(), @@ -870,6 +947,12 @@ fields("mqtt_quic_listener") -> desc => ?DESC(fields_mqtt_quic_listener_idle_timeout) } )}, + {"idle_timeout_ms", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(64), + "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" + )}, {"handshake_idle_timeout", sc( duration_ms(), @@ -878,6 +961,12 @@ fields("mqtt_quic_listener") -> desc => ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout) } )}, + {"handshake_idle_timeout_ms", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(64), + "How long a handshake can idle before it is discarded" + )}, {"keep_alive_interval", sc( duration_ms(), @@ -886,6 +975,105 @@ fields("mqtt_quic_listener") -> desc => ?DESC(fields_mqtt_quic_listener_keep_alive_interval) } )}, + {"keep_alive_interval_ms", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(32), + "How often to send PING frames to keep a connection alive. Default: 0 (Disabled)" + )}, + {"peer_bidi_stream_count", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(16), + "Number of bidirectional streams to allow the peer to open." + )}, + {"peer_unidi_stream_count", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(16), + "Number of unidirectional streams to allow the peer to open." + )}, + {"retry_memory_limit", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(16), + "The percentage of available memory usable for handshake connections before" + "stateless retry is used. Calculated as `N/65535`. Default: 65" + )}, + {"load_balancing_mode", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(16), + "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED, default: 0" + )}, + {"max_operations_per_drain", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(8), + "The maximum number of operations to drain per connection quantum. Default: 16" + )}, + {"send_buffering_enabled", + quic_feature_toggle( + "Buffer send data instead of holding application buffers until" + "sent data is acknowledged. Default: 1 (Enabled)" + )}, + {"pacing_enabled", + quic_feature_toggle( + "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" + )}, + {"migration_enabled", + quic_feature_toggle( + "Enable clients to migrate IP addresses and tuples. " + "Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" + )}, + {"datagram_receive_enabled", + quic_feature_toggle( + "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" + )}, + {"server_resumption_level", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(8), + "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" + )}, + {"minimum_mtu", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(16), + "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" + )}, + {"maximum_mtu", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(16), + "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" + )}, + {"mtu_discovery_search_complete_timeout_us", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(64), + "The time in microseconds to wait before reattempting MTU probing if" + "max was not reached. Default: 600000000" + )}, + {"mtu_discovery_missing_probe_count", + quic_lowlevel_settings_uint( + 1, + ?MAX_UINT(8), + "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" + )}, + {"max_binding_stateless_operations", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(16), + "The maximum number of stateless operations that may be queued on" + "a binding at any one time. Default: 100" + )}, + {"stateless_operation_expiration_ms", + quic_lowlevel_settings_uint( + 0, + ?MAX_UINT(16), + "The time limit between operations for the same endpoint, in milliseconds. Default: 100" + )}, {"ssl_options", sc( ref("listener_quic_ssl_opts"), @@ -2638,3 +2826,30 @@ parse_port(Port) -> _:_ -> throw("bad_port_number") end. + +quic_feature_toggle(Desc) -> + sc( + %% true, false are for user facing + %% 0, 1 are for internal represtation + typerefl:alias("boolean", typerefl:union([true, false, 0, 1])), + #{ + desc => Desc, + hidden => true, + required => false, + converter => fun + (true) -> 1; + (false) -> 0; + (Other) -> Other + end + } + ). + +quic_lowlevel_settings_uint(Low, High, Desc) -> + sc( + range(Low, High), + #{ + required => false, + hidden => true, + desc => Desc + } + ). diff --git a/changes/ce/feat-10019.en.md b/changes/ce/feat-10019.en.md new file mode 100644 index 000000000..b6cc0381c --- /dev/null +++ b/changes/ce/feat-10019.en.md @@ -0,0 +1 @@ +Add low level tuning settings for QUIC listeners. diff --git a/changes/ce/feat-10019.zh.md b/changes/ce/feat-10019.zh.md new file mode 100644 index 000000000..9ef671b3d --- /dev/null +++ b/changes/ce/feat-10019.zh.md @@ -0,0 +1 @@ +为 QUIC 侦听器添加更多底层调优选项。 From ee77c113aa0305506eb0e8addbf6bf56882770c0 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 22 Feb 2023 14:40:04 +0100 Subject: [PATCH 128/135] test(quic): tuned listener --- apps/emqx/test/emqx_common_test_helpers.erl | 6 +- .../test/emqx_quic_multistreams_SUITE.erl | 57 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 5149b8b8a..dd88b013d 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -44,6 +44,7 @@ client_ssl_twoway/1, ensure_mnesia_stopped/0, ensure_quic_listener/2, + ensure_quic_listener/3, is_all_tcp_servers_available/1, is_tcp_server_available/2, is_tcp_server_available/3, @@ -511,6 +512,9 @@ ensure_dashboard_listeners_started(_App) -> -spec ensure_quic_listener(Name :: atom(), UdpPort :: inet:port_number()) -> ok. ensure_quic_listener(Name, UdpPort) -> + ensure_quic_listener(Name, UdpPort, #{}). +-spec ensure_quic_listener(Name :: atom(), UdpPort :: inet:port_number(), map()) -> ok. +ensure_quic_listener(Name, UdpPort, ExtraSettings) -> application:ensure_all_started(quicer), Conf = #{ acceptors => 16, @@ -533,7 +537,7 @@ ensure_quic_listener(Name, UdpPort) -> mountpoint => <<>>, zone => default }, - emqx_config:put([listeners, quic, Name], Conf), + emqx_config:put([listeners, quic, Name], maps:merge(Conf, ExtraSettings)), case emqx_listeners:start_listener(quic, Name, Conf) of ok -> ok; {error, {already_started, _Pid}} -> ok diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 17ba85da7..a95597f07 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -32,7 +32,8 @@ all() -> [ {group, mstream}, {group, shutdown}, - {group, misc} + {group, misc}, + t_listener_with_lowlevel_settings ]. groups() -> @@ -1892,6 +1893,60 @@ t_multi_streams_sub_0_rtt_stream_data_cont(Config) -> ok = emqtt:disconnect(C), ok = emqtt:disconnect(C0). +t_listener_with_lowlevel_settings(_Config) -> + LPort = 24567, + LowLevelTunings = #{ + max_bytes_per_key => 274877906, + %% In conf schema we use handshake_idle_timeout + handshake_idle_timeout_ms => 2000, + %% In conf schema we use idle_timeout + idle_timeout_ms => 20000, + %% not use since we are server + %% tls_client_max_send_buffer, + tls_server_max_send_buffer => 10240, + stream_recv_window_default => 1024, + stream_recv_buffer_default => 1024, + conn_flow_control_window => 1024, + max_stateless_operations => 16, + initial_window_packets => 1300, + send_idle_timeout_ms => 12000, + initial_rtt_ms => 300, + max_ack_delay_ms => 6000, + disconnect_timeout_ms => 60000, + %% In conf schema, we use keep_alive_interval + keep_alive_interval_ms => 12000, + %% over written by conn opts + peer_bidi_stream_count => 100, + %% over written by conn opts + peer_unidi_stream_count => 100, + retry_memory_limit => 640, + load_balancing_mode => 1, + max_operations_per_drain => 32, + send_buffering_enabled => 1, + pacing_enabled => 0, + migration_enabled => 0, + datagram_receive_enabled => 1, + server_resumption_level => 0, + minimum_mtu => 1250, + maximum_mtu => 1600, + mtu_discovery_search_complete_timeout_us => 500000000, + mtu_discovery_missing_probe_count => 6, + max_binding_stateless_operations => 200, + stateless_operation_expiration_ms => 200 + }, + ?assertEqual( + ok, emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, LPort, LowLevelTunings) + ), + timer:sleep(1000), + {ok, C} = emqtt:start_link([{proto_ver, v5}, {port, LPort}]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, _} = emqtt:subscribe(C, <<"test/1/2">>, qos2), + {ok, _, [_SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {<<"test/1/3">>, [{qos, 2}]} + ]), + ok = emqtt:disconnect(C), + emqx_listeners:stop_listener(emqx_listeners:listener_id(quic, ?FUNCTION_NAME)). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- From 681fd80cabe6ebb51da88e09954cee13f45c00e0 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 23 Feb 2023 11:55:34 +0100 Subject: [PATCH 129/135] docs(quic): move desc to i18n. --- apps/emqx/i18n/emqx_schema_i18n.conf | 341 +++++++++++++++++++++++++++ apps/emqx/src/emqx_schema.erl | 68 +++--- 2 files changed, 372 insertions(+), 37 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 39d5b2828..6c19fb697 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1901,6 +1901,347 @@ base_listener_acceptors { } } +fields_mqtt_quic_listener_max_bytes_per_key { + desc { + en: "Maximum number of bytes to encrypt with a single 1-RTT encryption keybefore initiating key update. Default: 274877906944" + zh: "" + } + label { + en: "Max bytes per key" + zh: "" + } +} + +fields_mqtt_quic_listener_handshake_idle_timeout_ms { + desc { + en: "How long a handshake can idle before it is discarded. Default: 10 000" + zh: "" + } + label { + en: "Handshake idle timeout ms" + zh: "" + } +} + +fields_mqtt_quic_listener_tls_server_max_send_buffer { + desc { + en: "How much Server TLS data to buffer. Default: 8192" + zh: "" + } + label { + en: "Tls server max send buffer" + zh: "" + } +} + +fields_mqtt_quic_listener_stream_recv_window_default { + desc { + en: "Initial stream receive window size. Default: 32678" + zh: "" + } + label { + en: "Stream recv window default" + zh: "" + } +} + +fields_mqtt_quic_listener_stream_recv_buffer_default { + desc { + en: "Stream initial buffer size. Default: 4096" + zh: "" + } + label { + en: "Stream recv buffer default" + zh: "" + } +} + +fields_mqtt_quic_listener_conn_flow_control_window { + desc { + en: "Connection-wide flow control window. Default: 16777216" + zh: "" + } + label { + en: "Conn flow control window" + zh: "" + } +} + +fields_mqtt_quic_listener_max_stateless_operations { + desc { + en: "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" + zh: "" + } + label { + en: "Max stateless operations" + zh: "" + } +} + +fields_mqtt_quic_listener_initial_window_packets { + desc { + en: "The size (in packets) of the initial congestion window for a connection. Default: 10" + zh: "" + } + label { + en: "Initial window packets" + zh: "" + } +} + +fields_mqtt_quic_listener_send_idle_timeout_ms { + desc { + en: "Reset congestion control after being idle for amount of time. Default: 1000" + zh: "" + } + label { + en: "Send idle timeout ms" + zh: "" + } +} + +fields_mqtt_quic_listener_initial_rtt_ms { + desc { + en: "Initial RTT estimate." + zh: "" + } + label { + en: "Initial rtt ms" + zh: "" + } +} + +fields_mqtt_quic_listener_max_ack_delay_ms { + desc { + en: "How long to wait after receiving data before sending an ACK. Default: 25" + zh: "" + } + label { + en: "Max ack delay ms" + zh: "" + } +} + +fields_mqtt_quic_listener_disconnect_timeout_ms { + desc { + en: "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" + zh: "" + } + label { + en: "Disconnect timeout ms" + zh: "" + } +} + +fields_mqtt_quic_listener_idle_timeout_ms { + desc { + en: "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" + zh: "" + } + label { + en: "Idle timeout ms" + zh: "" + } +} + +fields_mqtt_quic_listener_handshake_idle_timeout_ms { + desc { + en: "How long a handshake can idle before it is discarded" + zh: "" + } + label { + en: "Handshake idle timeout ms" + zh: "" + } +} + +fields_mqtt_quic_listener_keep_alive_interval_ms { + desc { + en: "How often to send PING frames to keep a connection alive." + zh: "" + } + label { + en: "Keep alive interval ms" + zh: "" + } +} + +fields_mqtt_quic_listener_peer_bidi_stream_count { + desc { + en: "Number of bidirectional streams to allow the peer to open." + zh: "" + } + label { + en: "Peer bidi stream count" + zh: "" + } +} + +fields_mqtt_quic_listener_peer_unidi_stream_count { + desc { + en: "Number of unidirectional streams to allow the peer to open." + zh: "" + } + label { + en: "Peer unidi stream count" + zh: "" + } +} + +fields_mqtt_quic_listener_retry_memory_limit { + desc { + en: "The percentage of available memory usable for handshake connections beforestateless retry is used. Calculated as `N/65535`. Default: 65" + zh: "" + } + label { + en: "Retry memory limit" + zh: "" + } +} + +fields_mqtt_quic_listener_load_balancing_mode { + desc { + en: "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED, default: 0" + zh: "" + } + label { + en: "Load balancing mode" + zh: "" + } +} + +fields_mqtt_quic_listener_max_operations_per_drain { + desc { + en: "The maximum number of operations to drain per connection quantum. Default: 16" + zh: "" + } + label { + en: "Max operations per drain" + zh: "" + } +} + +fields_mqtt_quic_listener_send_buffering_enabled { + desc { + en: "Buffer send data instead of holding application buffers untilsent data is acknowledged. Default: 1 (Enabled)" + zh: "" + } + label { + en: "Send buffering enabled" + zh: "" + } +} + +fields_mqtt_quic_listener_pacing_enabled { + desc { + en: "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" + zh: "" + } + label { + en: "Pacing enabled" + zh: "" + } +} + +fields_mqtt_quic_listener_migration_enabled { + desc { + en: "Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" + zh: "" + } + label { + en: "Migration enabled" + zh: "" + } +} + +fields_mqtt_quic_listener_datagram_receive_enabled { + desc { + en: "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" + zh: "" + } + label { + en: "Datagram receive enabled" + zh: "" + } +} + +fields_mqtt_quic_listener_server_resumption_level { + desc { + en: "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" + zh: "" + } + label { + en: "Server resumption level" + zh: "" + } +} + +fields_mqtt_quic_listener_minimum_mtu { + desc { + en: "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" + zh: "" + } + label { + en: "Minimum mtu" + zh: "" + } +} + +fields_mqtt_quic_listener_maximum_mtu { + desc { + en: "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" + zh: "" + } + label { + en: "Maximum mtu" + zh: "" + } +} + +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { + desc { + en: "The time in microseconds to wait before reattempting MTU probing ifmax was not reached. Default: 600000000" + zh: "" + } + label { + en: "Mtu discovery search complete timeout us" + zh: "" + } +} + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count { + desc { + en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" + zh: "" + } + label { + en: "Mtu discovery missing probe count" + zh: "" + } +} + +fields_mqtt_quic_listener_max_binding_stateless_operations { + desc { + en: "The maximum number of stateless operations that may be queued ona binding at any one time. Default: 100" + zh: "" + } + label { + en: "Max binding stateless operations" + zh: "" + } +} + +fields_mqtt_quic_listener_stateless_operation_expiration_ms { + desc { + en: "The time limit between operations for the same endpoint, in milliseconds. Default: 100" + zh: "" + } + label { + en: "Stateless operation expiration ms" + zh: "" + } +} + base_listener_max_connections { desc { en: """The maximum number of concurrent connections allowed by the listener.""" diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index cae187686..7000ffe0a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -870,74 +870,73 @@ fields("mqtt_quic_listener") -> quic_lowlevel_settings_uint( 1, ?MAX_UINT(64), - "Maximum number of bytes to encrypt with a single 1-RTT encryption key" - "before initiating key update. Default: 274877906944" + ?DESC(fields_mqtt_quic_listener_max_bytes_per_key) )}, {"handshake_idle_timeout_ms", quic_lowlevel_settings_uint( 1, ?MAX_UINT(64), - "How long a handshake can idle before it is discarded. Default: 10 000" + ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout) )}, {"tls_server_max_send_buffer", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "How much Server TLS data to buffer. Default: 8192" + ?DESC(fields_mqtt_quic_listener_tls_server_max_send_buffer) )}, {"stream_recv_window_default", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "Initial stream receive window size. Default: 32678" + ?DESC(fields_mqtt_quic_listener_stream_recv_window_default) )}, {"stream_recv_buffer_default", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "Stream initial buffer size. Default: 4096" + ?DESC(fields_mqtt_quic_listener_stream_recv_buffer_default) )}, {"conn_flow_control_window", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "Connection-wide flow control window. Default: 16777216" + ?DESC(fields_mqtt_quic_listener_conn_flow_control_window) )}, {"max_stateless_operations", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" + ?DESC(fields_mqtt_quic_listener_max_stateless_operations) )}, {"initial_window_packets", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "The size (in packets) of the initial congestion window for a connection. Default: 10" + ?DESC(fields_mqtt_quic_listener_initial_window_packets) )}, {"send_idle_timeout_ms", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "Reset congestion control after being idle for amount of time. Default: 1000" + ?DESC(fields_mqtt_quic_listener_send_idle_timeout_ms) )}, {"initial_rtt_ms", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "Initial RTT estimate. Default: 333" + ?DESC(fields_mqtt_quic_listener_initial_rtt_ms) )}, {"max_ack_delay_ms", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "How long to wait after receiving data before sending an ACK. Default: 25" + ?DESC(fields_mqtt_quic_listener_max_ack_delay_ms) )}, {"disconnect_timeout_ms", quic_lowlevel_settings_uint( 1, ?MAX_UINT(32), - "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" + ?DESC(fields_mqtt_quic_listener_disconnect_timeout_ms) )}, {"idle_timeout", sc( @@ -951,7 +950,7 @@ fields("mqtt_quic_listener") -> quic_lowlevel_settings_uint( 0, ?MAX_UINT(64), - "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" + ?DESC(fields_mqtt_quic_listener_idle_timeout_ms) )}, {"handshake_idle_timeout", sc( @@ -965,7 +964,7 @@ fields("mqtt_quic_listener") -> quic_lowlevel_settings_uint( 1, ?MAX_UINT(64), - "How long a handshake can idle before it is discarded" + ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout_ms) )}, {"keep_alive_interval", sc( @@ -979,100 +978,95 @@ fields("mqtt_quic_listener") -> quic_lowlevel_settings_uint( 0, ?MAX_UINT(32), - "How often to send PING frames to keep a connection alive. Default: 0 (Disabled)" + ?DESC(fields_mqtt_quic_listener_keep_alive_interval_ms) )}, {"peer_bidi_stream_count", quic_lowlevel_settings_uint( 1, ?MAX_UINT(16), - "Number of bidirectional streams to allow the peer to open." + ?DESC(fields_mqtt_quic_listener_peer_bidi_stream_count) )}, {"peer_unidi_stream_count", quic_lowlevel_settings_uint( 0, ?MAX_UINT(16), - "Number of unidirectional streams to allow the peer to open." + ?DESC(fields_mqtt_quic_listener_peer_unidi_stream_count) )}, {"retry_memory_limit", quic_lowlevel_settings_uint( 0, ?MAX_UINT(16), - "The percentage of available memory usable for handshake connections before" - "stateless retry is used. Calculated as `N/65535`. Default: 65" + ?DESC(fields_mqtt_quic_listener_retry_memory_limit) )}, {"load_balancing_mode", quic_lowlevel_settings_uint( 0, ?MAX_UINT(16), - "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED, default: 0" + ?DESC(fields_mqtt_quic_listener_load_balancing_mode) )}, {"max_operations_per_drain", quic_lowlevel_settings_uint( 0, ?MAX_UINT(8), - "The maximum number of operations to drain per connection quantum. Default: 16" + ?DESC(fields_mqtt_quic_listener_max_operations_per_drain) )}, {"send_buffering_enabled", quic_feature_toggle( - "Buffer send data instead of holding application buffers until" - "sent data is acknowledged. Default: 1 (Enabled)" + ?DESC(fields_mqtt_quic_listener_send_buffering_enabled) )}, {"pacing_enabled", quic_feature_toggle( - "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" + ?DESC(fields_mqtt_quic_listener_pacing_enabled) )}, {"migration_enabled", quic_feature_toggle( - "Enable clients to migrate IP addresses and tuples. " - "Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" + ?DESC(fields_mqtt_quic_listener_migration_enabled) )}, {"datagram_receive_enabled", quic_feature_toggle( - "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" + ?DESC(fields_mqtt_quic_listener_datagram_receive_enabled) )}, {"server_resumption_level", quic_lowlevel_settings_uint( 0, ?MAX_UINT(8), - "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" + ?DESC(fields_mqtt_quic_listener_server_resumption_level) )}, {"minimum_mtu", quic_lowlevel_settings_uint( 1, ?MAX_UINT(16), - "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" + ?DESC(fields_mqtt_quic_listener_minimum_mtu) )}, {"maximum_mtu", quic_lowlevel_settings_uint( 1, ?MAX_UINT(16), - "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" + ?DESC(fields_mqtt_quic_listener_maximum_mtu) )}, {"mtu_discovery_search_complete_timeout_us", quic_lowlevel_settings_uint( 0, ?MAX_UINT(64), - "The time in microseconds to wait before reattempting MTU probing if" - "max was not reached. Default: 600000000" + ?DESC(fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us) )}, {"mtu_discovery_missing_probe_count", quic_lowlevel_settings_uint( 1, ?MAX_UINT(8), - "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" + ?DESC(fields_mqtt_quic_listener_mtu_discovery_missing_probe_count) )}, {"max_binding_stateless_operations", quic_lowlevel_settings_uint( 0, ?MAX_UINT(16), - "The maximum number of stateless operations that may be queued on" - "a binding at any one time. Default: 100" + ?DESC(fields_mqtt_quic_listener_max_binding_stateless_operations) )}, {"stateless_operation_expiration_ms", quic_lowlevel_settings_uint( 0, ?MAX_UINT(16), - "The time limit between operations for the same endpoint, in milliseconds. Default: 100" + ?DESC(fields_mqtt_quic_listener_stateless_operation_expiration_ms) )}, {"ssl_options", sc( From f61c3c47ca22d7ead41ba88165779a25161637ac Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 23 Feb 2023 12:53:04 +0100 Subject: [PATCH 130/135] docs(quic): add zh --- apps/emqx/i18n/emqx_schema_i18n.conf | 136 +++++++++++++-------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 6c19fb697..9039a66ce 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1904,307 +1904,307 @@ base_listener_acceptors { fields_mqtt_quic_listener_max_bytes_per_key { desc { en: "Maximum number of bytes to encrypt with a single 1-RTT encryption keybefore initiating key update. Default: 274877906944" - zh: "" + zh: "在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944" } label { en: "Max bytes per key" - zh: "" + zh: "每个密钥的最大字节数" } } fields_mqtt_quic_listener_handshake_idle_timeout_ms { desc { en: "How long a handshake can idle before it is discarded. Default: 10 000" - zh: "" + zh: "一个握手在被丢弃之前可以空闲多长时间。 默认值：10 000" } label { en: "Handshake idle timeout ms" - zh: "" + zh: "握手空闲超时毫秒" } } fields_mqtt_quic_listener_tls_server_max_send_buffer { desc { en: "How much Server TLS data to buffer. Default: 8192" - zh: "" + zh: "缓冲多少TLS数据。 默认值：8192" } label { - en: "Tls server max send buffer" - zh: "" + en: "TLS server max send buffer" + zh: "TLS 服务器最大发送缓冲区" } } fields_mqtt_quic_listener_stream_recv_window_default { desc { en: "Initial stream receive window size. Default: 32678" - zh: "" + zh: "初始流接收窗口大小。 默认值：32678" } label { en: "Stream recv window default" - zh: "" + zh: "流接收窗口默认" } } fields_mqtt_quic_listener_stream_recv_buffer_default { desc { en: "Stream initial buffer size. Default: 4096" - zh: "" + zh: "流的初始缓冲区大小。默认：4096" } label { en: "Stream recv buffer default" - zh: "" + zh: "流媒体接收缓冲区默认值" } } fields_mqtt_quic_listener_conn_flow_control_window { desc { en: "Connection-wide flow control window. Default: 16777216" - zh: "" + zh: "连接的流控窗口。默认：16777216" } label { en: "Conn flow control window" - zh: "" + zh: "流控窗口" } } fields_mqtt_quic_listener_max_stateless_operations { desc { en: "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" - zh: "" + zh: "无状态操作的最大数量，在任何时候都可以在一个工作者上排队。默认值：16" } label { en: "Max stateless operations" - zh: "" + zh: "最大无状态操作数" } } fields_mqtt_quic_listener_initial_window_packets { desc { en: "The size (in packets) of the initial congestion window for a connection. Default: 10" - zh: "" + zh: "一个连接的初始拥堵窗口的大小（以包为单位）。默认值：10" } label { en: "Initial window packets" - zh: "" + zh: "初始窗口数据包" } } fields_mqtt_quic_listener_send_idle_timeout_ms { desc { en: "Reset congestion control after being idle for amount of time. Default: 1000" - zh: "" + zh: "在闲置一定时间后重置拥堵控制。默认值：1000" } label { en: "Send idle timeout ms" - zh: "" + zh: "发送空闲超时毫秒" } } fields_mqtt_quic_listener_initial_rtt_ms { desc { en: "Initial RTT estimate." - zh: "" + zh: "初始RTT估计" } label { - en: "Initial rtt ms" - zh: "" + en: "Initial RTT ms" + zh: "Initial RTT 毫秒" } } fields_mqtt_quic_listener_max_ack_delay_ms { desc { en: "How long to wait after receiving data before sending an ACK. Default: 25" - zh: "" + zh: "在收到数据后要等待多长时间才能发送一个ACK。默认值：25" } label { en: "Max ack delay ms" - zh: "" + zh: "最大应答延迟 毫秒" } } fields_mqtt_quic_listener_disconnect_timeout_ms { desc { en: "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" - zh: "" + zh: "在判定路径无效和断开连接之前，要等待多长时间的ACK。默认：16000" } label { en: "Disconnect timeout ms" - zh: "" + zh: "断开连接超时 毫秒" } } fields_mqtt_quic_listener_idle_timeout_ms { desc { en: "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" - zh: "" + zh: "一个连接在被优雅地关闭之前可以空闲多长时间。0 表示禁用超时" } label { en: "Idle timeout ms" - zh: "" + zh: "空闲超时 毫秒" } } fields_mqtt_quic_listener_handshake_idle_timeout_ms { desc { en: "How long a handshake can idle before it is discarded" - zh: "" + zh: "一个握手在被丢弃之前可以空闲多长时间" } label { en: "Handshake idle timeout ms" - zh: "" + zh: "握手空闲超时 毫秒" } } fields_mqtt_quic_listener_keep_alive_interval_ms { desc { en: "How often to send PING frames to keep a connection alive." - zh: "" + zh: "多长时间发送一次PING帧以保活连接。" } label { en: "Keep alive interval ms" - zh: "" + zh: "保持活着的时间间隔 毫秒" } } fields_mqtt_quic_listener_peer_bidi_stream_count { desc { en: "Number of bidirectional streams to allow the peer to open." - zh: "" + zh: "允许对端打开的双向流的数量" } label { en: "Peer bidi stream count" - zh: "" + zh: "对端双向流的数量" } } fields_mqtt_quic_listener_peer_unidi_stream_count { desc { en: "Number of unidirectional streams to allow the peer to open." - zh: "" + zh: "允许对端打开的单向流的数量" } label { en: "Peer unidi stream count" - zh: "" + zh: "对端单向流的数量" } } fields_mqtt_quic_listener_retry_memory_limit { desc { en: "The percentage of available memory usable for handshake connections beforestateless retry is used. Calculated as `N/65535`. Default: 65" - zh: "" + zh: "在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65" } label { en: "Retry memory limit" - zh: "" + zh: "重试内存限制" } } fields_mqtt_quic_listener_load_balancing_mode { desc { - en: "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED, default: 0" - zh: "" + en: "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0" + zh: "0: 禁用, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. 默认: 0" } label { en: "Load balancing mode" - zh: "" + zh: "负载平衡模式" } } fields_mqtt_quic_listener_max_operations_per_drain { desc { en: "The maximum number of operations to drain per connection quantum. Default: 16" - zh: "" + zh: "每个连接操作的最大耗费操作数。默认：16" } label { en: "Max operations per drain" - zh: "" + zh: "每次操作最大操作数" } } fields_mqtt_quic_listener_send_buffering_enabled { desc { en: "Buffer send data instead of holding application buffers untilsent data is acknowledged. Default: 1 (Enabled)" - zh: "" + zh: "缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）" } label { en: "Send buffering enabled" - zh: "" + zh: "启用发送缓冲功能" } } fields_mqtt_quic_listener_pacing_enabled { desc { en: "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" - zh: "" + zh: "有节奏的发送，以避免路径上的缓冲区过度填充。默认值：1（已启用）" } label { en: "Pacing enabled" - zh: "" + zh: "启用节奏发送" } } fields_mqtt_quic_listener_migration_enabled { desc { en: "Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" - zh: "" + zh: "开启客户端地址迁移功能。需要一个支持的负载平衡器，或者没有负载平衡器。默认值：1（已启用）" } label { en: "Migration enabled" - zh: "" + zh: "启用地址迁移" } } fields_mqtt_quic_listener_datagram_receive_enabled { desc { en: "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" - zh: "" + zh: "宣传对QUIC Datagram 扩展的支持。为将来保留。默认为0（FALSE）" } label { en: "Datagram receive enabled" - zh: "" + zh: "启用 Datagram 接收" } } fields_mqtt_quic_listener_server_resumption_level { desc { en: "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" - zh: "" + zh: "连接恢复 和/或 0-RTT 服务器支持。默认值：0（无恢复功能）" } label { en: "Server resumption level" - zh: "" + zh: "服务端连接恢复支持" } } fields_mqtt_quic_listener_minimum_mtu { desc { en: "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" - zh: "" + zh: "一个连接所支持的最小MTU。这将被作为起始MTU使用。默认值：1248" } label { - en: "Minimum mtu" - zh: "" + en: "Minimum MTU" + zh: "最小 MTU" } } fields_mqtt_quic_listener_maximum_mtu { desc { en: "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" - zh: "" + zh: "一个连接所支持的最大MTU。这将是最大的探测值。默认值：1500" } label { - en: "Maximum mtu" - zh: "" + en: "Maximum MTU" + zh: "最大 MTU" } } fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { desc { en: "The time in microseconds to wait before reattempting MTU probing ifmax was not reached. Default: 600000000" - zh: "" + zh: "如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000" } label { - en: "Mtu discovery search complete timeout us" + en: "MTU discovery search complete timeout us" zh: "" } } @@ -2212,33 +2212,33 @@ fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { fields_mqtt_quic_listener_mtu_discovery_missing_probe_count { desc { en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" - zh: "" + zh: "在任何时候都可以在一个绑定上排队的无状态操作的最大数量。默认值：3" } label { - en: "Mtu discovery missing probe count" - zh: "" + en: "MTU discovery missing probe count" + zh: "MTU发现丢失的探针数量" } } fields_mqtt_quic_listener_max_binding_stateless_operations { desc { en: "The maximum number of stateless operations that may be queued ona binding at any one time. Default: 100" - zh: "" + zh: "在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100" } label { en: "Max binding stateless operations" - zh: "" + zh: "最大绑定无状态操作" } } fields_mqtt_quic_listener_stateless_operation_expiration_ms { desc { en: "The time limit between operations for the same endpoint, in milliseconds. Default: 100" - zh: "" + zh: "同一个对端的操作之间的时间限制，单位是毫秒。 默认：100" } label { en: "Stateless operation expiration ms" - zh: "" + zh: "无状态操作过期 毫秒" } } From fde19e2b4b338a0e59e88bb327cfb6b3393f2f7c Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 23 Feb 2023 15:07:12 +0100 Subject: [PATCH 131/135] docs(quic): fix typos --- apps/emqx/i18n/emqx_schema_i18n.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 9039a66ce..a3aebe796 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1903,7 +1903,7 @@ base_listener_acceptors { fields_mqtt_quic_listener_max_bytes_per_key { desc { - en: "Maximum number of bytes to encrypt with a single 1-RTT encryption keybefore initiating key update. Default: 274877906944" + en: "Maximum number of bytes to encrypt with a single 1-RTT encryption key before initiating key update. Default: 274877906944" zh: "在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944" } label { @@ -2090,7 +2090,7 @@ fields_mqtt_quic_listener_peer_unidi_stream_count { fields_mqtt_quic_listener_retry_memory_limit { desc { - en: "The percentage of available memory usable for handshake connections beforestateless retry is used. Calculated as `N/65535`. Default: 65" + en: "The percentage of available memory usable for handshake connections before stateless retry is used. Calculated as `N/65535`. Default: 65" zh: "在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65" } label { @@ -2123,7 +2123,7 @@ fields_mqtt_quic_listener_max_operations_per_drain { fields_mqtt_quic_listener_send_buffering_enabled { desc { - en: "Buffer send data instead of holding application buffers untilsent data is acknowledged. Default: 1 (Enabled)" + en: "Buffer send data instead of holding application buffers until sent data is acknowledged. Default: 1 (Enabled)" zh: "缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）" } label { @@ -2200,7 +2200,7 @@ fields_mqtt_quic_listener_maximum_mtu { fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { desc { - en: "The time in microseconds to wait before reattempting MTU probing ifmax was not reached. Default: 600000000" + en: "The time in microseconds to wait before reattempting MTU probing if max was not reached. Default: 600000000" zh: "如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000" } label { @@ -2222,7 +2222,7 @@ fields_mqtt_quic_listener_mtu_discovery_missing_probe_count { fields_mqtt_quic_listener_max_binding_stateless_operations { desc { - en: "The maximum number of stateless operations that may be queued ona binding at any one time. Default: 100" + en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 100" zh: "在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100" } label { From de740a2fd9506ac2df2ac884ab5d7fe5171cb06e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Feb 2023 15:03:03 +0300 Subject: [PATCH 132/135] ci: use official zookeeper image Former one ate almost all of my free memory for some reason. This one looks more predictable. --- .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 976b0bc1c..63e74fa11 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -2,7 +2,7 @@ version: '3.9' services: zookeeper: - image: wurstmeister/zookeeper + image: docker.io/library/zookeeper:3.6 ports: - "2181:2181" container_name: zookeeper From 9cbe64a132d7cc82a27eac2f7c3aeda03a375bf6 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Feb 2023 15:05:20 +0300 Subject: [PATCH 133/135] fix(test): make strings json-friendly in kafka testsuite --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index d06218397..9b38e98d3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -268,7 +268,7 @@ kafka_bridge_rest_api_helper(Config) -> CreateBodyTmp = #{ <<"type">> => <<"kafka">>, <<"name">> => <<"my_kafka_bridge">>, - <<"bootstrap_hosts">> => maps:get(<<"bootstrap_hosts">>, Config), + <<"bootstrap_hosts">> => iolist_to_binary(maps:get(<<"bootstrap_hosts">>, Config)), <<"enable">> => true, <<"authentication">> => maps:get(<<"authentication">>, Config), <<"producer">> => #{ @@ -276,7 +276,7 @@ kafka_bridge_rest_api_helper(Config) -> topic => <<"t/#">> }, <<"kafka">> => #{ - <<"topic">> => erlang:list_to_binary(KafkaTopic), + <<"topic">> => iolist_to_binary(KafkaTopic), <<"buffer">> => #{ <<"memory_overload_protection">> => <<"false">> }, From 2b4e49e7df5d286141713fa2d15a0cbc65f619b4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Feb 2023 15:06:49 +0300 Subject: [PATCH 134/135] fix(bufworker): handle replies of simple async queries Before that change, simple queries were treated as "retries" essentially, thus skipping all the reply processing there is. --- .../src/emqx_resource_buffer_worker.erl | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 38de2dc34..a8ae4454d 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -109,6 +109,7 @@ start_link(Id, Index, Opts) -> -spec sync_query(id(), request(), query_opts()) -> Result :: term(). sync_query(Id, Request, Opts0) -> + ?tp(sync_query, #{id => Id, request => Request, query_opts => Opts0}), Opts1 = ensure_timeout_query_opts(Opts0, sync), Opts = ensure_expire_at(Opts1), PickKey = maps:get(pick_key, Opts, self()), @@ -118,6 +119,7 @@ sync_query(Id, Request, Opts0) -> -spec async_query(id(), request(), query_opts()) -> Result :: term(). async_query(Id, Request, Opts0) -> + ?tp(async_query, #{id => Id, request => Request, query_opts => Opts0}), Opts1 = ensure_timeout_query_opts(Opts0, async), Opts = ensure_expire_at(Opts1), PickKey = maps:get(pick_key, Opts, self()), @@ -133,6 +135,7 @@ simple_sync_query(Id, Request) -> %% call ends up calling buffering functions, that's a bug and %% would mess up the metrics anyway. `undefined' is ignored by %% `emqx_resource_metrics:*_shift/3'. + ?tp(simple_sync_query, #{id => Id, request => Request}), Index = undefined, QueryOpts = simple_query_opts(), emqx_resource_metrics:matched_inc(Id), @@ -144,6 +147,7 @@ simple_sync_query(Id, Request) -> %% simple async-query the resource without batching and queuing. -spec simple_async_query(id(), request(), query_opts()) -> term(). simple_async_query(Id, Request, QueryOpts0) -> + ?tp(simple_async_query, #{id => Id, request => Request, query_opts => QueryOpts0}), Index = undefined, QueryOpts = maps:merge(simple_query_opts(), QueryOpts0), emqx_resource_metrics:matched_inc(Id), @@ -877,7 +881,7 @@ handle_async_worker_down(Data0, Pid) -> {keep_state, Data}. call_query(QM0, Id, Index, Ref, Query, QueryOpts) -> - ?tp(call_query_enter, #{id => Id, query => Query}), + ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM0}), case emqx_resource_manager:ets_lookup(Id) of {ok, _Group, #{status := stopped}} -> ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); @@ -994,11 +998,12 @@ apply_query_fun(async, Mod, Id, Index, Ref, [?QUERY(_, _, _, _) | _] = Batch, Re handle_async_reply( #{ request_ref := Ref, - inflight_tid := InflightTID + inflight_tid := InflightTID, + query_opts := Opts } = ReplyContext, Result ) -> - case maybe_handle_unknown_async_reply(InflightTID, Ref) of + case maybe_handle_unknown_async_reply(InflightTID, Ref, Opts) of discard -> ok; continue -> @@ -1068,11 +1073,12 @@ do_handle_async_reply( handle_async_batch_reply( #{ inflight_tid := InflightTID, - request_ref := Ref + request_ref := Ref, + query_opts := Opts } = ReplyContext, Result ) -> - case maybe_handle_unknown_async_reply(InflightTID, Ref) of + case maybe_handle_unknown_async_reply(InflightTID, Ref, Opts) of discard -> ok; continue -> @@ -1206,7 +1212,9 @@ maybe_flush_after_async_reply(_WasFullBeforeReplyHandled = true) -> %% 2. If the request was previously failed and now pending on a retry, %% then this function will return 'continue' as there is no way to %% tell if this reply is stae or not. -maybe_handle_unknown_async_reply(InflightTID, Ref) -> +maybe_handle_unknown_async_reply(undefined, _Ref, #{simple_query := true}) -> + continue; +maybe_handle_unknown_async_reply(InflightTID, Ref, #{}) -> try ets:member(InflightTID, Ref) of true -> continue; From c883e4b36a1246ef8dcd07bfee64e64550af4880 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Feb 2023 18:16:35 +0300 Subject: [PATCH 135/135] test: drop custom `loop_wait` in favor of snabkaffe's `?retry` --- .../test/emqx_resource_SUITE.erl | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 9b1031c42..af72e86f9 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1539,23 +1539,26 @@ t_async_reply_multi_eval(_Config) -> end, #{} ), - F = fun() -> - Metrics = tap_metrics(?LINE), - #{ - counters := Counters, - gauges := #{queuing := 0, inflight := 0} - } = Metrics, - #{ - matched := Matched, - success := Success, - dropped := Dropped, - late_reply := LateReply, - failed := Failed - } = Counters, - ?assertEqual(TotalQueries, Matched - 1), - ?assertEqual(Matched, Success + Dropped + LateReply + Failed) - end, - loop_wait(F, _Interval = 5, TotalTime). + ?retry( + ResumeInterval, + TotalTime div ResumeInterval, + begin + Metrics = tap_metrics(?LINE), + #{ + counters := Counters, + gauges := #{queuing := 0, inflight := 0} + } = Metrics, + #{ + matched := Matched, + success := Success, + dropped := Dropped, + late_reply := LateReply, + failed := Failed + } = Counters, + ?assertEqual(TotalQueries, Matched - 1), + ?assertEqual(Matched, Success + Dropped + LateReply + Failed) + end + ). t_retry_async_inflight_batch(_Config) -> ResumeInterval = 1_000, @@ -2622,15 +2625,3 @@ assert_async_retry_fail_then_succeed_inflight(Trace) -> ) ), ok. - -loop_wait(F, Interval, TotalTime) when Interval >= TotalTime -> - %% do it for the last time - F(); -loop_wait(F, Interval, TotalTime) -> - try - F() - catch - _:_ -> - timer:sleep(Interval), - loop_wait(F, Interval, TotalTime - Interval) - end.