From 2fc0468e0c7715794304118ea766660aafcc3e89 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 11 Aug 2023 17:12:02 +0800 Subject: [PATCH 01/37] feat(banned): add a new API used to clear all banned data --- apps/emqx/src/emqx_banned.erl | 7 ++++- .../src/emqx_mgmt_api_banned.erl | 12 +++++++- .../test/emqx_mgmt_api_banned_SUITE.erl | 28 +++++++++++++++++++ rel/i18n/emqx_mgmt_api_banned.hocon | 5 ++++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index a5c46da19..ddd491b7c 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -38,7 +38,8 @@ delete/1, info/1, format/1, - parse/1 + parse/1, + clear/0 ]). %% gen_server callbacks @@ -226,6 +227,10 @@ delete(Who) -> info(InfoKey) -> mnesia:table_info(?BANNED_TAB, InfoKey). +clear() -> + _ = mria:clear_table(?BANNED_TAB), + ok. + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 508cf7d07..5a988cbfc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -79,6 +79,13 @@ schema("/banned") -> ?DESC(create_banned_api_response400) ) } + }, + delete => #{ + description => ?DESC(clear_banned_api), + tags => ?TAGS, + parameters => [], + 'requestBody' => [], + responses => #{204 => <<"No Content">>} } }; schema("/banned/:as/:who") -> @@ -168,7 +175,10 @@ banned(post, #{body := Body}) -> OldBannedFormat = emqx_utils_json:encode(format(Old)), {400, 'ALREADY_EXISTS', OldBannedFormat} end - end. + end; +banned(delete, _) -> + emqx_banned:clear(), + {204}. delete_banned(delete, #{bindings := Params}) -> case emqx_banned:look_up(Params) of diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index 9f1b560f7..3167a5621 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -157,6 +157,30 @@ t_delete(_Config) -> ), ok. +t_clear(_Config) -> + Now = erlang:system_time(second), + At = emqx_banned:to_rfc3339(Now), + Until = emqx_banned:to_rfc3339(Now + 3), + Who = <<"TestClient-"/utf8>>, + By = <<"banned suite 中"/utf8>>, + Reason = <<"test测试"/utf8>>, + As = <<"clientid">>, + Banned = #{ + as => clientid, + who => Who, + by => By, + reason => Reason, + at => At, + until => Until + }, + {ok, _} = create_banned(Banned), + ?assertMatch({ok, _}, clear_banned()), + ?assertMatch( + {error, {"HTTP/1.1", 404, "Not Found"}}, + delete_banned(binary_to_list(As), binary_to_list(Who)) + ), + ok. + list_banned() -> Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(get, Path) of @@ -176,5 +200,9 @@ delete_banned(As, Who) -> DeletePath = emqx_mgmt_api_test_util:api_path(["banned", As, Who]), emqx_mgmt_api_test_util:request_api(delete, DeletePath). +clear_banned() -> + ClearPath = emqx_mgmt_api_test_util:api_path(["banned"]), + emqx_mgmt_api_test_util:request_api(delete, ClearPath). + to_rfc3339(Sec) -> list_to_binary(calendar:system_time_to_rfc3339(Sec)). diff --git a/rel/i18n/emqx_mgmt_api_banned.hocon b/rel/i18n/emqx_mgmt_api_banned.hocon index 0a5439402..4bf72103f 100644 --- a/rel/i18n/emqx_mgmt_api_banned.hocon +++ b/rel/i18n/emqx_mgmt_api_banned.hocon @@ -57,4 +57,9 @@ who.desc: who.label: """Ban Object""" +clear_banned_api.desc: +"""Clear all banned data.""" +clear_banned_api.label: +"""Clear""" + } From e86a60fee3cda32d2c2b03590c6d88a3409c87d6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Aug 2023 11:27:02 +0200 Subject: [PATCH 02/37] docs: update build status badge in readme to point to actual workflow --- README-CN.md | 2 +- README-RU.md | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README-CN.md b/README-CN.md index 314b34b9a..8c6f8d8c3 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,7 +1,7 @@ # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) diff --git a/README-RU.md b/README-RU.md index 6baf38e2c..c22df7f3f 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,7 +1,7 @@ # Брокер EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) diff --git a/README.md b/README.md index 87d611c5e..3624e90f8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) -[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml) +[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml) [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx) [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/) From 81d200023d377820827ffbbc8f1f1c0cad505cfe Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 11 Aug 2023 17:37:45 +0800 Subject: [PATCH 03/37] chore: update changes --- changes/ce/feat-11436.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-11436.en.md diff --git a/changes/ce/feat-11436.en.md b/changes/ce/feat-11436.en.md new file mode 100644 index 000000000..e4e53f19d --- /dev/null +++ b/changes/ce/feat-11436.en.md @@ -0,0 +1 @@ +Add a new API endpoint `DELETE /banned` to clear all `banned` data. From bc417a676476229080812595b11c7633e22d19e6 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 10 Aug 2023 17:52:02 +0800 Subject: [PATCH 04/37] fix(datetime): make sure the epoch is not larger than the maximum supported value --- apps/emqx/src/emqx_datetime.erl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_datetime.erl b/apps/emqx/src/emqx_datetime.erl index 60f40130b..70e099af4 100644 --- a/apps/emqx/src/emqx_datetime.erl +++ b/apps/emqx/src/emqx_datetime.erl @@ -38,6 +38,12 @@ -typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}). -typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}). +%% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl, +%% here minus SECONDS_PER_DAY to tolerate timezone time offset, +%% so the maximum date can reach 9999-12-31 which is ample. +-define(MAXIMUM_EPOCH, 253402214400). +-define(MAXIMUM_EPOCH_MILLI, 253402214400_000). + to_epoch_second(DateTime) -> to_epoch(DateTime, second). @@ -47,8 +53,7 @@ to_epoch_millisecond(DateTime) -> to_epoch(DateTime, Unit) -> try case string:to_integer(DateTime) of - {Epoch, []} when Epoch >= 0 -> {ok, Epoch}; - {_Epoch, []} -> {error, bad_epoch}; + {Epoch, []} -> validate_epoch(Epoch, Unit); _ -> {ok, calendar:rfc3339_to_system_time(DateTime, [{unit, Unit}])} end catch @@ -71,6 +76,15 @@ human_readable_duration_string(Milliseconds) -> L2 = lists:map(fun({Time, Unit}) -> [integer_to_list(Time), Unit] end, L1), lists:flatten(lists:join(", ", L2)). +validate_epoch(Epoch, _Unit) when Epoch < 0 -> + {error, bad_epoch}; +validate_epoch(Epoch, second) when Epoch =< ?MAXIMUM_EPOCH -> + {ok, Epoch}; +validate_epoch(Epoch, millisecond) when Epoch =< ?MAXIMUM_EPOCH_MILLI -> + {ok, Epoch}; +validate_epoch(_Epoch, _Unit) -> + {error, bad_epoch}. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -compile(nowarn_export_all). @@ -90,9 +104,11 @@ fields(bar) -> ). epoch_ok_test() -> + BigStamp = 1 bsl 37, Args = [ {0, 0, 0, 0}, {1, 1, 1, 1}, + {BigStamp, BigStamp * 1000, BigStamp, BigStamp * 1000}, {"2022-01-01T08:00:00+08:00", "2022-01-01T08:00:00+08:00", 1640995200, 1640995200000} ], lists:foreach( @@ -112,9 +128,12 @@ check_ok(Input, Sec, Ms) -> ok. epoch_failed_test() -> + BigStamp = 1 bsl 38, Args = [ {-1, -1}, {"1s", "1s"}, + {BigStamp, 0}, + {0, BigStamp * 1000}, {"2022-13-13T08:00:00+08:00", "2022-13-13T08:00:00+08:00"} ], lists:foreach( From 749c2d075fcbbd33937f6826873d43c279815d35 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 10 Aug 2023 18:28:56 +0800 Subject: [PATCH 05/37] chore: update changes && bump app version --- apps/emqx/src/emqx.app.src | 2 +- changes/ce/fix-11424.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-11424.en.md diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index f324d6ae0..513f88612 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.7"}, + {vsn, "5.1.8"}, {modules, []}, {registered, []}, {applications, [ diff --git a/changes/ce/fix-11424.en.md b/changes/ce/fix-11424.en.md new file mode 100644 index 000000000..1d44d9745 --- /dev/null +++ b/changes/ce/fix-11424.en.md @@ -0,0 +1 @@ +Add a check for the maximum value of the timestamp in the API to ensure it is a valid Unix timestamp. From e10f9e5e9b382feb397ddef26fe29b8f8acc89f8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 12 Aug 2023 10:12:31 +0800 Subject: [PATCH 06/37] feat: change mqtt.max_packet_size type from string to bytesize --- apps/emqx/src/emqx_schema.erl | 17 +++++++++++++---- apps/emqx_dashboard/src/emqx_dashboard.app.src | 4 ++-- .../src/emqx_dashboard_swagger.erl | 2 -- .../src/emqx_opentelemetry.app.src | 2 +- apps/emqx_opentelemetry/src/emqx_otel.erl | 4 ++-- changes/ce/feat-11438.en.md | 2 ++ rel/i18n/emqx_schema.hocon | 2 +- 7 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 changes/ce/feat-11438.en.md diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e6bff790e..c65a4eaa6 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -46,7 +46,6 @@ -type timeout_duration_s() :: 0..?MAX_INT_TIMEOUT_S. -type timeout_duration_ms() :: 0..?MAX_INT_TIMEOUT_MS. -type bytesize() :: integer(). --type mqtt_max_packet_size() :: 1..?MAX_INT_MQTT_PACKET_SIZE. -type wordsize() :: bytesize(). -type percent() :: float(). -type file() :: string(). @@ -73,7 +72,6 @@ -typerefl_from_string({timeout_duration_s/0, emqx_schema, to_timeout_duration_s}). -typerefl_from_string({timeout_duration_ms/0, emqx_schema, to_timeout_duration_ms}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). --typerefl_from_string({mqtt_max_packet_size/0, emqx_schema, to_bytesize}). -typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}). -typerefl_from_string({percent/0, emqx_schema, to_percent}). -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). @@ -93,6 +91,7 @@ -export([ validate_heap_size/1, + validate_packet_size/1, user_lookup_fun_tr/2, validate_alarm_actions/1, validate_keepalive_multiplier/1, @@ -154,7 +153,6 @@ timeout_duration_s/0, timeout_duration_ms/0, bytesize/0, - mqtt_max_packet_size/0, wordsize/0, percent/0, file/0, @@ -2618,6 +2616,16 @@ validate_heap_size(Siz) when is_integer(Siz) -> validate_heap_size(_SizStr) -> {error, invalid_heap_size}. +validate_packet_size(Siz) when is_integer(Siz) andalso Siz < 1 -> + {error, #{reason => max_heap_size_too_small, minimum => 1}}; +validate_packet_size(Siz) when is_integer(Siz) andalso Siz > ?MAX_INT_MQTT_PACKET_SIZE -> + Max = integer_to_list(round(?MAX_INT_MQTT_PACKET_SIZE / 1024 / 1024)) ++ "M", + {error, #{reason => max_heap_size_too_large, maximum => Max}}; +validate_packet_size(Siz) when is_integer(Siz) -> + ok; +validate_packet_size(_SizStr) -> + {error, invalid_packet_size}. + validate_keepalive_multiplier(Multiplier) when is_number(Multiplier) andalso Multiplier >= 1.0 andalso Multiplier =< 65535.0 -> @@ -3380,9 +3388,10 @@ mqtt_general() -> )}, {"max_packet_size", sc( - mqtt_max_packet_size(), + bytesize(), #{ default => <<"1MB">>, + validator => fun ?MODULE:validate_packet_size/1, desc => ?DESC(mqtt_max_packet_size) } )}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index ee4e60118..f8395025e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl, emqx_bridge_http]}, @@ -12,6 +12,6 @@ {maintainers, ["EMQX Team "]}, {links, [ {"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-dashboard"} + {"Github", "https://github.com/emqx/emqx-dashboard5"} ]} ]}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index b0c78f0fe..a86c30893 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -856,8 +856,6 @@ typename_to_spec("timeout()", _Mod) -> }; typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; -typename_to_spec("mqtt_max_packet_size()", _Mod) -> - #{type => string, example => <<"32MB">>}; typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; typename_to_spec("map()", _Mod) -> diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index adffd4c88..7202b24c8 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_opentelemetry, [ {description, "OpenTelemetry for EMQX Broker"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_otel_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl index e17850b59..0b0e16cab 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -150,9 +150,9 @@ get_vm_gauge(Name) -> [{emqx_mgmt:vm_stats(Name), #{}}]. get_cluster_gauge('node.running') -> - length(emqx:cluster_nodes(running)); + [{length(emqx:cluster_nodes(running)), #{}}]; get_cluster_gauge('node.stopped') -> - length(emqx:cluster_nodes(stopped)). + [{length(emqx:cluster_nodes(stopped)), #{}}]. get_metric_counter(Name) -> [{emqx_metrics:val(Name), #{}}]. diff --git a/changes/ce/feat-11438.en.md b/changes/ce/feat-11438.en.md new file mode 100644 index 000000000..65cab5494 --- /dev/null +++ b/changes/ce/feat-11438.en.md @@ -0,0 +1,2 @@ +Changed the type of the `mqtt.mqx_packet_size` from string to byteSize to better represent the valid numeric range. +Strings will still be accepted for backwards compatibility. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 64de73b24..ffeaf83da 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1369,7 +1369,7 @@ sysmon_vm_process_low_watermark.label: """Process low watermark""" mqtt_max_packet_size.desc: -"""Maximum MQTT packet size allowed.""" +"""Maximum MQTT packet size allowed. Default: 1MB, Maximum: 256MB""" mqtt_max_packet_size.label: """Max Packet Size""" From 4ee4f59b412d96dec93d076a46e2cf452b31f7d2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 14 Aug 2023 10:37:15 +0800 Subject: [PATCH 07/37] chore: make spellcheck happy --- rel/i18n/emqx_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index ffeaf83da..6cf20f90c 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1369,7 +1369,7 @@ sysmon_vm_process_low_watermark.label: """Process low watermark""" mqtt_max_packet_size.desc: -"""Maximum MQTT packet size allowed. Default: 1MB, Maximum: 256MB""" +"""Maximum MQTT packet size allowed. Default: 1 MB, Maximum: 256 MB""" mqtt_max_packet_size.label: """Max Packet Size""" From 41180170f47e6db1d1ba18472b481f9972778580 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 14 Aug 2023 15:45:57 +0800 Subject: [PATCH 08/37] fix: update error msg --- apps/emqx/src/emqx_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c65a4eaa6..86ddb6577 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2617,10 +2617,10 @@ validate_heap_size(_SizStr) -> {error, invalid_heap_size}. validate_packet_size(Siz) when is_integer(Siz) andalso Siz < 1 -> - {error, #{reason => max_heap_size_too_small, minimum => 1}}; + {error, #{reason => max_mqtt_packet_size_too_small, minimum => 1}}; validate_packet_size(Siz) when is_integer(Siz) andalso Siz > ?MAX_INT_MQTT_PACKET_SIZE -> Max = integer_to_list(round(?MAX_INT_MQTT_PACKET_SIZE / 1024 / 1024)) ++ "M", - {error, #{reason => max_heap_size_too_large, maximum => Max}}; + {error, #{reason => max_mqtt_packet_size_too_large, maximum => Max}}; validate_packet_size(Siz) when is_integer(Siz) -> ok; validate_packet_size(_SizStr) -> From 0b066fa20c71e7381d0175eedf26d6f1c8cc2650 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 14 Aug 2023 16:55:03 +0800 Subject: [PATCH 09/37] fix(ldap): fix dependency problem --- apps/emqx_connector/rebar.config | 1 - apps/emqx_connector/src/emqx_connector.app.src | 3 +-- apps/emqx_ldap/src/emqx_ldap.app.src | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 132863127..78515abe6 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -9,7 +9,6 @@ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, - {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}} ]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index cd8ce864c..397cd0093 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.29"}, + {vsn, "0.1.30"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ @@ -12,7 +12,6 @@ eredis_cluster, eredis, epgsql, - eldap2, ehttpc, jose, emqx, diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index bdc9493c7..7a252dd33 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,10 +1,11 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, stdlib, + eldap, emqx_authn, emqx_authz ]}, From 1e6ba2e7486f98324bf35c810a2c0c5f851e52e7 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 14 Aug 2023 11:39:55 +0200 Subject: [PATCH 10/37] ci: add daily build of emqx-enterprise for debian10 platform --- .github/workflows/build_packages_cron.yaml | 43 +++++++++++++--------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index b245078da..431c4f5c4 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -13,8 +13,6 @@ jobs: linux: if: github.repository_owner == 'emqx' runs-on: aws-${{ matrix.arch }} - # always run in builder container because the host might have the wrong OTP version etc. - # otherwise buildx.sh does not run docker if arch and os matches the target arch and os. container: image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" @@ -24,11 +22,13 @@ jobs: profile: - ['emqx', 'master'] - ['emqx-enterprise', 'release-51'] + - ['emqx-enterprise', 'release-52'] otp: - 25.3.2-1 arch: - amd64 os: + - debian10 - ubuntu22.04 - amzn2023 builder: @@ -47,25 +47,32 @@ jobs: ref: ${{ matrix.profile[1] }} fetch-depth: 0 - - name: build emqx packages - env: - ELIXIR: ${{ matrix.elixir }} - PROFILE: ${{ matrix.profile[0] }} - ARCH: ${{ matrix.arch }} + - name: fix workdir run: | set -eu git config --global --add safe.directory "$GITHUB_WORKSPACE" - PKGTYPES="tgz pkg" - IS_ELIXIR="no" - for PKGTYPE in ${PKGTYPES}; - do - ./scripts/buildx.sh \ - --profile "${PROFILE}" \ - --pkgtype "${PKGTYPE}" \ - --arch "${ARCH}" \ - --elixir "${IS_ELIXIR}" \ - --builder "force_host" - done + # Align path for CMake caches + if [ ! "$PWD" = "/emqx" ]; then + ln -s $PWD /emqx + cd /emqx + fi + echo "pwd is $PWD" + + - name: build emqx packages + env: + PROFILE: ${{ matrix.profile[0] }} + ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal" + run: | + set -eu + make "${PROFILE}-tgz" + make "${PROFILE}-pkg" + - name: test emqx packages + env: + PROFILE: ${{ matrix.profile[0] }} + run: | + set -eu + ./scripts/pkg-tests.sh "${PROFILE}-tgz" + ./scripts/pkg-tests.sh "${PROFILE}-pkg" - uses: actions/upload-artifact@v3 if: success() with: From 82b8538041fe1ae0d472fe7645ba73b502f0fb59 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 28 Jul 2023 15:01:09 -0300 Subject: [PATCH 11/37] feat(gcp_producer): add support for defining message attributes and ordering key Fixes https://emqx.atlassian.net/browse/EMQX-10652 --- apps/emqx/rebar.config | 2 +- .../src/emqx_bridge_gcp_pubsub.app.src | 2 +- .../src/emqx_bridge_gcp_pubsub.erl | 30 +++ .../emqx_bridge_gcp_pubsub_impl_producer.erl | 85 ++++++++- .../emqx_bridge_gcp_pubsub_producer_SUITE.erl | 173 +++++++++++++++++- .../test/emqx_bridge_gcp_pubsub_tests.erl | 149 +++++++++++++++ changes/ee/feat-11403.en.md | 3 + mix.exs | 2 +- rebar.config | 2 +- rel/i18n/emqx_bridge_gcp_pubsub.hocon | 42 +++++ 10 files changed, 475 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl create mode 100644 changes/ee/feat-11403.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index c2dfccad6..355f005a8 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.10"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.14"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 9faf65860..c7dcea5c0 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index b3792da71..d1e827d84 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -113,6 +113,22 @@ fields(connector_config) -> ]; fields(producer) -> [ + {attributes_template, + sc( + hoconsc:array(ref(key_value_pair)), + #{ + default => [], + desc => ?DESC("attributes_template") + } + )}, + {ordering_key_template, + sc( + binary(), + #{ + default => <<>>, + desc => ?DESC("ordering_key_template") + } + )}, {payload_template, sc( binary(), @@ -203,6 +219,18 @@ fields("consumer_resource_opts") -> fun({Field, _Sc}) -> lists:member(Field, SupportedFields) end, ResourceFields ); +fields(key_value_pair) -> + [ + {key, + mk(binary(), #{ + required => true, + validator => [ + emqx_resource_validator:not_empty("Key templates must not be empty") + ], + desc => ?DESC(kv_pair_key) + })}, + {value, mk(binary(), #{required => true, desc => ?DESC(kv_pair_value)})} + ]; fields("get_producer") -> emqx_bridge_schema:status_fields() ++ fields("post_producer"); fields("post_producer") -> @@ -218,6 +246,8 @@ fields("put_consumer") -> desc("config_producer") -> ?DESC("desc_config"); +desc(key_value_pair) -> + ?DESC("kv_pair_desc"); desc("config_consumer") -> ?DESC("desc_config"); desc("consumer_resource_opts") -> diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index b1ded2121..f80fdc333 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -9,15 +9,20 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -type config() :: #{ + attributes_template := [#{key := binary(), value := binary()}], connect_timeout := emqx_schema:duration_ms(), max_retries := non_neg_integer(), + ordering_key_template := binary(), + payload_template := binary(), pubsub_topic := binary(), resource_opts := #{request_ttl := infinity | emqx_schema:duration_ms(), any() => term()}, service_account_json := emqx_bridge_gcp_pubsub_client:service_account_json(), any() => term() }. -type state() :: #{ + attributes_template := #{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()}, client := emqx_bridge_gcp_pubsub_client:state(), + ordering_key_template := emqx_placeholder:tmpl_token(), payload_template := emqx_placeholder:tmpl_token(), project_id := emqx_bridge_gcp_pubsub_client:project_id(), pubsub_topic := binary() @@ -57,6 +62,8 @@ on_start(InstanceId, Config0) -> }), Config = maps:update_with(service_account_json, fun emqx_utils_maps:binary_key_map/1, Config0), #{ + attributes_template := AttributesTemplate, + ordering_key_template := OrderingKeyTemplate, payload_template := PayloadTemplate, pubsub_topic := PubSubTopic, service_account_json := #{<<"project_id">> := ProjectId} @@ -65,6 +72,8 @@ on_start(InstanceId, Config0) -> {ok, Client} -> State = #{ client => Client, + attributes_template => preproc_attributes(AttributesTemplate), + ordering_key_template => emqx_placeholder:preproc_tmpl(OrderingKeyTemplate), payload_template => emqx_placeholder:preproc_tmpl(PayloadTemplate), project_id => ProjectId, pubsub_topic => PubSubTopic @@ -197,14 +206,76 @@ do_send_requests_async(State, Requests, ReplyFunAndArgs0) -> Request, ReplyFunAndArgs, Client ). --spec encode_payload(state(), Selected :: map()) -> #{data := binary()}. -encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) -> - Interpolated = - case PayloadTemplate of - [] -> emqx_utils_json:encode(Selected); - _ -> emqx_placeholder:proc_tmpl(PayloadTemplate, Selected) +-spec encode_payload(state(), Selected :: map()) -> + #{ + data := binary(), + attributes => #{binary() => binary()}, + 'orderingKey' => binary() + }. +encode_payload(State, Selected) -> + #{ + attributes_template := AttributesTemplate, + ordering_key_template := OrderingKeyTemplate, + payload_template := PayloadTemplate + } = State, + Data = render_payload(PayloadTemplate, Selected), + OrderingKey = render(OrderingKeyTemplate, Selected), + Attributes = proc_attributes(AttributesTemplate, Selected), + Payload0 = #{data => base64:encode(Data)}, + Payload1 = put_if(Payload0, attributes, Attributes, map_size(Attributes) > 0), + put_if(Payload1, 'orderingKey', OrderingKey, OrderingKey =/= <<>>). + +put_if(Acc, K, V, true) -> + Acc#{K => V}; +put_if(Acc, _K, _V, false) -> + Acc. + +-spec render_payload(emqx_placeholder:tmpl_token(), map()) -> binary(). +render_payload([] = _Template, Selected) -> + emqx_utils_json:encode(Selected); +render_payload(Template, Selected) -> + render(Template, Selected). + +render(Template, Selected) -> + Opts = #{ + return => full_binary, + var_trans => fun + (undefined) -> <<>>; + (X) -> emqx_utils_conv:bin(X) + end + }, + emqx_placeholder:proc_tmpl(Template, Selected, Opts). + +-spec preproc_attributes([#{key := binary(), value := binary()}]) -> + #{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()}. +preproc_attributes(AttributesTemplate) -> + lists:foldl( + fun(#{key := K, value := V}, Acc) -> + KT = emqx_placeholder:preproc_tmpl(K), + VT = emqx_placeholder:preproc_tmpl(V), + Acc#{KT => VT} end, - #{data => base64:encode(Interpolated)}. + #{}, + AttributesTemplate + ). + +-spec proc_attributes(#{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()}, map()) -> + #{binary() => binary()}. +proc_attributes(AttributesTemplate, Selected) -> + maps:fold( + fun(KT, VT, Acc) -> + K = render(KT, Selected), + case K =:= <<>> of + true -> + Acc; + false -> + V = render(VT, Selected), + Acc#{K => V} + end + end, + #{}, + AttributesTemplate + ). -spec to_pubsub_request([#{data := binary()}]) -> binary(). to_pubsub_request(Payloads) -> diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl index a9bbf6178..74affb9c8 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl @@ -63,7 +63,8 @@ single_config_tests() -> t_get_status_down, t_get_status_no_worker, t_get_status_timeout_calling_workers, - t_on_start_ehttpc_pool_already_started + t_on_start_ehttpc_pool_already_started, + t_attributes ]. only_sync_tests() -> @@ -212,7 +213,9 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> Error end, ct:pal("bridge creation result: ~p", [Res]), - ?assertEqual(element(1, ProbeResult), element(1, Res)), + ?assertEqual(element(1, ProbeResult), element(1, Res), #{ + creation_result => Res, probe_result => ProbeResult + }), case ProbeResult of {error, {{_, 500, _}, _, _}} -> error({bad_probe_result, ProbeResult}); _ -> ok @@ -456,6 +459,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) -> assert_valid_request_body(Body) -> BodyMap = emqx_utils_json:decode(Body, [return_maps]), ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap), + ct:pal("request: ~p", [BodyMap]), #{<<"messages">> := Messages} = BodyMap, lists:map( fun(Msg) -> @@ -480,6 +484,25 @@ assert_http_request(ServiceAccountJSON) -> error({timeout, #{mailbox => Mailbox}}) end. +receive_http_request(ServiceAccountJSON) -> + receive + {http, Headers, Body} -> + assert_valid_request_headers(Headers, ServiceAccountJSON), + #{<<"messages">> := Msgs} = emqx_utils_json:decode(Body, [return_maps]), + lists:map( + fun(Msg) -> + #{<<"data">> := Content64} = Msg, + Content = base64:decode(Content64), + Decoded = emqx_utils_json:decode(Content, [return_maps]), + Msg#{<<"data">> := Decoded} + end, + Msgs + ) + after 5_000 -> + {messages, Mailbox} = process_info(self(), messages), + error({timeout, #{mailbox => Mailbox}}) + end. + install_telemetry_handler(TestCase) -> Tid = ets:new(TestCase, [ordered_set, public]), HandlerId = TestCase, @@ -585,8 +608,8 @@ t_publish_success(Config) -> <<"topic">> := Topic, <<"payload">> := Payload, <<"metadata">> := #{<<"rule_id">> := RuleId} - } - ], + } = Msg + ] when not (is_map_key(<<"attributes">>, Msg) orelse is_map_key(<<"orderingKey">>, Msg)), DecodedMessages ), %% to avoid test flakiness @@ -1524,3 +1547,145 @@ t_query_sync(Config) -> [] ), ok. + +t_attributes(Config) -> + Name = ?config(gcp_pubsub_name, Config), + ServiceAccountJSON = ?config(service_account_json, Config), + LocalTopic = <<"t/topic">>, + ?check_trace( + begin + {ok, _} = create_bridge_http( + Config, + #{ + <<"local_topic">> => LocalTopic, + <<"attributes_template">> => + [ + #{ + <<"key">> => <<"${.payload.key}">>, + <<"value">> => <<"fixed_value">> + }, + #{ + <<"key">> => <<"${.payload.key}2">>, + <<"value">> => <<"${.payload.value}">> + }, + #{ + <<"key">> => <<"fixed_key">>, + <<"value">> => <<"fixed_value">> + }, + #{ + <<"key">> => <<"fixed_key2">>, + <<"value">> => <<"${.payload.value}">> + } + ], + <<"ordering_key_template">> => <<"${.payload.ok}">> + } + ), + %% without ordering key + Payload0 = + emqx_utils_json:encode( + #{ + <<"value">> => <<"payload_value">>, + <<"key">> => <<"payload_key">> + } + ), + Message0 = emqx_message:make(LocalTopic, Payload0), + emqx:publish(Message0), + DecodedMessages0 = receive_http_request(ServiceAccountJSON), + ?assertMatch( + [ + #{ + <<"attributes">> := + #{ + <<"fixed_key">> := <<"fixed_value">>, + <<"fixed_key2">> := <<"payload_value">>, + <<"payload_key">> := <<"fixed_value">>, + <<"payload_key2">> := <<"payload_value">> + }, + <<"data">> := #{ + <<"topic">> := _, + <<"payload">> := _ + } + } = Msg + ] when not is_map_key(<<"orderingKey">>, Msg), + DecodedMessages0 + ), + %% with ordering key + Payload1 = + emqx_utils_json:encode( + #{ + <<"value">> => <<"payload_value">>, + <<"key">> => <<"payload_key">>, + <<"ok">> => <<"ordering_key">> + } + ), + Message1 = emqx_message:make(LocalTopic, Payload1), + emqx:publish(Message1), + DecodedMessages1 = receive_http_request(ServiceAccountJSON), + ?assertMatch( + [ + #{ + <<"attributes">> := + #{ + <<"fixed_key">> := <<"fixed_value">>, + <<"fixed_key2">> := <<"payload_value">>, + <<"payload_key">> := <<"fixed_value">>, + <<"payload_key2">> := <<"payload_value">> + }, + <<"orderingKey">> := <<"ordering_key">>, + <<"data">> := #{ + <<"topic">> := _, + <<"payload">> := _ + } + } + ], + DecodedMessages1 + ), + %% will result in empty key + Payload2 = + emqx_utils_json:encode( + #{ + <<"value">> => <<"payload_value">>, + <<"ok">> => <<"ordering_key">> + } + ), + Message2 = emqx_message:make(LocalTopic, Payload2), + emqx:publish(Message2), + [DecodedMessage2] = receive_http_request(ServiceAccountJSON), + ?assertEqual( + #{ + <<"fixed_key">> => <<"fixed_value">>, + <<"fixed_key2">> => <<"payload_value">>, + <<"2">> => <<"payload_value">> + }, + maps:get(<<"attributes">>, DecodedMessage2) + ), + %% ensure loading cluster override file doesn't mangle the attribute + %% placeholders... + #{<<"bridges">> := #{?BRIDGE_TYPE_BIN := #{Name := RawConf}}} = + emqx_config:read_override_conf(#{override_to => cluster}), + ?assertEqual( + [ + #{ + <<"key">> => <<"${.payload.key}">>, + <<"value">> => <<"fixed_value">> + }, + #{ + <<"key">> => <<"${.payload.key}2">>, + <<"value">> => <<"${.payload.value}">> + }, + #{ + <<"key">> => <<"fixed_key">>, + <<"value">> => <<"fixed_value">> + }, + #{ + <<"key">> => <<"fixed_key2">>, + <<"value">> => <<"${.payload.value}">> + } + ], + maps:get(<<"attributes_template">>, RawConf) + ), + ok + end, + [] + ), + ok. diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl new file mode 100644 index 000000000..885754470 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl @@ -0,0 +1,149 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_gcp_pubsub_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%%=========================================================================== +%% Data section +%%=========================================================================== + +%% erlfmt-ignore +gcp_pubsub_producer_hocon() -> +""" +bridges.gcp_pubsub.my_producer { + attributes_template = [ + {key = \"${payload.key}\", value = fixed_value} + {key = \"${payload.key}2\", value = \"${.payload.value}\"} + {key = fixed_key, value = fixed_value} + {key = fixed_key2, value = \"${.payload.value}\"} + ] + connect_timeout = 15s + enable = false + local_topic = \"t/gcp/produ\" + max_retries = 2 + ordering_key_template = \"${.payload.ok}\" + payload_template = \"${.}\" + pipelining = 100 + pool_size = 8 + pubsub_topic = my-topic + resource_opts { + batch_size = 1 + batch_time = 0ms + health_check_interval = 15s + inflight_window = 100 + max_buffer_bytes = 256MB + query_mode = async + request_ttl = 15s + start_after_created = true + start_timeout = 5s + worker_pool_size = 16 + } + service_account_json { + auth_provider_x509_cert_url = \"https://www.googleapis.com/oauth2/v1/certs\" + auth_uri = \"https://accounts.google.com/o/oauth2/auth\" + client_email = \"test@myproject.iam.gserviceaccount.com\" + client_id = \"123812831923812319190\" + client_x509_cert_url = \"https://www.googleapis.com/robot/v1/metadata/x509/...\" + private_key = \"-----BEGIN PRIVATE KEY-----...\" + private_key_id = \"kid\" + project_id = myproject + token_uri = \"https://oauth2.googleapis.com/token\" + type = service_account + } +} +""". + +%%=========================================================================== +%% Helper functions +%%=========================================================================== + +parse(Hocon) -> + {ok, Conf} = hocon:binary(Hocon), + Conf. + +check(Conf) when is_map(Conf) -> + hocon_tconf:check_plain(emqx_bridge_schema, Conf). + +-define(validation_error(Reason, Value), + {emqx_bridge_schema, [ + #{ + kind := validation_error, + reason := Reason, + value := Value + } + ]} +). + +-define(ok_config(Cfg), #{ + <<"bridges">> := + #{ + <<"gcp_pubsub">> := + #{ + <<"my_producer">> := + Cfg + } + } +}). + +%%=========================================================================== +%% Test cases +%%=========================================================================== + +producer_attributes_validator_test_() -> + %% ensure this module is loaded when testing only this file + _ = emqx_bridge_enterprise:module_info(), + BaseConf = parse(gcp_pubsub_producer_hocon()), + Override = fun(Cfg) -> + emqx_utils_maps:deep_merge( + BaseConf, + #{ + <<"bridges">> => + #{ + <<"gcp_pubsub">> => + #{<<"my_producer">> => Cfg} + } + } + ) + end, + [ + {"base config", + ?_assertMatch( + ?ok_config(#{ + <<"attributes_template">> := [_, _, _, _] + }), + check(BaseConf) + )}, + {"empty key template", + ?_assertThrow( + ?validation_error("Key templates must not be empty", _), + check( + Override(#{ + <<"attributes_template">> => [ + #{ + <<"key">> => <<>>, + <<"value">> => <<"some_value">> + } + ] + }) + ) + )}, + {"empty value template", + ?_assertMatch( + ?ok_config(#{ + <<"attributes_template">> := [_] + }), + check( + Override(#{ + <<"attributes_template">> => [ + #{ + <<"key">> => <<"some_key">>, + <<"value">> => <<>> + } + ] + }) + ) + )} + ]. diff --git a/changes/ee/feat-11403.en.md b/changes/ee/feat-11403.en.md new file mode 100644 index 000000000..9942a2490 --- /dev/null +++ b/changes/ee/feat-11403.en.md @@ -0,0 +1,3 @@ +Added support for defining message attributes and ordering key templates for GCP PubSub Producer bridge. + +Also updated our HOCON library to fix an issue where objects in an array were being concatenated even if they lay on different lines. diff --git a/mix.exs b/mix.exs index 3d29642ba..2580cc3d8 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.14", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.16", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index dd1e139f8..2d605c18f 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.14"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} diff --git a/rel/i18n/emqx_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_bridge_gcp_pubsub.hocon index 39c4b7417..b5dffec1f 100644 --- a/rel/i18n/emqx_bridge_gcp_pubsub.hocon +++ b/rel/i18n/emqx_bridge_gcp_pubsub.hocon @@ -46,6 +46,18 @@ payload_template.desc: payload_template.label: """Payload template""" +attributes_template.desc: +"""The template for formatting the outgoing message attributes. Undefined values will be rendered as empty string values. Empty keys are removed from the attribute map.""" + +attributes_template.label: +"""Attributes template""" + +ordering_key_template.desc: +"""The template for formatting the outgoing message ordering key. Undefined values will be rendered as empty string values. This value will not be added to the message if it's empty.""" + +ordering_key_template.label: +"""Ordering Key template""" + pipelining.desc: """A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request.""" @@ -64,6 +76,36 @@ pubsub_topic.desc: pubsub_topic.label: """GCP PubSub Topic""" +producer_attributes.desc: +"""List of key-value pairs representing templates to construct the attributes for a given GCP PubSub message. Both keys and values support the placeholder `${var_name}` notation. Keys that are undefined or resolve to an empty string are omitted from the attribute map.""" + +producer_attributes.label: +"""Attributes Template""" + +producer_ordering_key.desc: +"""Template for the Ordering Key of a given GCP PubSub message. If the resolved value is undefined or an empty string, the ordering key property is omitted from the message.""" + +producer_ordering_key.label: +"""Ordering Key Template""" + +kv_pair_desc.desc: +"""Key-value pair.""" + +kv_pair_desc.label: +"""Key-value pair""" + +kv_pair_key.desc: +"""Key""" + +kv_pair_key.label: +"""Key""" + +kv_pair_value.desc: +"""Value""" + +kv_pair_value.label: +"""Value""" + service_account_json.desc: """JSON containing the GCP Service Account credentials to be used with PubSub. When a GCP Service Account is created (as described in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount), you have the option of downloading the credentials in JSON form. That's the file needed.""" From 926eb4e3dd27f532bc865563a778e397578c6d36 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 14 Aug 2023 10:33:24 -0300 Subject: [PATCH 12/37] test: rm unused var warning --- .../test/emqx_bridge_pulsar_impl_producer_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index 38b112e99..fb358906f 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -1089,7 +1089,7 @@ t_strategy_key_validation(Config) -> #{ <<"kind">> := <<"validation_error">>, <<"reason">> := <<"Message key cannot be empty", _/binary>> - } = Msg + } }}}, probe_bridge_api( Config, @@ -1103,7 +1103,7 @@ t_strategy_key_validation(Config) -> #{ <<"kind">> := <<"validation_error">>, <<"reason">> := <<"Message key cannot be empty", _/binary>> - } = Msg + } }}}, create_bridge_api( Config, From d93e1bbf08c771ad378eaaccaa75a51a725e5900 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 10 Aug 2023 17:01:18 -0300 Subject: [PATCH 13/37] feat(mongodb): add configurable option to override legacy protocol usage Fixes https://emqx.atlassian.net/browse/EMQX-10750 Fixes https://github.com/emqx/emqx/discussions/11428 See https://github.com/emqx/mongodb-erlang/pull/39 --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- .../test/emqx_bridge_mongodb_SUITE.erl | 58 +++++++++++++++++-- apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_mongodb/rebar.config | 2 +- apps/emqx_mongodb/src/emqx_mongodb.app.src | 2 +- apps/emqx_mongodb/src/emqx_mongodb.erl | 7 +++ apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- changes/ce/feat-11429.en.md | 1 + rel/i18n/emqx_mongodb.hocon | 6 ++ 9 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 changes/ce/feat-11429.en.md diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 4ab86ef4a..ae7bea5da 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.24"}, + {vsn, "0.1.25"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [ diff --git a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl index 758124713..785afc4a0 100644 --- a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl +++ b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl @@ -29,7 +29,8 @@ group_tests() -> t_payload_template, t_collection_template, t_mongo_date_rule_engine_functions, - t_get_status_server_selection_too_short + t_get_status_server_selection_too_short, + t_use_legacy_protocol_option ]. groups() -> @@ -180,6 +181,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type, Config) -> " replica_set_name = rs0\n" " servers = [~p]\n" " w_mode = safe\n" + " use_legacy_protocol = auto\n" " database = mqtt\n" " resource_opts = {\n" " query_mode = ~s\n" @@ -205,6 +207,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type, Config) -> " collection = mycol\n" " servers = [~p]\n" " w_mode = safe\n" + " use_legacy_protocol = auto\n" " database = mqtt\n" " resource_opts = {\n" " query_mode = ~s\n" @@ -230,6 +233,7 @@ mongo_config(MongoHost, MongoPort0, single = Type, Config) -> " collection = mycol\n" " server = ~p\n" " w_mode = safe\n" + " use_legacy_protocol = auto\n" " database = mqtt\n" " resource_opts = {\n" " query_mode = ~s\n" @@ -286,10 +290,8 @@ clear_db(Config) -> mongo_api:disconnect(Client). find_all(Config) -> - Type = mongo_type_bin(?config(mongo_type, Config)), - Name = ?config(mongo_name, Config), #{<<"collection">> := Collection} = ?config(mongo_config, Config), - ResourceID = emqx_bridge_resource:resource_id(Type, Name), + ResourceID = resource_id(Config), emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}). find_all_wait_until_non_empty(Config) -> @@ -340,6 +342,27 @@ probe_bridge_api(Config, Overrides) -> ct:pal("bridge probe result: ~p", [Res]), Res. +resource_id(Config) -> + Type0 = ?config(mongo_type, Config), + Name = ?config(mongo_name, Config), + Type = mongo_type_bin(Type0), + emqx_bridge_resource:resource_id(Type, Name). + +get_worker_pids(Config) -> + ResourceID = resource_id(Config), + %% abusing health check api a bit... + GetWorkerPid = fun(TopologyPid) -> + mongoc:transaction_query(TopologyPid, fun(#{pool := WorkerPid}) -> WorkerPid end) + end, + {ok, WorkerPids = [_ | _]} = + emqx_resource_pool:health_check_workers( + ResourceID, + GetWorkerPid, + 5_000, + #{return_values => true} + ), + WorkerPids. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -494,3 +517,30 @@ t_get_status_server_selection_too_short(Config) -> emqx_utils_json:decode(Body) ), ok. + +t_use_legacy_protocol_option(Config) -> + ResourceID = resource_id(Config), + {ok, _} = create_bridge(Config, #{<<"use_legacy_protocol">> => <<"true">>}), + ?retry( + _Interval0 = 200, + _NAttempts0 = 20, + ?assertMatch({ok, connected}, emqx_resource_manager:health_check(ResourceID)) + ), + WorkerPids0 = get_worker_pids(Config), + Expected0 = maps:from_keys(WorkerPids0, true), + LegacyOptions0 = maps:from_list([{Pid, mc_utils:use_legacy_protocol(Pid)} || Pid <- WorkerPids0]), + ?assertEqual(Expected0, LegacyOptions0), + {ok, _} = delete_bridge(Config), + + {ok, _} = create_bridge(Config, #{<<"use_legacy_protocol">> => <<"false">>}), + ?retry( + _Interval0 = 200, + _NAttempts0 = 20, + ?assertMatch({ok, connected}, emqx_resource_manager:health_check(ResourceID)) + ), + WorkerPids1 = get_worker_pids(Config), + Expected1 = maps:from_keys(WorkerPids1, false), + LegacyOptions1 = maps:from_list([{Pid, mc_utils:use_legacy_protocol(Pid)} || Pid <- WorkerPids1]), + ?assertEqual(Expected1, LegacyOptions1), + + ok. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index e86351556..dd1915cfb 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.11"}, + {vsn, "0.2.12"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_mongodb/rebar.config b/apps/emqx_mongodb/rebar.config index cfd7dc9be..577dee8b8 100644 --- a/apps/emqx_mongodb/rebar.config +++ b/apps/emqx_mongodb/rebar.config @@ -3,5 +3,5 @@ {erl_opts, [debug_info]}. {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.20"}}} + , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.21"}}} ]}. diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 00dcb0cfb..eb846a7ab 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index dfa732a7b..77161911a 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -141,6 +141,11 @@ mongo_fields() -> {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {username, fun emqx_connector_schema_lib:username/1}, {password, fun emqx_connector_schema_lib:password/1}, + {use_legacy_protocol, + hoconsc:mk(hoconsc:enum([auto, true, false]), #{ + default => auto, + desc => ?DESC("use_legacy_protocol") + })}, {auth_source, #{ type => binary(), required => false, @@ -429,6 +434,8 @@ init_worker_options([{w_mode, V} | R], Acc) -> init_worker_options(R, [{w_mode, V} | Acc]); init_worker_options([{r_mode, V} | R], Acc) -> init_worker_options(R, [{r_mode, V} | Acc]); +init_worker_options([{use_legacy_protocol, V} | R], Acc) -> + init_worker_options(R, [{use_legacy_protocol, V} | Acc]); init_worker_options([_ | R], Acc) -> init_worker_options(R, Acc); init_worker_options([], Acc) -> diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 5238328f0..8f7c9aa17 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/changes/ce/feat-11429.en.md b/changes/ce/feat-11429.en.md new file mode 100644 index 000000000..5c0028774 --- /dev/null +++ b/changes/ce/feat-11429.en.md @@ -0,0 +1 @@ +Added option to configure detection of legacy protocol in MondoDB connectors and bridges. diff --git a/rel/i18n/emqx_mongodb.hocon b/rel/i18n/emqx_mongodb.hocon index b1830868d..162460281 100644 --- a/rel/i18n/emqx_mongodb.hocon +++ b/rel/i18n/emqx_mongodb.hocon @@ -149,4 +149,10 @@ wait_queue_timeout.desc: wait_queue_timeout.label: """Wait Queue Timeout""" +use_legacy_protocol.desc: +"""Whether to use MongoDB's legacy protocol for communicating with the database. The default is to attempt to automatically determine if the newer protocol is supported.""" + +use_legacy_protocol.label: +"""Use legacy protocol""" + } From 23f5cea4826677c720b15063be2a02a03f4aa73e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 14 Aug 2023 13:39:38 -0300 Subject: [PATCH 14/37] feat: handle strange key values when resolving placeholders --- .../emqx_bridge_gcp_pubsub_impl_producer.erl | 41 ++++++- .../emqx_bridge_gcp_pubsub_producer_SUITE.erl | 115 +++++++++++++++++- 2 files changed, 150 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index f80fdc333..dc5eb01aa 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -219,7 +219,7 @@ encode_payload(State, Selected) -> payload_template := PayloadTemplate } = State, Data = render_payload(PayloadTemplate, Selected), - OrderingKey = render(OrderingKeyTemplate, Selected), + OrderingKey = render_key(OrderingKeyTemplate, Selected), Attributes = proc_attributes(AttributesTemplate, Selected), Payload0 = #{data => base64:encode(Data)}, Payload1 = put_if(Payload0, attributes, Attributes, map_size(Attributes) > 0), @@ -234,9 +234,40 @@ put_if(Acc, _K, _V, false) -> render_payload([] = _Template, Selected) -> emqx_utils_json:encode(Selected); render_payload(Template, Selected) -> - render(Template, Selected). + render_value(Template, Selected). -render(Template, Selected) -> +render_key(Template, Selected) -> + Opts = #{ + return => full_binary, + var_trans => fun + (_Var, undefined) -> + <<>>; + (Var, X) when is_boolean(X) -> + throw({bad_value_for_key, Var, X}); + (_Var, X) when is_binary(X); is_number(X); is_atom(X) -> + emqx_utils_conv:bin(X); + (Var, X) -> + throw({bad_value_for_key, Var, X}) + end + }, + try + emqx_placeholder:proc_tmpl(Template, Selected, Opts) + catch + throw:{bad_value_for_key, Var, X} -> + ?tp( + warning, + "gcp_pubsub_producer_bad_value_for_key", + #{ + placeholder => Var, + value => X, + action => "key ignored", + hint => "only plain values like strings and numbers can be used in keys" + } + ), + <<>> + end. + +render_value(Template, Selected) -> Opts = #{ return => full_binary, var_trans => fun @@ -264,12 +295,12 @@ preproc_attributes(AttributesTemplate) -> proc_attributes(AttributesTemplate, Selected) -> maps:fold( fun(KT, VT, Acc) -> - K = render(KT, Selected), + K = render_key(KT, Selected), case K =:= <<>> of true -> Acc; false -> - V = render(VT, Selected), + V = render_value(VT, Selected), Acc#{K => V} end end, diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl index 74affb9c8..acfe3df8b 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl @@ -64,7 +64,8 @@ single_config_tests() -> t_get_status_no_worker, t_get_status_timeout_calling_workers, t_on_start_ehttpc_pool_already_started, - t_attributes + t_attributes, + t_bad_attributes ]. only_sync_tests() -> @@ -484,9 +485,15 @@ assert_http_request(ServiceAccountJSON) -> error({timeout, #{mailbox => Mailbox}}) end. +receive_http_requests(ServiceAccountJSON, Opts) -> + Default = #{n => 1}, + #{n := N} = maps:merge(Default, Opts), + lists:flatmap(fun(_) -> receive_http_request(ServiceAccountJSON) end, lists:seq(1, N)). + receive_http_request(ServiceAccountJSON) -> receive {http, Headers, Body} -> + ct:pal("received publish:\n ~p", [#{headers => Headers, body => Body}]), assert_valid_request_headers(Headers, ServiceAccountJSON), #{<<"messages">> := Msgs} = emqx_utils_json:decode(Body, [return_maps]), lists:map( @@ -1689,3 +1696,109 @@ t_attributes(Config) -> [] ), ok. + +t_bad_attributes(Config) -> + ServiceAccountJSON = ?config(service_account_json, Config), + LocalTopic = <<"t/topic">>, + ?check_trace( + begin + {ok, _} = create_bridge_http( + Config, + #{ + <<"local_topic">> => LocalTopic, + <<"attributes_template">> => + [ + #{ + <<"key">> => <<"${.payload.key}">>, + <<"value">> => <<"${.payload.value}">> + } + ], + <<"ordering_key_template">> => <<"${.payload.ok}">> + } + ), + %% Ok: attribute value is a map or list + lists:foreach( + fun(OkValue) -> + Payload0 = + emqx_utils_json:encode( + #{ + <<"ok">> => <<"ord_key">>, + <<"value">> => OkValue, + <<"key">> => <<"attr_key">> + } + ), + Message0 = emqx_message:make(LocalTopic, Payload0), + emqx:publish(Message0) + end, + [ + #{<<"some">> => <<"map">>}, + [1, <<"str">>, #{<<"deep">> => true}] + ] + ), + DecodedMessages0 = receive_http_requests(ServiceAccountJSON, #{n => 1}), + ?assertMatch( + [ + #{ + <<"attributes">> := + #{<<"attr_key">> := <<"{\"some\":\"map\"}">>}, + <<"orderingKey">> := <<"ord_key">> + }, + #{ + <<"attributes">> := + #{<<"attr_key">> := <<"[1,\"str\",{\"deep\":true}]">>}, + <<"orderingKey">> := <<"ord_key">> + } + ], + DecodedMessages0 + ), + %% Bad: key is not a plain value + lists:foreach( + fun(BadKey) -> + Payload1 = + emqx_utils_json:encode( + #{ + <<"value">> => <<"v">>, + <<"key">> => BadKey, + <<"ok">> => BadKey + } + ), + Message1 = emqx_message:make(LocalTopic, Payload1), + emqx:publish(Message1) + end, + [ + #{<<"some">> => <<"map">>}, + [1, <<"list">>, true], + true, + false + ] + ), + DecodedMessages1 = receive_http_request(ServiceAccountJSON), + lists:foreach( + fun(DMsg) -> + ?assertNot(is_map_key(<<"orderingKey">>, DMsg), #{decoded_message => DMsg}), + ?assertNot(is_map_key(<<"attributes">>, DMsg), #{decoded_message => DMsg}), + ok + end, + DecodedMessages1 + ), + ok + end, + fun(Trace) -> + ct:pal("trace:\n ~p", [Trace]), + ?assertMatch( + [ + #{placeholder := [<<"payload">>, <<"ok">>], value := #{}}, + #{placeholder := [<<"payload">>, <<"key">>], value := #{}}, + #{placeholder := [<<"payload">>, <<"ok">>], value := [_ | _]}, + #{placeholder := [<<"payload">>, <<"key">>], value := [_ | _]}, + #{placeholder := [<<"payload">>, <<"ok">>], value := true}, + #{placeholder := [<<"payload">>, <<"key">>], value := true}, + #{placeholder := [<<"payload">>, <<"ok">>], value := false}, + #{placeholder := [<<"payload">>, <<"key">>], value := false} + ], + ?of_kind("gcp_pubsub_producer_bad_value_for_key", Trace) + ), + ok + end + ), + ok. From 8a6bb6f5f34b3656feee3fca0752523d93c9132d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 15 Aug 2023 13:47:23 -0300 Subject: [PATCH 15/37] ci: restore emqx app standalone tests Those were accidentally removed during a refactoring. They are needed because we provide the `emqx` application as a standalone dependency for plugins. --- .github/workflows/_pr_entrypoint.yaml | 9 +++ .github/workflows/_push-entrypoint.yaml | 9 +++ .github/workflows/run_emqx_app_tests.yaml | 67 +++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 .github/workflows/run_emqx_app_tests.yaml diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index ec2bbf2e1..81a4b7f71 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -146,6 +146,15 @@ jobs: path: ${{ matrix.profile }}.zip retention-days: 1 + run_emqx_app_tests: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_emqx_app_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + run_test_cases: needs: - sanity-checks diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 9ef517ab9..7ba8cbce7 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -158,6 +158,15 @@ jobs: path: ${{ matrix.profile }}.zip retention-days: 1 + run_emqx_app_tests: + needs: + - sanity-checks + - compile + uses: ./.github/workflows/run_emqx_app_tests.yaml + with: + runner: ${{ needs.sanity-checks.outputs.runner }} + builder: ${{ needs.sanity-checks.outputs.builder }} + run_test_cases: if: needs.prepare.outputs.release != 'true' needs: diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml new file mode 100644 index 000000000..13a8b42d6 --- /dev/null +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -0,0 +1,67 @@ +name: Check emqx app standalone + +# These tests are needed because we provide the `emqx` application as a standalone +# dependency for plugins. + +concurrency: + group: test-standalone-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + runner: + required: true + type: string + builder: + required: true + type: string + +env: + IS_CI: "yes" + +jobs: + run_emqx_app_tests: + runs-on: ${{ inputs.runner }} + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-ubuntu22.04" + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: run + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo "git diff base: $GITHUB_BASE_REF" + if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then + # base is a commit sha1 + compare_base="$GITHUB_BASE_REF" + else + repo="${GITHUB_REPOSITORY}" + git remote -v + remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')" + git fetch "$remote" "$GITHUB_BASE_REF" + compare_base="$remote/$GITHUB_BASE_REF" + fi + changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)" + if [ "$changed_files" = '' ]; then + echo "nothing changed in apps/emqx, ignored." + exit 0 + fi + make ensure-rebar3 + cp rebar3 apps/emqx/ + cd apps/emqx + ./rebar3 xref + ./rebar3 dialyzer + ./rebar3 eunit -v + ./rebar3 ct --name 'test@127.0.0.1' -v --readable=true + ./rebar3 proper -d test/props + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: logs-${{ matrix.runs-on }} + path: apps/emqx/_build/test/logs From 124c5d94e566ca32ad10ec224d209d4c5fb96075 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 15 Aug 2023 14:25:38 -0300 Subject: [PATCH 16/37] ci: fix variable usage Co-authored-by: Ivan Dyachkov --- .github/workflows/_push-entrypoint.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 7ba8cbce7..a59511259 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -160,7 +160,7 @@ jobs: run_emqx_app_tests: needs: - - sanity-checks + - prepare - compile uses: ./.github/workflows/run_emqx_app_tests.yaml with: diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 13a8b42d6..b5b821741 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -23,7 +23,7 @@ env: jobs: run_emqx_app_tests: runs-on: ${{ inputs.runner }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-ubuntu22.04" + container: ${{ inputs.builder }} defaults: run: @@ -63,5 +63,5 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.runs-on }} + name: logs-${{ inputs.runner }} path: apps/emqx/_build/test/logs From 5560e6ed4f6e0d1bfcffc7502e99f9c6d1ae9be1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 9 Aug 2023 19:21:51 +0400 Subject: [PATCH 17/37] fix(cthsuite): also prevent `emqx` from loading default conf --- apps/emqx/test/emqx_cth_suite.erl | 51 +++++++++++++++++++------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 1ae6ceded..9b3e58da4 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -55,6 +55,11 @@ -type config() :: #{atom() => scalar() | [scalar()] | config() | [config()]}. -type scalar() :: atom() | number() | string() | binary(). +-type hookfun(R) :: + fun(() -> R) + | fun((appname()) -> R) + | fun((appname(), appspec_opts()) -> R). + -type appspec_opts() :: #{ %% 1. Enable loading application config %% If not defined or set to `false`, this step will be skipped. @@ -70,19 +75,19 @@ %% 3. Perform anything right before starting the application %% If not defined or set to `false`, this step will be skipped. %% Merging amounts to redefining. - before_start => fun(() -> _) | fun((appname()) -> _) | false, + before_start => hookfun(_) | false, %% 4. Starting the application %% If not defined or set to `true`, `application:ensure_all_started/1` is used. %% If custom function is used, it should return list of all applications that were started. %% If set to `false`, application will not be started. %% Merging amounts to redefining. - start => fun(() -> {ok, [appname()]}) | fun((appname()) -> {ok, [appname()]}) | boolean(), + start => hookfun({ok, [appname()]}) | boolean(), %% 5. Perform anything right after starting the application %% If not defined or set to `false`, this step will be skipped. %% Merging amounts to redefining. - after_start => fun(() -> _) | fun((appname()) -> _) | false + after_start => hookfun(_) | false }. %% @doc Start applications with a clean slate. @@ -214,29 +219,30 @@ maybe_override_env(App, #{override_env := Env = [{_, _} | _]}) -> maybe_override_env(_App, #{}) -> ok. -maybe_before_start(App, #{before_start := Fun}) when is_function(Fun, 1) -> - Fun(App); -maybe_before_start(_App, #{before_start := Fun}) when is_function(Fun, 0) -> - Fun(); +maybe_before_start(App, #{before_start := Fun} = Opts) when is_function(Fun) -> + apply_hookfun(Fun, App, Opts); maybe_before_start(_App, #{}) -> ok. maybe_start(_App, #{start := false}) -> {ok, []}; -maybe_start(_App, #{start := Fun}) when is_function(Fun, 0) -> - Fun(); -maybe_start(App, #{start := Fun}) when is_function(Fun, 1) -> - Fun(App); +maybe_start(App, #{start := Fun} = Opts) when is_function(Fun) -> + apply_hookfun(Fun, App, Opts); maybe_start(App, #{}) -> application:ensure_all_started(App). -maybe_after_start(App, #{after_start := Fun}) when is_function(Fun, 1) -> - Fun(App); -maybe_after_start(_App, #{after_start := Fun}) when is_function(Fun, 0) -> - Fun(); +maybe_after_start(App, #{after_start := Fun} = Opts) when is_function(Fun) -> + apply_hookfun(Fun, App, Opts); maybe_after_start(_App, #{}) -> ok. +apply_hookfun(Fun, _App, _Opts) when is_function(Fun, 0) -> + Fun(); +apply_hookfun(Fun, App, _Opts) when is_function(Fun, 1) -> + Fun(App); +apply_hookfun(Fun, App, Opts) when is_function(Fun, 2) -> + Fun(App, Opts). + -spec merge_appspec(appspec_opts(), appspec_opts()) -> appspec_opts(). merge_appspec(Opts1, Opts2) -> @@ -270,7 +276,11 @@ default_appspec(ekka, _SuiteOpts) -> }; default_appspec(emqx, SuiteOpts) -> #{ - override_env => [{data_dir, maps:get(work_dir, SuiteOpts, "data")}] + override_env => [{data_dir, maps:get(work_dir, SuiteOpts, "data")}], + % NOTE + % We inform `emqx` of our config loader before starting it so that it won't + % overwrite everything with a default configuration. + before_start => fun inhibit_config_loader/2 }; default_appspec(emqx_authz, _SuiteOpts) -> #{ @@ -307,9 +317,7 @@ default_appspec(emqx_conf, SuiteOpts) -> % NOTE % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't % overwrite everything with a default configuration. - before_start => fun() -> - emqx_app:set_config_loader(?MODULE) - end + before_start => fun inhibit_config_loader/2 }; default_appspec(emqx_dashboard, _SuiteOpts) -> #{ @@ -329,6 +337,11 @@ start_ekka() -> ok = emqx_common_test_helpers:start_ekka(), {ok, [mnesia, ekka]}. +inhibit_config_loader(_App, #{config := Config}) when Config /= false -> + ok = emqx_app:set_config_loader(?MODULE); +inhibit_config_loader(_App, #{}) -> + ok. + %% -spec stop(_StartedApps :: [appname()]) -> From 0b21b87296b1ccdcf4ec57d01d5f0d2090dc8bcd Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 15 Aug 2023 21:29:42 +0400 Subject: [PATCH 18/37] fix(test): switch `emqx_flapping_SUITE` to cth tooling --- apps/emqx/test/emqx_flapping_SUITE.erl | 34 ++++++++++++-------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index 942a262a6..6204d9b6d 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -20,31 +20,27 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - %% update global default config - {ok, _} = emqx:update_config( - [flapping_detect], - #{ - <<"enable">> => true, - <<"max_count">> => 3, - % 0.1s - <<"window_time">> => <<"100ms">>, - %% 2s - <<"ban_time">> => <<"2s">> - } + Apps = emqx_cth_suite:start( + [ + {emqx, + "flapping_detect {" + "\n enable = true" + "\n max_count = 3" + "\n window_time = 100ms" + "\n ban_time = 2s" + "\n }"} + ], + #{work_dir => ?config(priv_dir, Config)} ), - Config. + [{suite_apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]), - %% Clean emqx_banned table - mria_mnesia:delete_schema(), - ok. +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)). t_detect_check(_) -> ClientInfo = #{ From 4151fe5045d55fc8519d1d338d08a5e80abe9afd Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 15 Aug 2023 14:33:39 -0300 Subject: [PATCH 19/37] ci(refactor): compute before/after references in parent workflow --- .github/workflows/_pr_entrypoint.yaml | 2 ++ .github/workflows/_push-entrypoint.yaml | 2 ++ .github/workflows/run_emqx_app_tests.yaml | 22 ++++++++++------------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 81a4b7f71..dce8a91eb 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -154,6 +154,8 @@ jobs: with: runner: ${{ needs.sanity-checks.outputs.runner }} builder: ${{ needs.sanity-checks.outputs.builder }} + before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + after_ref: ${{ github.sha }} run_test_cases: needs: diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index a59511259..bccb21c78 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -166,6 +166,8 @@ jobs: with: runner: ${{ needs.sanity-checks.outputs.runner }} builder: ${{ needs.sanity-checks.outputs.builder }} + before_ref: ${{ github.event.before }} + after_ref: ${{ github.sha }} run_test_cases: if: needs.prepare.outputs.release != 'true' diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index b5b821741..5479a86ab 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -16,6 +16,12 @@ on: builder: required: true type: string + before_ref: + required: true + type: string + after_ref: + required: true + type: string env: IS_CI: "yes" @@ -34,20 +40,12 @@ jobs: with: fetch-depth: 0 - name: run + env: + BEFORE_REF: ${{ inputs.before_ref }} + AFTER_REF: ${{ inputs.after_ref }} run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" - echo "git diff base: $GITHUB_BASE_REF" - if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then - # base is a commit sha1 - compare_base="$GITHUB_BASE_REF" - else - repo="${GITHUB_REPOSITORY}" - git remote -v - remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')" - git fetch "$remote" "$GITHUB_BASE_REF" - compare_base="$remote/$GITHUB_BASE_REF" - fi - changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)" + changed_files="$(git diff --name-only ${BEFORE_REF} ${AFTER_REF} apps/emqx)" if [ "$changed_files" = '' ]; then echo "nothing changed in apps/emqx, ignored." exit 0 From e91d22b0d555f550ea5c3aeb82243f1187a16cf7 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 15 Aug 2023 19:28:31 +0200 Subject: [PATCH 20/37] ci(docker): emqx-enterprise repo does not exist in ECR --- .github/workflows/_pr_entrypoint.yaml | 3 ++- .github/workflows/build_and_push_docker_images.yaml | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index ec2bbf2e1..d52001c3c 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -63,7 +63,8 @@ jobs: ./actionlint -color \ -shellcheck= \ -ignore 'label ".+" is unknown' \ - -ignore 'value "emqx-enterprise" in "exclude"' + -ignore 'value "emqx-enterprise" in "exclude"' \ + -ignore 'value "emqx-enterprise-elixir" in "exclude"' - name: Check line-break at EOF run: | ./scripts/check-nl-at-eof.sh diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index b0d6aa481..b2bfe735b 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -88,6 +88,11 @@ jobs: registry: - 'docker.io' - 'public.ecr.aws' + exclude: + - profile: emqx-enterprise + registry: 'public.ecr.aws' + - profile: emqx-enterprise-elixir + registry: 'public.ecr.aws' steps: - uses: actions/checkout@v3 From f00553a17407c24aa661dcfe2e2f24307efa0670 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 15 Aug 2023 19:29:09 +0200 Subject: [PATCH 21/37] ci(release): do not publish prerelease artifacts to emqx.io, packagecloud and helm repo --- .github/workflows/release.yaml | 3 ++- .github/workflows/upload-helm-charts.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1945caab0..ab145a764 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -59,7 +59,7 @@ jobs: with: asset_paths: '["packages/*"]' - name: update to emqx.io - if: startsWith(github.ref_name, 'v') && (github.event_name == 'release' || inputs.publish_release_artefacts) + if: startsWith(github.ref_name, 'v') && ((github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts) run: | set -eux curl -w %{http_code} \ @@ -70,6 +70,7 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: Push to packagecloud.io + if: (github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts env: PROFILE: ${{ steps.profile.outputs.profile }} VERSION: ${{ steps.profile.outputs.version }} diff --git a/.github/workflows/upload-helm-charts.yaml b/.github/workflows/upload-helm-charts.yaml index 139809441..593a78a7c 100644 --- a/.github/workflows/upload-helm-charts.yaml +++ b/.github/workflows/upload-helm-charts.yaml @@ -43,6 +43,7 @@ jobs: ;; esac - uses: emqx/push-helm-action@v1.1 + if: github.event_name == 'release' && !github.event.prerelease with: charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}" version: ${{ steps.profile.outputs.version }} From b4161009436c9ec010e33a0785cc405d00a913d9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 15 Aug 2023 14:40:19 -0300 Subject: [PATCH 22/37] ci: fix output source --- .github/workflows/_push-entrypoint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index bccb21c78..bc3bc486e 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -164,8 +164,8 @@ jobs: - compile uses: ./.github/workflows/run_emqx_app_tests.yaml with: - runner: ${{ needs.sanity-checks.outputs.runner }} - builder: ${{ needs.sanity-checks.outputs.builder }} + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} before_ref: ${{ github.event.before }} after_ref: ${{ github.sha }} From b08102269aad21955d00d9b181b1ef5bb36c9dcf Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 15 Aug 2023 18:56:01 +0800 Subject: [PATCH 23/37] refactor(calendar): refactor datetime-related code and remove redundant --- apps/emqx/src/emqx_alarm.erl | 2 +- apps/emqx/src/emqx_banned.erl | 2 +- apps/emqx/src/emqx_datetime.erl | 156 ---------- .../src/emqx_trace/emqx_trace_formatter.erl | 2 +- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage.erl | 2 +- .../src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 2 +- .../src/emqx_gateway_api_clients.erl | 20 +- apps/emqx_gateway/src/emqx_gateway_cli.erl | 4 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 7 +- .../test/emqx_sn_protocol_SUITE.erl | 4 +- apps/emqx_management/src/emqx_mgmt.erl | 2 +- .../src/emqx_mgmt_api_api_keys.erl | 4 +- .../src/emqx_mgmt_api_banned.erl | 4 +- .../src/emqx_mgmt_api_clients.erl | 16 +- .../src/emqx_mgmt_api_trace.erl | 12 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 4 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 2 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 2 +- apps/emqx_modules/src/emqx_delayed.erl | 7 +- apps/emqx_modules/src/emqx_topic_metrics.erl | 2 +- .../src/emqx_topic_metrics_api.erl | 4 +- apps/emqx_retainer/src/emqx_retainer_api.erl | 7 +- apps/emqx_rule_engine/src/emqx_rule_date.erl | 270 ------------------ .../src/emqx_rule_engine_api.erl | 2 +- .../src/emqx_rule_engine_cli.erl | 4 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 33 +-- .../src/emqx_utils_calendar.erl} | 199 +++++++++++-- 30 files changed, 249 insertions(+), 532 deletions(-) delete mode 100644 apps/emqx/src/emqx_datetime.erl delete mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl rename apps/{emqx/src/emqx_calendar.erl => emqx_utils/src/emqx_utils_calendar.erl} (75%) diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 056f36050..8c0c35334 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -213,7 +213,7 @@ format(Node, #deactivated_alarm{ to_rfc3339(Timestamp) -> %% rfc3339 accuracy to millisecond - list_to_binary(calendar:system_time_to_rfc3339(Timestamp div 1000, [{unit, millisecond}])). + emqx_utils_calendar:epoch_to_rfc3339(Timestamp div 1000). %%-------------------------------------------------------------------- %% gen_server callbacks diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index ddd491b7c..e246bb2c5 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -172,7 +172,7 @@ maybe_format_host({As, Who}) -> {As, Who}. to_rfc3339(Timestamp) -> - list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second). -spec create(emqx_types:banned() | map()) -> {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}. diff --git a/apps/emqx/src/emqx_datetime.erl b/apps/emqx/src/emqx_datetime.erl deleted file mode 100644 index 70e099af4..000000000 --- a/apps/emqx/src/emqx_datetime.erl +++ /dev/null @@ -1,156 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_datetime). - --include_lib("typerefl/include/types.hrl"). - -%% API --export([ - to_epoch_millisecond/1, - to_epoch_second/1, - human_readable_duration_string/1 -]). --export([ - epoch_to_rfc3339/1, - epoch_to_rfc3339/2 -]). - --reflect_type([ - epoch_millisecond/0, - epoch_second/0 -]). - --type epoch_second() :: non_neg_integer(). --type epoch_millisecond() :: non_neg_integer(). --typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}). --typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}). - -%% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl, -%% here minus SECONDS_PER_DAY to tolerate timezone time offset, -%% so the maximum date can reach 9999-12-31 which is ample. --define(MAXIMUM_EPOCH, 253402214400). --define(MAXIMUM_EPOCH_MILLI, 253402214400_000). - -to_epoch_second(DateTime) -> - to_epoch(DateTime, second). - -to_epoch_millisecond(DateTime) -> - to_epoch(DateTime, millisecond). - -to_epoch(DateTime, Unit) -> - try - case string:to_integer(DateTime) of - {Epoch, []} -> validate_epoch(Epoch, Unit); - _ -> {ok, calendar:rfc3339_to_system_time(DateTime, [{unit, Unit}])} - end - catch - error:_ -> - {error, bad_rfc3339_timestamp} - end. - -epoch_to_rfc3339(TimeStamp) -> - epoch_to_rfc3339(TimeStamp, millisecond). - -epoch_to_rfc3339(TimeStamp, Unit) when is_integer(TimeStamp) -> - list_to_binary(calendar:system_time_to_rfc3339(TimeStamp, [{unit, Unit}])). - --spec human_readable_duration_string(integer()) -> string(). -human_readable_duration_string(Milliseconds) -> - Seconds = Milliseconds div 1000, - {D, {H, M, S}} = calendar:seconds_to_daystime(Seconds), - L0 = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}], - L1 = lists:dropwhile(fun({K, _}) -> K =:= 0 end, L0), - L2 = lists:map(fun({Time, Unit}) -> [integer_to_list(Time), Unit] end, L1), - lists:flatten(lists:join(", ", L2)). - -validate_epoch(Epoch, _Unit) when Epoch < 0 -> - {error, bad_epoch}; -validate_epoch(Epoch, second) when Epoch =< ?MAXIMUM_EPOCH -> - {ok, Epoch}; -validate_epoch(Epoch, millisecond) when Epoch =< ?MAXIMUM_EPOCH_MILLI -> - {ok, Epoch}; -validate_epoch(_Epoch, _Unit) -> - {error, bad_epoch}. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --compile(nowarn_export_all). --compile(export_all). -roots() -> [bar]. - -fields(bar) -> - [ - {second, ?MODULE:epoch_second()}, - {millisecond, ?MODULE:epoch_millisecond()} - ]. - --define(FORMAT(_Sec_, _Ms_), - lists:flatten( - io_lib:format("bar={second=~w,millisecond=~w}", [_Sec_, _Ms_]) - ) -). - -epoch_ok_test() -> - BigStamp = 1 bsl 37, - Args = [ - {0, 0, 0, 0}, - {1, 1, 1, 1}, - {BigStamp, BigStamp * 1000, BigStamp, BigStamp * 1000}, - {"2022-01-01T08:00:00+08:00", "2022-01-01T08:00:00+08:00", 1640995200, 1640995200000} - ], - lists:foreach( - fun({Sec, Ms, EpochSec, EpochMs}) -> - check_ok(?FORMAT(Sec, Ms), EpochSec, EpochMs) - end, - Args - ), - ok. - -check_ok(Input, Sec, Ms) -> - {ok, Data} = hocon:binary(Input, #{}), - ?assertMatch( - #{bar := #{second := Sec, millisecond := Ms}}, - hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar]) - ), - ok. - -epoch_failed_test() -> - BigStamp = 1 bsl 38, - Args = [ - {-1, -1}, - {"1s", "1s"}, - {BigStamp, 0}, - {0, BigStamp * 1000}, - {"2022-13-13T08:00:00+08:00", "2022-13-13T08:00:00+08:00"} - ], - lists:foreach( - fun({Sec, Ms}) -> - check_failed(?FORMAT(Sec, Ms)) - end, - Args - ), - ok. - -check_failed(Input) -> - {ok, Data} = hocon:binary(Input, #{}), - ?assertException( - throw, - _, - hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar]) - ), - ok. - --endif. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 42623e91a..4f74b9983 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -28,7 +28,7 @@ format( #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg}, #{payload_encode := PEncode} ) -> - Time = calendar:system_time_to_rfc3339(erlang:system_time(microsecond), [{unit, microsecond}]), + Time = emqx_utils_calendar:now_to_rfc3339(microsecond), ClientId = to_iolist(maps:get(clientid, Meta, "")), Peername = maps:get(peername, Meta, ""), MetaBin = format_meta(Meta, PEncode), diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 34dfc09a7..41046907b 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -71,7 +71,7 @@ %% the resulting file is corrupted during transmission). size => _Bytes :: non_neg_integer(), checksum => checksum(), - expire_at := emqx_datetime:epoch_second(), + expire_at := emqx_utils_calendar:epoch_second(), %% TTL of individual segments %% Somewhat confusing that we won't know it on the nodes where the filemeta %% is missing. diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index c4877fc68..be99618ca 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -278,7 +278,7 @@ format_file_info( end. format_timestamp(Timestamp) -> - iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second). format_name(NameBin) when is_binary(NameBin) -> NameBin; diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 2d068466c..04fac3b38 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -68,7 +68,7 @@ transfer := emqx_ft:transfer(), name := file:name(), size := _Bytes :: non_neg_integer(), - timestamp := emqx_datetime:epoch_second(), + timestamp := emqx_utils_calendar:epoch_second(), uri => uri_string:uri_string(), meta => emqx_ft:filemeta() }. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index ac06ab957..844896a2f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -43,7 +43,7 @@ transfer := transfer(), name := file:name(), uri := uri_string:uri_string(), - timestamp := emqx_datetime:epoch_second(), + timestamp := emqx_utils_calendar:epoch_second(), size := _Bytes :: non_neg_integer(), filemeta => filemeta() }. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 1fd4d3a5d..5d0395989 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -76,7 +76,7 @@ % TODO naming -type filefrag(T) :: #{ path := file:name(), - timestamp := emqx_datetime:epoch_second(), + timestamp := emqx_utils_calendar:epoch_second(), size := _Bytes :: non_neg_integer(), fragment := T }. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 8cfcb70e6..b698446b9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -397,13 +397,13 @@ format_channel_info(WhichNode, {_, Infos, Stats} = R) -> {ip_address, {peername, ConnInfo, fun peer_to_binary_addr/1}}, {port, {peername, ConnInfo, fun peer_to_port/1}}, {is_bridge, ClientInfo, false}, - {connected_at, {connected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}}, - {disconnected_at, {disconnected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}}, + {connected_at, {connected_at, ConnInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}}, + {disconnected_at, {disconnected_at, ConnInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}}, {connected, {conn_state, Infos, fun conn_state_to_connected/1}}, {keepalive, ClientInfo, 0}, {clean_start, ConnInfo, true}, {expiry_interval, ConnInfo, 0}, - {created_at, {created_at, SessInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}}, + {created_at, {created_at, SessInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}}, {subscriptions_cnt, Stats, 0}, {subscriptions_max, Stats, infinity}, {inflight_cnt, Stats, 0}, @@ -640,28 +640,28 @@ params_client_searching_in_qs() -> )}, {gte_created_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), M#{ desc => ?DESC(param_gte_created_at) } )}, {lte_created_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), M#{ desc => ?DESC(param_lte_created_at) } )}, {gte_connected_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), M#{ desc => ?DESC(param_gte_connected_at) } )}, {lte_connected_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), M#{ desc => ?DESC(param_lte_connected_at) } @@ -888,12 +888,12 @@ common_client_props() -> )}, {connected_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), #{desc => ?DESC(connected_at)} )}, {disconnected_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), #{ desc => ?DESC(disconnected_at) } @@ -931,7 +931,7 @@ common_client_props() -> )}, {created_at, mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), #{desc => ?DESC(created_at)} )}, {subscriptions_cnt, diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index fb4261065..36d61e458 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -313,9 +313,9 @@ format_gateway( [ Name, Status, - emqx_gateway_utils:unix_ts_to_rfc3339(CreatedAt), + emqx_utils_calendar:epoch_to_rfc3339(CreatedAt), StopOrStart, - emqx_gateway_utils:unix_ts_to_rfc3339(Timestamp), + emqx_utils_calendar:epoch_to_rfc3339(Timestamp), Config ] ). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index eb4ce9fdf..10c71e3a7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -38,7 +38,6 @@ -export([ apply/2, parse_listenon/1, - unix_ts_to_rfc3339/1, unix_ts_to_rfc3339/2, listener_id/3, parse_listener_id/1, @@ -364,14 +363,10 @@ unix_ts_to_rfc3339(Key, Map) -> Map; Ts -> Map#{ - Key => - emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>) + Key => emqx_utils_calendar:epoch_to_rfc3339(Ts) } end. -unix_ts_to_rfc3339(Ts) -> - emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>). - -spec stringfy(term()) -> binary(). stringfy(T) when is_list(T); is_binary(T) -> iolist_to_binary(T); diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index c3fa89c70..a0afd90c1 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -2312,9 +2312,7 @@ t_socket_passvice(_) -> ok. t_clients_api(_) -> - TsNow = emqx_gateway_utils:unix_ts_to_rfc3339( - erlang:system_time(millisecond) - ), + TsNow = emqx_utils_calendar:now_to_rfc3339(millisecond), ClientId = <<"client_id_test1">>, {ok, Socket} = gen_udp:open(0, [binary]), send_connect_msg(Socket, ClientId), diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 059c323ff..f51c11301 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -230,7 +230,7 @@ broker_info() -> Info#{node => node(), otp_release => otp_rel(), node_status => 'running'}. convert_broker_info({uptime, Uptime}, M) -> - M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)}; + M#{uptime => emqx_utils_calendar:human_readable_duration_string(Uptime)}; convert_broker_info({K, V}, M) -> M#{K => iolist_to_binary(V)}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index 432734688..78bbef540 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -127,7 +127,7 @@ fields(app) -> )}, {expired_at, hoconsc:mk( - hoconsc:union([infinity, emqx_datetime:epoch_second()]), + hoconsc:union([infinity, emqx_utils_calendar:epoch_second()]), #{ desc => "No longer valid datetime", example => <<"2021-12-05T02:01:34.186Z">>, @@ -137,7 +137,7 @@ fields(app) -> )}, {created_at, hoconsc:mk( - emqx_datetime:epoch_second(), + emqx_utils_calendar:epoch_second(), #{ desc => "ApiKey create datetime", example => <<"2021-12-01T00:00:00.000Z">> diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 5a988cbfc..6c1d407b5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -147,13 +147,13 @@ fields(ban) -> example => <<"Too many requests">> })}, {at, - hoconsc:mk(emqx_datetime:epoch_second(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_second(), #{ desc => ?DESC(at), required => false, example => <<"2021-10-25T21:48:47+08:00">> })}, {until, - hoconsc:mk(emqx_datetime:epoch_second(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_second(), #{ desc => ?DESC(until), required => false, example => <<"2021-10-25T21:53:47+08:00">> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 2b47fdb11..18ac65ae6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -161,7 +161,7 @@ schema("/clients") -> desc => <<"Fuzzy search `username` as substring">> })}, {gte_created_at, - hoconsc:mk(emqx_datetime:epoch_millisecond(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_millisecond(), #{ in => query, required => false, desc => @@ -169,7 +169,7 @@ schema("/clients") -> " than or equal method, rfc3339 or timestamp(millisecond)">> })}, {lte_created_at, - hoconsc:mk(emqx_datetime:epoch_millisecond(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_millisecond(), #{ in => query, required => false, desc => @@ -177,7 +177,7 @@ schema("/clients") -> " than or equal method, rfc3339 or timestamp(millisecond)">> })}, {gte_connected_at, - hoconsc:mk(emqx_datetime:epoch_millisecond(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_millisecond(), #{ in => query, required => false, desc => << @@ -186,7 +186,7 @@ schema("/clients") -> >> })}, {lte_connected_at, - hoconsc:mk(emqx_datetime:epoch_millisecond(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_millisecond(), #{ in => query, required => false, desc => << @@ -399,16 +399,16 @@ fields(client) -> {connected, hoconsc:mk(boolean(), #{desc => <<"Whether the client is connected">>})}, {connected_at, hoconsc:mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), #{desc => <<"Client connection time, rfc3339 or timestamp(millisecond)">>} )}, {created_at, hoconsc:mk( - emqx_datetime:epoch_millisecond(), + emqx_utils_calendar:epoch_millisecond(), #{desc => <<"Session creation time, rfc3339 or timestamp(millisecond)">>} )}, {disconnected_at, - hoconsc:mk(emqx_datetime:epoch_millisecond(), #{ + hoconsc:mk(emqx_utils_calendar:epoch_millisecond(), #{ desc => << "Client offline time." @@ -950,7 +950,7 @@ result_format_time_fun(Key, NClientInfoMap) -> case NClientInfoMap of #{Key := TimeStamp} -> NClientInfoMap#{ - Key => emqx_datetime:epoch_to_rfc3339(TimeStamp) + Key => emqx_utils_calendar:epoch_to_rfc3339(TimeStamp) }; #{} -> NClientInfoMap diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 27789fff9..17adf7460 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -281,7 +281,7 @@ fields(trace) -> })}, {start_at, hoconsc:mk( - emqx_datetime:epoch_second(), + emqx_utils_calendar:epoch_second(), #{ description => ?DESC(time_format), required => false, @@ -290,7 +290,7 @@ fields(trace) -> )}, {end_at, hoconsc:mk( - emqx_datetime:epoch_second(), + emqx_utils_calendar:epoch_second(), #{ description => ?DESC(time_format), required => false, @@ -410,8 +410,8 @@ trace(get, _Params) -> Trace0#{ log_size => LogSize, Type => iolist_to_binary(Filter), - start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)), - end_at => list_to_binary(calendar:system_time_to_rfc3339(End)), + start_at => emqx_utils_calendar:epoch_to_rfc3339(Start, second), + end_at => emqx_utils_calendar:epoch_to_rfc3339(End, second), status => status(Enable, Start, End, Now) } end, @@ -468,8 +468,8 @@ format_trace(Trace0) -> Trace2#{ log_size => LogSize, Type => iolist_to_binary(Filter), - start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)), - end_at => list_to_binary(calendar:system_time_to_rfc3339(End)), + start_at => emqx_utils_calendar:epoch_to_rfc3339(Start, second), + end_at => emqx_utils_calendar:epoch_to_rfc3339(Start, second), status => status(Enable, Start, End, Now) }. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 4fe47cf93..ace4c155a 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -142,11 +142,11 @@ format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) -> ExpiredAt = case ExpiredAt0 of infinity -> <<"infinity">>; - _ -> list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt0)) + _ -> emqx_utils_calendar:epoch_to_rfc3339(ExpiredAt0, second) end, App#{ expired_at => ExpiredAt, - created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt)) + created_at => emqx_utils_calendar:epoch_to_rfc3339(CreateAt, second) }. list() -> diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 9692441a6..aeed5b922 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -87,7 +87,7 @@ broker([]) -> Funs = [sysdescr, version, datetime], [emqx_ctl:print("~-10s: ~ts~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs], emqx_ctl:print("~-10s: ~ts~n", [ - uptime, emqx_datetime:human_readable_duration_string(emqx_sys:uptime()) + uptime, emqx_utils_calendar:human_readable_duration_string(emqx_sys:uptime()) ]); broker(["stats"]) -> [ diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index efdaa9c96..f428009cb 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -260,7 +260,7 @@ t_query_clients_with_time(_) -> %% Do not uri_encode `=` to `%3D` Rfc3339String = emqx_http_lib:uri_encode( binary:bin_to_list( - emqx_datetime:epoch_to_rfc3339(NowTimeStampInt) + emqx_utils_calendar:epoch_to_rfc3339(NowTimeStampInt) ) ), TimeStampString = emqx_http_lib:uri_encode(integer_to_list(NowTimeStampInt)), diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 32219a139..559648bdd 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -208,8 +208,8 @@ format_delayed( }, WithPayload ) -> - PublishTime = to_rfc3339(PublishTimeStamp div 1000), - ExpectTime = to_rfc3339(ExpectTimeStamp div 1000), + PublishTime = emqx_utils_calendar:epoch_to_rfc3339(PublishTimeStamp), + ExpectTime = emqx_utils_calendar:epoch_to_rfc3339(ExpectTimeStamp), RemainingTime = ExpectTimeStamp - ?NOW, Result = #{ msgid => emqx_guid:to_hexstr(Id), @@ -230,9 +230,6 @@ format_delayed( Result end. -to_rfc3339(Timestamp) -> - list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). - -spec get_delayed_message(binary()) -> with_id_return(map()). get_delayed_message(Id) -> case ets:select(?TAB, ?QUERY_MS(Id)) of diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index efe309b9e..987b0b69b 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -295,7 +295,7 @@ terminate(_Reason, _State) -> reset_topic({Topic, Data}, Speeds) -> CRef = maps:get(counter_ref, Data), ok = reset_counter(CRef), - ResetTime = emqx_rule_funcs:now_rfc3339(), + ResetTime = emqx_utils_calendar:now_to_rfc3339(), true = ets:insert(?TAB, {Topic, Data#{reset_time => ResetTime}}), Fun = fun(Metric, CurrentSpeeds) -> diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index 50b586228..49b3071e0 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -183,7 +183,7 @@ fields(topic_metrics) -> )}, {create_time, mk( - emqx_datetime:epoch_second(), + emqx_utils_calendar:epoch_second(), #{ desc => ?DESC(create_time), required => true, @@ -192,7 +192,7 @@ fields(topic_metrics) -> )}, {reset_time, mk( - emqx_datetime:epoch_second(), + emqx_utils_calendar:epoch_second(), #{ desc => ?DESC(reset_time), required => false, diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 3274f0e4c..446679325 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -211,11 +211,8 @@ format_message(#message{ msgid => emqx_guid:to_hexstr(ID), qos => Qos, topic => Topic, - publish_at => list_to_binary( - calendar:system_time_to_rfc3339( - Timestamp, [{unit, millisecond}] - ) - ), + publish_at => + emqx_utils_calendar:epoch_to_rfc3339(Timestamp), from_clientid => to_bin_string(From), from_username => maps:get(username, Headers, <<>>) }. diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl deleted file mode 100644 index aeb5d7a1b..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_date.erl +++ /dev/null @@ -1,270 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_date). - --export([date/3, date/4, parse_date/4]). - --export([ - is_int_char/1, - is_symbol_char/1, - is_m_char/1 -]). - --record(result, { - %%year() - year = "1970" :: string(), - %%month() - month = "1" :: string(), - %%day() - day = "1" :: string(), - %%hour() - hour = "0" :: string(), - %%minute() %% epoch in millisecond precision - minute = "0" :: string(), - %%second() %% epoch in millisecond precision - second = "0" :: string(), - %%integer() %% zone maybe some value - zone = "+00:00" :: string() -}). - -%% -type time_unit() :: 'microsecond' -%% | 'millisecond' -%% | 'nanosecond' -%% | 'second'. -%% -type offset() :: [byte()] | (Time :: integer()). -date(TimeUnit, Offset, FormatString) -> - date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). - -date(TimeUnit, Offset, FormatString, TimeEpoch) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - Res = lists:map( - fun(Expr) -> - eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) - end, - R - ), - lists:concat(Res). - -parse_date(TimeUnit, Offset, FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - IsZ = fun(V) -> - case V of - {tag, $Z} -> true; - _ -> false - end - end, - R1 = lists:filter(IsZ, R), - IfFun = fun(Con, A, B) -> - case Con of - [] -> A; - _ -> B - end - end, - Res = parse_input(FormatString, InputString), - Str = - Res#result.year ++ "-" ++ - Res#result.month ++ "-" ++ - Res#result.day ++ "T" ++ - Res#result.hour ++ ":" ++ - Res#result.minute ++ ":" ++ - Res#result.second ++ - IfFun(R1, Offset, Res#result.zone), - calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). - -mlist(R) -> - %% %H Shows hour in 24-hour format [15] - [ - {$H, R#result.hour}, - %% %M Displays minutes [00-59] - {$M, R#result.minute}, - %% %S Displays seconds [00-59] - {$S, R#result.second}, - %% %y Displays year YYYY [2021] - {$y, R#result.year}, - %% %m Displays the number of the month [01-12] - {$m, R#result.month}, - %% %d Displays the number of the month [01-12] - {$d, R#result.day}, - %% %Z Displays Time zone - {$Z, R#result.zone} - ]. - -rmap(Result) -> - maps:from_list(mlist(Result)). - -support_char() -> "HMSymdZ". - -create_tag(Head, []) -> - Head; -create_tag(Head, [Val1 | RVal]) -> - case Val1 of - [] -> - create_tag(Head ++ [{st, [$%]}], RVal); - [H | Other] -> - case lists:member(H, support_char()) of - true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); - false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) - end - end. - -eval_tag(_, {st, Str}) -> - Str; -eval_tag(Map, {tag, Char}) -> - maps:get(Char, Map, "undefined"). - -%% make_time(TimeUnit, Offset) -> -%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). -make_time(TimeUnit, Offset, TimeEpoch) -> - Res = calendar:system_time_to_rfc3339( - TimeEpoch, - [{unit, TimeUnit}, {offset, Offset}] - ), - [ - Y1, - Y2, - Y3, - Y4, - $-, - Mon1, - Mon2, - $-, - D1, - D2, - _T, - H1, - H2, - $:, - Min1, - Min2, - $:, - S1, - S2 - | TimeStr - ] = Res, - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), - #result{ - year = [Y1, Y2, Y3, Y4], - month = [Mon1, Mon2], - day = [D1, D2], - hour = [H1, H2], - minute = [Min1, Min2], - second = [S1, S2] ++ FractionStr, - zone = UtcOffset - }. - -is_int_char(C) -> - C >= $0 andalso C =< $9. -is_symbol_char(C) -> - C =:= $- orelse C =:= $+. -is_m_char(C) -> - C =:= $:. - -parse_char_with_fun(_, []) -> - error(null_input); -parse_char_with_fun(ValidFun, [C | Other]) -> - Res = - case erlang:is_function(ValidFun) of - true -> ValidFun(C); - false -> erlang:apply(emqx_rule_date, ValidFun, [C]) - end, - case Res of - true -> {C, Other}; - false -> error({unexpected, [C | Other]}) - end. -parse_string([], Input) -> - {[], Input}; -parse_string([C | Other], Input) -> - {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), - {Res, Input2} = parse_string(Other, Input1), - {[C1 | Res], Input2}. - -parse_times(0, _, Input) -> - {[], Input}; -parse_times(Times, Fun, Input) -> - {C1, Input1} = parse_char_with_fun(Fun, Input), - {Res, Input2} = parse_times((Times - 1), Fun, Input1), - {[C1 | Res], Input2}. - -parse_int_times(Times, Input) -> - parse_times(Times, is_int_char, Input). - -parse_fraction(Input) -> - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - lists:splitwith(IsFractionChar, Input). - -parse_second(Input) -> - {M, Input1} = parse_int_times(2, Input), - {M1, Input2} = parse_fraction(Input1), - {M ++ M1, Input2}. - -parse_zone(Input) -> - {S, Input1} = parse_char_with_fun(is_symbol_char, Input), - {M, Input2} = parse_int_times(2, Input1), - {C, Input3} = parse_char_with_fun(is_m_char, Input2), - {V, Input4} = parse_int_times(2, Input3), - {[S | M ++ [C | V]], Input4}. - -mlist1() -> - maps:from_list( - %% %H Shows hour in 24-hour format [15] - [ - {$H, fun(Input) -> parse_int_times(2, Input) end}, - %% %M Displays minutes [00-59] - {$M, fun(Input) -> parse_int_times(2, Input) end}, - %% %S Displays seconds [00-59] - {$S, fun(Input) -> parse_second(Input) end}, - %% %y Displays year YYYY [2021] - {$y, fun(Input) -> parse_int_times(4, Input) end}, - %% %m Displays the number of the month [01-12] - {$m, fun(Input) -> parse_int_times(2, Input) end}, - %% %d Displays the number of the month [01-12] - {$d, fun(Input) -> parse_int_times(2, Input) end}, - %% %Z Displays Time zone - {$Z, fun(Input) -> parse_zone(Input) end} - ] - ). - -update_result($H, Res, Str) -> Res#result{hour = Str}; -update_result($M, Res, Str) -> Res#result{minute = Str}; -update_result($S, Res, Str) -> Res#result{second = Str}; -update_result($y, Res, Str) -> Res#result{year = Str}; -update_result($m, Res, Str) -> Res#result{month = Str}; -update_result($d, Res, Str) -> Res#result{day = Str}; -update_result($Z, Res, Str) -> Res#result{zone = Str}. - -parse_tag(Res, {st, St}, InputString) -> - {_A, B} = parse_string(St, InputString), - {Res, B}; -parse_tag(Res, {tag, St}, InputString) -> - Fun = maps:get(St, mlist1()), - {A, B} = Fun(InputString), - NRes = update_result(St, Res, A), - {NRes, B}. - -parse_tags(Res, [], _) -> - Res; -parse_tags(Res, [Tag | Others], InputString) -> - {NRes, B} = parse_tag(Res, Tag, InputString), - parse_tags(NRes, Others, B). - -parse_input(FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - parse_tags(#result{}, R, InputString). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 2e6952920..79be197aa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -514,7 +514,7 @@ format_rule_engine_resp(Config) -> maps:remove(rules, Config). format_datetime(Timestamp, Unit) -> - list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). + emqx_utils_calendar:epoch_to_rfc3339(Timestamp, Unit). format_action(Actions) -> [do_format_action(Act) || Act <- Actions]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index 7f4e06252..037a3c2ee 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -74,8 +74,8 @@ pretty_print_rule(ID) -> "Updated at:\n ~ts\n" "Actions:\n ~s\n" ,[Id, Name, left_pad(Descr), Enable, left_pad(SQL), - calendar:system_time_to_rfc3339(CreatedAt, [{unit, millisecond}]), - calendar:system_time_to_rfc3339(UpdatedAt, [{unit, millisecond}]), + emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, second), + emqx_utils_calendar:epoch_to_rfc3339(UpdatedAt, second), [left_pad(format_action(A)) || A <- Actions] ] ); diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 64522ee60..038edea48 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -276,6 +276,8 @@ ]} ). +-import(emqx_utils_calendar, [time_unit/1, now_to_rfc3339/0, now_to_rfc3339/1, epoch_to_rfc3339/2]). + %% @doc "msgid()" Func msgid() -> fun @@ -1077,23 +1079,19 @@ kv_store_del(Key) -> %%-------------------------------------------------------------------- now_rfc3339() -> - now_rfc3339(<<"second">>). + now_to_rfc3339(). now_rfc3339(Unit) -> - unix_ts_to_rfc3339(now_timestamp(Unit), Unit). + now_to_rfc3339(time_unit(Unit)). unix_ts_to_rfc3339(Epoch) -> - unix_ts_to_rfc3339(Epoch, <<"second">>). + epoch_to_rfc3339(Epoch, second). unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) -> - emqx_utils_conv:bin( - calendar:system_time_to_rfc3339( - Epoch, [{unit, time_unit(Unit)}] - ) - ). + epoch_to_rfc3339(Epoch, time_unit(Unit)). rfc3339_to_unix_ts(DateTime) -> - rfc3339_to_unix_ts(DateTime, <<"second">>). + rfc3339_to_unix_ts(DateTime, second). rfc3339_to_unix_ts(DateTime, Unit) when is_binary(DateTime) -> calendar:rfc3339_to_system_time( @@ -1107,15 +1105,6 @@ now_timestamp() -> now_timestamp(Unit) -> erlang:system_time(time_unit(Unit)). -time_unit(<<"second">>) -> second; -time_unit(<<"millisecond">>) -> millisecond; -time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond; -time_unit(second) -> second; -time_unit(millisecond) -> millisecond; -time_unit(microsecond) -> microsecond; -time_unit(nanosecond) -> nanosecond. - format_date(TimeUnit, Offset, FormatString) -> Unit = time_unit(TimeUnit), TimeEpoch = erlang:system_time(Unit), @@ -1125,17 +1114,17 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> Unit = time_unit(TimeUnit), emqx_utils_conv:bin( lists:concat( - emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString) + emqx_utils_calendar:format(TimeEpoch, Unit, Offset, FormatString) ) ). date_to_unix_ts(TimeUnit, FormatString, InputString) -> Unit = time_unit(TimeUnit), - emqx_calendar:parse(InputString, Unit, FormatString). + emqx_utils_calendar:parse(InputString, Unit, FormatString). date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> Unit = time_unit(TimeUnit), - OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetSecond = emqx_utils_calendar:offset_second(Offset), OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. @@ -1143,7 +1132,7 @@ timezone_to_second(TimeZone) -> timezone_to_offset_seconds(TimeZone). timezone_to_offset_seconds(TimeZone) -> - emqx_calendar:offset_second(TimeZone). + emqx_utils_calendar:offset_second(TimeZone). '$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); diff --git a/apps/emqx/src/emqx_calendar.erl b/apps/emqx_utils/src/emqx_utils_calendar.erl similarity index 75% rename from apps/emqx/src/emqx_calendar.erl rename to apps/emqx_utils/src/emqx_utils_calendar.erl index 8a424ac2b..a42b8d0ca 100644 --- a/apps/emqx/src/emqx_calendar.erl +++ b/apps/emqx_utils/src/emqx_utils_calendar.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,7 +14,32 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_calendar). +-module(emqx_utils_calendar). + +-include_lib("typerefl/include/types.hrl"). + +-export([ + formatter/1, + format/3, + format/4, + parse/3, + offset_second/1 +]). + +%% API +-export([ + to_epoch_millisecond/1, + to_epoch_second/1, + human_readable_duration_string/1 +]). +-export([ + epoch_to_rfc3339/1, + epoch_to_rfc3339/2, + now_to_rfc3339/0, + now_to_rfc3339/1 +]). + +-export([time_unit/1]). -define(SECONDS_PER_MINUTE, 60). -define(SECONDS_PER_HOUR, 3600). @@ -24,13 +49,11 @@ -define(DAYS_FROM_0_TO_1970, 719528). -define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY)). --export([ - formatter/1, - format/3, - format/4, - parse/3, - offset_second/1 -]). +%% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl, +%% here minus SECONDS_PER_DAY to tolerate timezone time offset, +%% so the maximum date can reach 9999-12-31 which is ample. +-define(MAXIMUM_EPOCH, 253402214400). +-define(MAXIMUM_EPOCH_MILLI, 253402214400_000). -define(DATE_PART, [ year, @@ -50,6 +73,72 @@ timezone2 ]). +-reflect_type([ + epoch_millisecond/0, + epoch_second/0 +]). + +-type epoch_second() :: non_neg_integer(). +-type epoch_millisecond() :: non_neg_integer(). +-typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}). +-typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}). + +%%-------------------------------------------------------------------- +%% Epoch <-> RFC 3339 +%%-------------------------------------------------------------------- + +to_epoch_second(DateTime) -> + to_epoch(DateTime, second). + +to_epoch_millisecond(DateTime) -> + to_epoch(DateTime, millisecond). + +to_epoch(DateTime, Unit) -> + try + case string:to_integer(DateTime) of + {Epoch, []} -> validate_epoch(Epoch, Unit); + _ -> {ok, calendar:rfc3339_to_system_time(DateTime, [{unit, Unit}])} + end + catch + error:_ -> + {error, bad_rfc3339_timestamp} + end. + +epoch_to_rfc3339(Timestamp) -> + epoch_to_rfc3339(Timestamp, millisecond). + +epoch_to_rfc3339(Timestamp, Unit) when is_integer(Timestamp) -> + list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). + +now_to_rfc3339() -> + now_to_rfc3339(second). + +now_to_rfc3339(Unit) -> + epoch_to_rfc3339(erlang:system_time(Unit), Unit). + +-spec human_readable_duration_string(integer()) -> string(). +human_readable_duration_string(Milliseconds) -> + Seconds = Milliseconds div 1000, + {D, {H, M, S}} = calendar:seconds_to_daystime(Seconds), + L0 = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}], + L1 = lists:dropwhile(fun({K, _}) -> K =:= 0 end, L0), + L2 = lists:map(fun({Time, Unit}) -> [integer_to_list(Time), Unit] end, L1), + lists:flatten(lists:join(", ", L2)). + +validate_epoch(Epoch, _Unit) when Epoch < 0 -> + {error, bad_epoch}; +validate_epoch(Epoch, second) when Epoch =< ?MAXIMUM_EPOCH -> + {ok, Epoch}; +validate_epoch(Epoch, millisecond) when Epoch =< ?MAXIMUM_EPOCH_MILLI -> + {ok, Epoch}; +validate_epoch(_Epoch, _Unit) -> + {error, bad_epoch}. + +%%-------------------------------------------------------------------- +%% Timestamp <-> any format date string +%% Timestamp treat as a superset for epoch, it can be any positive integer +%%-------------------------------------------------------------------- + formatter(FormatterStr) when is_list(FormatterStr) -> formatter(list_to_binary(FormatterStr)); formatter(FormatterBin) when is_binary(FormatterBin) -> @@ -70,8 +159,10 @@ parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> parse(DateStr, Unit, formatter(FormatterBin)); parse(DateStr, Unit, Formatter) -> do_parse(DateStr, Unit, Formatter). -%% ------------------------------------------------------------------------------------------------- -%% internal + +%%-------------------------------------------------------------------- +%% Time unit +%%-------------------------------------------------------------------- time_unit(second) -> second; time_unit(millisecond) -> millisecond; @@ -84,10 +175,12 @@ time_unit("nanosecond") -> nanosecond; time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(Any) -> error({invalid_time_unit, Any}). -%% ------------------------------------------------------------------------------------------------- +%%-------------------------------------------------------------------- %% internal: format part +%%-------------------------------------------------------------------- do_formatter(<<>>, Formatter) -> lists:reverse(Formatter); @@ -357,9 +450,9 @@ padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> padding(Data, _Len) -> Data. -%% ------------------------------------------------------------------------------------------------- -%% internal -%% parse part +%%-------------------------------------------------------------------- +%% internal: parse part +%%-------------------------------------------------------------------- do_parse(DateStr, Unit, Formatter) -> DateInfo = do_parse_date_str(DateStr, Formatter, #{}), @@ -476,3 +569,77 @@ str_to_int_or_error(Str, Error) -> _ -> error(Error) end. + +%%-------------------------------------------------------------------- +%% Unit Test +%%-------------------------------------------------------------------- + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-compile(nowarn_export_all). +-compile(export_all). +roots() -> [bar]. + +fields(bar) -> + [ + {second, ?MODULE:epoch_second()}, + {millisecond, ?MODULE:epoch_millisecond()} + ]. + +-define(FORMAT(_Sec_, _Ms_), + lists:flatten( + io_lib:format("bar={second=~w,millisecond=~w}", [_Sec_, _Ms_]) + ) +). + +epoch_ok_test() -> + BigStamp = 1 bsl 37, + Args = [ + {0, 0, 0, 0}, + {1, 1, 1, 1}, + {BigStamp, BigStamp * 1000, BigStamp, BigStamp * 1000}, + {"2022-01-01T08:00:00+08:00", "2022-01-01T08:00:00+08:00", 1640995200, 1640995200000} + ], + lists:foreach( + fun({Sec, Ms, EpochSec, EpochMs}) -> + check_ok(?FORMAT(Sec, Ms), EpochSec, EpochMs) + end, + Args + ), + ok. + +check_ok(Input, Sec, Ms) -> + {ok, Data} = hocon:binary(Input, #{}), + ?assertMatch( + #{bar := #{second := Sec, millisecond := Ms}}, + hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar]) + ), + ok. + +epoch_failed_test() -> + BigStamp = 1 bsl 38, + Args = [ + {-1, -1}, + {"1s", "1s"}, + {BigStamp, 0}, + {0, BigStamp * 1000}, + {"2022-13-13T08:00:00+08:00", "2022-13-13T08:00:00+08:00"} + ], + lists:foreach( + fun({Sec, Ms}) -> + check_failed(?FORMAT(Sec, Ms)) + end, + Args + ), + ok. + +check_failed(Input) -> + {ok, Data} = hocon:binary(Input, #{}), + ?assertException( + throw, + _, + hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar]) + ), + ok. + +-endif. From 8cd21da94b9d6941b55c3f68af1c375457a66260 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 15 Aug 2023 19:04:54 +0800 Subject: [PATCH 24/37] chore: update apps version --- apps/emqx_ft/src/emqx_ft.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_utils/src/emqx_utils.app.src | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index ac498d6c6..8518958e0 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index b5fe5e100..582269ce6 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index b7a9d7f4d..cd2f6c8b9 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.19"}, + {vsn, "5.0.20"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 09d57a4f9..e6d00bcae 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index f8905b513..539bfd3b7 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, [ emqx_utils, emqx_utils_api, From 0965387e46e5b75ca3fdf296d406b534009d8b3e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 16 Aug 2023 11:13:16 +0800 Subject: [PATCH 25/37] chore: update changes --- changes/ce/feat-11446.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-11446.en.md diff --git a/changes/ce/feat-11446.en.md b/changes/ce/feat-11446.en.md new file mode 100644 index 000000000..aa420136c --- /dev/null +++ b/changes/ce/feat-11446.en.md @@ -0,0 +1 @@ +Refactored datetime-related modules and functions to simplify the code. From 1af0407c4843e9d96cb4aa1f36d83274ce336e22 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 15:26:06 +0800 Subject: [PATCH 26/37] fix: don't crash when debug huge payload --- apps/emqx/include/emqx_mqtt.hrl | 1 + apps/emqx/src/emqx_packet.erl | 28 +++++++--- .../src/emqx_trace/emqx_trace_formatter.erl | 9 +--- apps/emqx/test/emqx_trace_SUITE.erl | 54 +++++++++++++++++++ 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index d8922fb98..4d0188f71 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -680,6 +680,7 @@ end). -define(THROW_SERIALIZE_ERROR(Reason), erlang:throw({?FRAME_SERIALIZE_ERROR, Reason})). -define(MAX_PAYLOAD_FORMAT_SIZE, 1024). +-define(TRUNCATED_PAYLOAD_SIZE, 100). -define(MAX_PAYLOAD_FORMAT_LIMIT(Bin), (byte_size(Bin) =< ?MAX_PAYLOAD_FORMAT_SIZE)). -endif. diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 1f7aaa4c9..9cb23be2e 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -55,6 +55,8 @@ format/2 ]). +-export([format_truncated_payload/3]). + -define(TYPE_NAMES, {'CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL', 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK', 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH'} @@ -614,21 +616,33 @@ format_password(undefined) -> ""; format_password(<<>>) -> ""; format_password(_Password) -> "******". +format_payload(_, hidden) -> + "Payload=******"; format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> ["Payload=", unicode:characters_to_list(Payload)]; format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> ["Payload(hex)=", binary:encode_hex(Payload)]; -format_payload(_, hidden) -> - "Payload=******"; -format_payload(<> = Payload, _) -> +format_payload(<> = Payload, Type) -> [ "Payload=", - Part, - "... The ", - integer_to_list(byte_size(Payload) - 100), - " bytes of this log are truncated" + format_truncated_payload(Part, byte_size(Payload), Type) ]. +format_truncated_payload(Bin, Size, Type) -> + Bin2 = + case Type of + text -> Bin; + hex -> binary:encode_hex(Bin) + end, + unicode:characters_to_list( + [ + Bin2, + "... The ", + integer_to_list(Size - ?TRUNCATED_PAYLOAD_SIZE), + " bytes of this log are truncated" + ] + ). + i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 42623e91a..be3d858f5 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -76,13 +76,8 @@ format_payload(_, hidden) -> format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> unicode:characters_to_list(Payload); format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> binary:encode_hex(Payload); -format_payload(<> = Payload, _) -> - [ - Part, - "... The ", - integer_to_list(byte_size(Payload) - 100), - " bytes of this log are truncated" - ]. +format_payload(<> = Payload, Type) -> + emqx_packet:format_truncated_payload(Part, byte_size(Payload), Type). to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index 1bbe084fd..ce7d7e887 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -311,6 +311,60 @@ t_client_event(_Config) -> ?assert(erlang:byte_size(Bin3) > 0), ok. +t_client_huge_payload_truncated(_Config) -> + ClientId = <<"client-truncated1">>, + Now = erlang:system_time(second), + Name = <<"test_client_id_truncated1">>, + {ok, _} = emqx_trace:create([ + {<<"name">>, Name}, + {<<"type">>, clientid}, + {<<"clientid">>, ClientId}, + {<<"start_at">>, Now} + ]), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + NormalPayload = iolist_to_binary(lists:duplicate(1024, "x")), + ok = emqtt:publish(Client, <<"/test">>, #{}, NormalPayload, [{qos, 0}]), + HugePayload1 = iolist_to_binary(lists:duplicate(1025, "y")), + ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload1, [{qos, 0}]), + HugePayload2 = iolist_to_binary(lists:duplicate(1024 * 10, "y")), + ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload2, [{qos, 0}]), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + {ok, _} = emqx_trace:create([ + {<<"name">>, <<"test_topic">>}, + {<<"type">>, topic}, + {<<"topic">>, <<"/test">>}, + {<<"start_at">>, Now} + ]), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), + {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, NormalPayload, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload1, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload2, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), + {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 1024), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 1024), + + %% Don't have format crash + CrashBin = <<"CRASH">>, + ?assertEqual(nomatch, binary:match(Bin, [CrashBin])), + ?assertEqual(nomatch, binary:match(Bin2, [CrashBin])), + ?assertEqual(nomatch, binary:match(Bin3, [CrashBin])), + %% have "this log are truncated" for huge payload + TruncatedLog = <<"this log are truncated">>, + ?assertNotEqual(nomatch, binary:match(Bin, [TruncatedLog])), + ?assertNotEqual(nomatch, binary:match(Bin2, [TruncatedLog])), + ?assertNotEqual(nomatch, binary:match(Bin3, [TruncatedLog])), + ok. + t_get_log_filename(_Config) -> Now = erlang:system_time(second), Name = <<"name1">>, From e2681f46f63a94d354fd10639bd68fbef25d50f3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 15:45:36 +0800 Subject: [PATCH 27/37] chore: add changelog for 11454 --- changes/ce/fix-11454.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11454.en.md diff --git a/changes/ce/fix-11454.en.md b/changes/ce/fix-11454.en.md new file mode 100644 index 000000000..50e7fe826 --- /dev/null +++ b/changes/ce/fix-11454.en.md @@ -0,0 +1 @@ +Fixed crashing when debugging/tracing with large payloads(introduce when [#11279](https://github.com/emqx/emqx/pull/11279)) From 040d28c35e3630b2a5b1f74d8d5bd291771ff5c0 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 16 Aug 2023 18:46:17 +0800 Subject: [PATCH 28/37] fix(cli): fix two typos in the time unit --- apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index 037a3c2ee..1ba924864 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -74,8 +74,8 @@ pretty_print_rule(ID) -> "Updated at:\n ~ts\n" "Actions:\n ~s\n" ,[Id, Name, left_pad(Descr), Enable, left_pad(SQL), - emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, second), - emqx_utils_calendar:epoch_to_rfc3339(UpdatedAt, second), + emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, millisecond), + emqx_utils_calendar:epoch_to_rfc3339(UpdatedAt, millisecond), [left_pad(format_action(A)) || A <- Actions] ] ); From b733adca06bb5c0420406e5db2e24067afd8010f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 20:15:21 +0800 Subject: [PATCH 29/37] fix: allow empty cacertfile pem --- apps/emqx/src/emqx_schema.erl | 30 +++++++++++++-------------- apps/emqx/src/emqx_tls_lib.erl | 9 ++++++++ apps/emqx/test/emqx_tls_lib_tests.erl | 13 +++++++++++- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 86ddb6577..88243a308 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2001,8 +2001,8 @@ filter(Opts) -> %% SSL listener and client. -spec common_ssl_opts_schema(map(), server | client) -> hocon_schema:field_schema(). common_ssl_opts_schema(Defaults, Type) -> - D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end, - Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, + D = fun(Field) -> maps:get(Field, Defaults, undefined) end, + Df = fun(Field, Default) -> maps:get(Field, Defaults, Default) end, Collection = maps:get(versions, Defaults, tls_all_available), DefaultVersions = default_tls_vsns(Collection), [ @@ -2045,7 +2045,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( hoconsc:enum([verify_peer, verify_none]), #{ - default => Df("verify", verify_none), + default => Df(verify, verify_none), desc => ?DESC(common_ssl_opts_schema_verify) } )}, @@ -2053,7 +2053,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( boolean(), #{ - default => Df("reuse_sessions", true), + default => Df(reuse_sessions, true), desc => ?DESC(common_ssl_opts_schema_reuse_sessions) } )}, @@ -2061,7 +2061,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( non_neg_integer(), #{ - default => Df("depth", 10), + default => Df(depth, 10), desc => ?DESC(common_ssl_opts_schema_depth) } )}, @@ -2088,7 +2088,7 @@ common_ssl_opts_schema(Defaults, Type) -> validator => fun(Input) -> validate_tls_versions(Collection, Input) end } )}, - {"ciphers", ciphers_schema(D("ciphers"))}, + {"ciphers", ciphers_schema(D(ciphers))}, {"user_lookup_fun", sc( typerefl:alias("string", any()), @@ -2103,7 +2103,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( boolean(), #{ - default => Df("secure_renegotiate", true), + default => Df(secure_renegotiate, true), desc => ?DESC(common_ssl_opts_schema_secure_renegotiate) } )}, @@ -2123,7 +2123,7 @@ common_ssl_opts_schema(Defaults, Type) -> sc( duration(), #{ - default => Df("hibernate_after", <<"5s">>), + default => Df(hibernate_after, <<"5s">>), desc => ?DESC(common_ssl_opts_schema_hibernate_after) } )} @@ -2132,15 +2132,15 @@ common_ssl_opts_schema(Defaults, Type) -> %% @doc Make schema for SSL listener options. -spec server_ssl_opts_schema(map(), boolean()) -> hocon_schema:field_schema(). server_ssl_opts_schema(Defaults, IsRanchListener) -> - D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end, - Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, + D = fun(Field) -> maps:get(Field, Defaults, undefined) end, + Df = fun(Field, Default) -> maps:get(Field, Defaults, Default) end, common_ssl_opts_schema(Defaults, server) ++ [ {"dhfile", sc( string(), #{ - default => D("dhfile"), + default => D(dhfile), required => false, desc => ?DESC(server_ssl_opts_schema_dhfile) } @@ -2149,7 +2149,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( boolean(), #{ - default => Df("fail_if_no_peer_cert", false), + default => Df(fail_if_no_peer_cert, false), desc => ?DESC(server_ssl_opts_schema_fail_if_no_peer_cert) } )}, @@ -2157,7 +2157,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( boolean(), #{ - default => Df("honor_cipher_order", true), + default => Df(honor_cipher_order, true), desc => ?DESC(server_ssl_opts_schema_honor_cipher_order) } )}, @@ -2165,7 +2165,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( boolean(), #{ - default => Df("client_renegotiation", true), + default => Df(client_renegotiation, true), desc => ?DESC(server_ssl_opts_schema_client_renegotiation) } )}, @@ -2173,7 +2173,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( duration(), #{ - default => Df("handshake_timeout", <<"15s">>), + default => Df(handshake_timeout, <<"15s">>), desc => ?DESC(server_ssl_opts_schema_handshake_timeout) } )} diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index b5b653f56..9113bd5e6 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -62,6 +62,8 @@ [ocsp, issuer_pem] ]). +-define(ALLOW_EMPTY_PEM, [[<<"cacertfile">>], [cacertfile]]). + %% non-empty string -define(IS_STRING(L), (is_list(L) andalso L =/= [] andalso is_integer(hd(L)))). %% non-empty list of strings @@ -330,6 +332,13 @@ ensure_ssl_files_per_key(Dir, SSL, [KeyPath | KeyPaths], Opts) -> ensure_ssl_file(_Dir, _KeyPath, SSL, undefined, _Opts) -> {ok, SSL}; +ensure_ssl_file(_Dir, KeyPath, SSL, MaybePem, _Opts) when + MaybePem =:= "" orelse MaybePem =:= <<"">> +-> + case lists:member(KeyPath, ?ALLOW_EMPTY_PEM) of + true -> {ok, SSL}; + false -> {error, #{reason => pem_file_path_or_string_is_required}} + end; ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, Opts) -> case is_valid_string(MaybePem) of true -> diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 481b9378e..ae7caa4fb 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -113,11 +113,22 @@ ssl_files_failure_test_() -> }) ) end}, + {"empty_cacertfile", fun() -> + ?assertMatch( + {ok, _}, + emqx_tls_lib:ensure_ssl_files("/tmp", #{ + <<"keyfile">> => test_key(), + <<"certfile">> => test_key(), + <<"cacertfile">> => <<"">> + }) + ) + end}, {"bad_pem_string", fun() -> %% empty string ?assertMatch( {error, #{ - reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]] + reason := pem_file_path_or_string_is_required, + which_options := [[<<"keyfile">>]] }}, emqx_tls_lib:ensure_ssl_files("/tmp", #{ <<"keyfile">> => <<>>, From 960944f90cb50bf49655b075bb0ea9c6436784be Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 20:19:55 +0800 Subject: [PATCH 30/37] chore: add 11456 changelog --- changes/ce/fix-11456.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-11456.en.md diff --git a/changes/ce/fix-11456.en.md b/changes/ce/fix-11456.en.md new file mode 100644 index 000000000..8ace3f88a --- /dev/null +++ b/changes/ce/fix-11456.en.md @@ -0,0 +1,2 @@ +Removed validation that enforced non-empty PEM for CA cert file. +CA certificate file PEM can now be empty. From 848eb7e3c34eef20810adb56079ce46ca5a37771 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 15 Aug 2023 16:47:43 +0800 Subject: [PATCH 31/37] fix: remove os_mon application in Windows release --- apps/emqx/src/emqx.app.src | 1 - apps/emqx/src/emqx_os_mon.erl | 14 ++++++------- apps/emqx/src/emqx_schema.erl | 8 +++++++- apps/emqx/src/emqx_sys_sup.erl | 25 +++++++++++++++-------- apps/emqx_machine/priv/reboot_lists.eterm | 1 - rebar.config.erl | 19 ++++++++++------- rel/i18n/emqx_schema.hocon | 12 +++++------ 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 513f88612..a80d6482a 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -14,7 +14,6 @@ esockd, cowboy, sasl, - os_mon, lc, hocon, emqx_durable_storage diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 144d2bfe5..f84636a84 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -56,7 +56,7 @@ start_link() -> gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []). update(OS) -> - erlang:send(?MODULE, {monitor_conf_update, OS}). + gen_server:cast(?MODULE, {monitor_conf_update, OS}). %%-------------------------------------------------------------------- %% API @@ -110,6 +110,12 @@ handle_call({set_sysmem_high_watermark, New}, _From, #{sysmem_high_watermark := handle_call(Req, _From, State) -> {reply, {error, {unexpected_call, Req}}, State}. +handle_cast({monitor_conf_update, OS}, State) -> + cancel_outdated_timer(State), + SysHW = init_os_monitor(OS), + MemRef = start_mem_check_timer(), + CpuRef = start_cpu_check_timer(), + {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}; handle_cast(Msg, State) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. @@ -151,12 +157,6 @@ handle_info({timeout, _Timer, cpu_check}, State) -> end, Ref = start_cpu_check_timer(), {noreply, State#{cpu_time_ref => Ref}}; -handle_info({monitor_conf_update, OS}, State) -> - cancel_outdated_timer(State), - SysHW = init_os_monitor(OS), - MemRef = start_mem_check_timer(), - CpuRef = start_cpu_check_timer(), - {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}; handle_info(Info, State) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 86ddb6577..d746d3393 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1582,7 +1582,7 @@ fields("sysmon_os") -> sc( hoconsc:union([disabled, duration()]), #{ - default => <<"60s">>, + default => default_mem_check_interval(), desc => ?DESC(sysmon_os_mem_check_interval) } )}, @@ -3657,3 +3657,9 @@ shared_subscription_strategy() -> desc => ?DESC(broker_shared_subscription_strategy) } )}. + +default_mem_check_interval() -> + case emqx_sys_sup:is_os_mon_supported() of + true -> <<"60s">>; + false -> disabled + end. diff --git a/apps/emqx/src/emqx_sys_sup.erl b/apps/emqx/src/emqx_sys_sup.erl index 8c26df020..75f4976c9 100644 --- a/apps/emqx/src/emqx_sys_sup.erl +++ b/apps/emqx/src/emqx_sys_sup.erl @@ -19,6 +19,7 @@ -behaviour(supervisor). -export([start_link/0]). +-export([is_os_mon_supported/0]). -export([init/1]). @@ -26,19 +27,27 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Childs = [ - child_spec(emqx_sys), - child_spec(emqx_alarm), - child_spec(emqx_sys_mon), - child_spec(emqx_os_mon), - child_spec(emqx_vm_mon) - ], - {ok, {{one_for_one, 10, 100}, Childs}}. + OsMon = + case is_os_mon_supported() of + true -> [child_spec(emqx_os_mon)]; + false -> [] + end, + Children = + [ + child_spec(emqx_sys), + child_spec(emqx_alarm), + child_spec(emqx_sys_mon), + child_spec(emqx_vm_mon) + ] ++ OsMon, + {ok, {{one_for_one, 10, 100}, Children}}. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +is_os_mon_supported() -> + erlang:function_exported(memsup, get_procmem_high_watermark, 0). + child_spec(Mod) -> child_spec(Mod, []). diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index b38c124e7..09c7fba37 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -36,7 +36,6 @@ [ emqx, emqx_conf, - esasl, observer_cli, tools, diff --git a/rebar.config.erl b/rebar.config.erl index 9c556cd9f..ad6f425a0 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -405,12 +405,13 @@ relx_apps(ReleaseType, Edition) -> ce -> CEBusinessApps end, BusinessApps = CommonBusinessApps ++ EditionSpecificApps, - ExcludedApps = excluded_apps(ReleaseType), - SystemApps ++ - %% EMQX starts the DB and the business applications: - [{App, load} || App <- (DBApps -- ExcludedApps)] ++ - [emqx_machine] ++ - [{App, load} || App <- (BusinessApps -- ExcludedApps)]. + Apps = + (SystemApps ++ + %% EMQX starts the DB and the business applications: + [{App, load} || App <- DBApps] ++ + [emqx_machine] ++ + [{App, load} || App <- BusinessApps]), + lists:foldl(fun proplists:delete/2, Apps, excluded_apps(ReleaseType)). excluded_apps(ReleaseType) -> OptionalApps = [ @@ -418,7 +419,8 @@ excluded_apps(ReleaseType) -> {bcrypt, provide_bcrypt_release(ReleaseType)}, {jq, is_jq_supported()}, {observer, is_app(observer)}, - {mnesia_rocksdb, is_rocksdb_supported()} + {mnesia_rocksdb, is_rocksdb_supported()}, + {os_mon, provide_os_mon_release()} ], [App || {App, false} <- OptionalApps]. @@ -524,6 +526,9 @@ is_debug(VarName) -> provide_bcrypt_dep() -> not is_win32(). +provide_os_mon_release() -> + not is_win32(). + provide_bcrypt_release(ReleaseType) -> provide_bcrypt_dep() andalso ReleaseType =:= cloud. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 6cf20f90c..251dcdcb9 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -156,7 +156,7 @@ persistent_session_builtin_messages_table.label: sysmon_os_cpu_low_watermark.desc: """The threshold, as percentage of system CPU load, - for how much system cpu can be used before the corresponding alarm is cleared.""" + for how much system cpu can be used before the corresponding alarm is cleared. Disabled on Windows platform""" sysmon_os_cpu_low_watermark.label: """CPU low watermark""" @@ -278,7 +278,7 @@ fields_ws_opts_mqtt_path.label: sysmon_os_procmem_high_watermark.desc: """The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before - the corresponding alarm is raised.""" + the corresponding alarm is raised. Disabled on Windows platform.""" sysmon_os_procmem_high_watermark.label: """ProcMem high wartermark""" @@ -389,7 +389,7 @@ fields_tcp_opts_sndbuf.label: """TCP send buffer""" sysmon_os_mem_check_interval.desc: -"""The time interval for the periodic memory check.""" +"""The time interval for the periodic memory check. Disabled on Windows platform.""" sysmon_os_mem_check_interval.label: """Mem check interval""" @@ -742,7 +742,7 @@ common_ssl_opts_schema_keyfile.label: sysmon_os_cpu_high_watermark.desc: """The threshold, as percentage of system CPU load, - for how much system cpu can be used before the corresponding alarm is raised.""" + for how much system cpu can be used before the corresponding alarm is raised. Disabled on Windows platform""" sysmon_os_cpu_high_watermark.label: """CPU high watermark""" @@ -798,7 +798,7 @@ fields_ws_opts_proxy_address_header.label: sysmon_os_sysmem_high_watermark.desc: """The threshold, as percentage of system memory, - for how much system memory can be allocated before the corresponding alarm is raised.""" + for how much system memory can be allocated before the corresponding alarm is raised. Disabled on Windows platform""" sysmon_os_sysmem_high_watermark.label: """SysMem high wartermark""" @@ -1521,7 +1521,7 @@ fields_tcp_opts_send_timeout_close.label: """TCP send timeout close""" sysmon_os_cpu_check_interval.desc: -"""The time interval for the periodic CPU check.""" +"""The time interval for the periodic CPU check. Disabled on Windows platform.""" sysmon_os_cpu_check_interval.label: """The time interval for the periodic CPU check.""" From b817e03c08b2a30ae3e3abb916b8701d58a6ba88 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 15 Aug 2023 17:48:59 +0800 Subject: [PATCH 32/37] fix: start os_mon application temporary --- apps/emqx/src/emqx_os_mon.erl | 18 +++++---- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_sys_mon.erl | 8 +++- apps/emqx/src/emqx_sys_sup.erl | 7 +--- apps/emqx/src/emqx_vm.erl | 32 ++++++++++----- apps/emqx/test/emqx_os_mon_SUITE.erl | 48 ++++++++++++++++------- apps/emqx_machine/priv/reboot_lists.eterm | 3 +- apps/emqx_management/src/emqx_mgmt.erl | 13 +++--- changes/ce/fix-11445.en.md | 2 + 9 files changed, 86 insertions(+), 47 deletions(-) create mode 100644 changes/ce/fix-11445.en.md diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index f84636a84..ec2fb1d5f 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -38,15 +38,14 @@ %% gen_server callbacks -export([ init/1, + handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3 ]). --ifdef(TEST). --export([is_sysmem_check_supported/0]). --endif. +-export([is_os_check_supported/0]). -include("emqx.hrl"). @@ -83,12 +82,17 @@ current_sysmem_percent() -> %%-------------------------------------------------------------------- init([]) -> + %% start os_mon temporarily + {ok, _} = application:ensure_all_started(os_mon), + {ok, undefined, {continue, setup}}. + +handle_continue(setup, undefined) -> %% memsup is not reliable, ignore memsup:set_sysmem_high_watermark(1.0), SysHW = init_os_monitor(), MemRef = start_mem_check_timer(), CpuRef = start_cpu_check_timer(), - {ok, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}. + {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}. init_os_monitor() -> init_os_monitor(emqx:get_config([sysmon, os])). @@ -182,12 +186,12 @@ start_cpu_check_timer() -> _ -> start_timer(Interval, cpu_check) end. -is_sysmem_check_supported() -> +is_os_check_supported() -> {unix, linux} =:= os:type(). start_mem_check_timer() -> Interval = emqx:get_config([sysmon, os, mem_check_interval]), - case is_integer(Interval) andalso is_sysmem_check_supported() of + case is_integer(Interval) andalso is_os_check_supported() of true -> start_timer(Interval, mem_check); false -> @@ -205,7 +209,7 @@ update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> <<"Deactivated mem usage alarm due to out of range threshold">> ); update_mem_alarm_status(HWM) -> - is_sysmem_check_supported() andalso + is_os_check_supported() andalso do_update_mem_alarm_status(HWM), ok. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d746d3393..14881d54d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3659,7 +3659,7 @@ shared_subscription_strategy() -> )}. default_mem_check_interval() -> - case emqx_sys_sup:is_os_mon_supported() of + case emqx_os_mon:is_os_check_supported() of true -> <<"60s">>; false -> disabled end. diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index f1190f586..1d3d32199 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -29,6 +29,7 @@ %% gen_server callbacks -export([ init/1, + handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, @@ -70,11 +71,14 @@ update(VM) -> init([]) -> emqx_logger:set_proc_metadata(#{sysmon => true}), - init_system_monitor(), + {ok, undefined, {continue, setup}}. +handle_continue(setup, undefined) -> + init_system_monitor(), %% Monitor cluster partition event ekka:monitor(partition, fun handle_partition_event/1), - {ok, start_timer(#{timer => undefined, events => []})}. + NewState = start_timer(#{timer => undefined, events => []}), + {noreply, NewState, hibernate}. start_timer(State) -> State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}. diff --git a/apps/emqx/src/emqx_sys_sup.erl b/apps/emqx/src/emqx_sys_sup.erl index 75f4976c9..25718ba76 100644 --- a/apps/emqx/src/emqx_sys_sup.erl +++ b/apps/emqx/src/emqx_sys_sup.erl @@ -19,8 +19,6 @@ -behaviour(supervisor). -export([start_link/0]). --export([is_os_mon_supported/0]). - -export([init/1]). start_link() -> @@ -28,7 +26,7 @@ start_link() -> init([]) -> OsMon = - case is_os_mon_supported() of + case emqx_os_mon:is_os_check_supported() of true -> [child_spec(emqx_os_mon)]; false -> [] end, @@ -45,9 +43,6 @@ init([]) -> %% Internal functions %%-------------------------------------------------------------------- -is_os_mon_supported() -> - erlang:function_exported(memsup, get_procmem_high_watermark, 0). - child_spec(Mod) -> child_spec(Mod, []). diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 0d861f671..d3f98e06c 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -44,7 +44,7 @@ get_otp_version/0 ]). --export([cpu_util/0]). +-export([cpu_util/0, cpu_util/1]). -ifdef(TEST). -compile(export_all). @@ -378,18 +378,30 @@ avg15() -> cpu_util() -> compat_windows(fun cpu_sup:util/0). +cpu_util(Args) -> + compat_windows(fun cpu_sup:util/1, Args). + compat_windows(Fun) -> - case os:type() of - {win32, nt} -> - 0.0; - _Type -> - case catch Fun() of - Val when is_float(Val) -> floor(Val * 100) / 100; - Val when is_number(Val) -> Val; - _Error -> 0.0 - end + case compat_windows(Fun, []) of + Val when is_float(Val) -> floor(Val * 100) / 100; + Val when is_number(Val) -> Val; + _ -> 0.0 end. +compat_windows(Fun, Args) -> + try + case is_windows() of + true -> 0.0; + false when Args =:= [] -> Fun(); + false -> Fun(Args) + end + catch + _:_ -> 0.0 + end. + +is_windows() -> + os:type() =:= {win32, nt}. + load(Avg) -> floor((Avg / 256) * 100) / 100. diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index e76928114..1833be48e 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -39,29 +39,47 @@ init_per_testcase(t_cpu_check_alarm, Config) -> %% 200ms cpu_check_interval => 200 }), - ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), - {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), + restart_os_mon(), Config; init_per_testcase(t_sys_mem_check_alarm, Config) -> - case emqx_os_mon:is_sysmem_check_supported() of + case emqx_os_mon:is_os_check_supported() of true -> SysMon = emqx_config:get([sysmon, os], #{}), emqx_config:put([sysmon, os], SysMon#{ sysmem_high_watermark => 0.51, %% 200ms mem_check_interval => 200 - }), - ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), - {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon), - Config; + }); false -> - Config - end; + ok + end, + restart_os_mon(), + Config; init_per_testcase(_, Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), + restart_os_mon(), Config. +restart_os_mon() -> + case emqx_os_mon:is_os_check_supported() of + true -> + ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), + {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon); + false -> + _ = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon), + _ = supervisor:delete_child(emqx_sys_sup, emqx_os_mon), + %% run test on mac/windows. + Mod = emqx_os_mon, + OsMon = #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod] + }, + {ok, _} = supervisor:start_child(emqx_sys_sup, OsMon) + end. + t_api(_) -> ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()), ?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)), @@ -81,7 +99,7 @@ t_api(_) -> ok. t_sys_mem_check_disable(Config) -> - case emqx_os_mon:is_sysmem_check_supported() of + case emqx_os_mon:is_os_check_supported() of true -> do_sys_mem_check_disable(Config); false -> skip end. @@ -100,7 +118,7 @@ do_sys_mem_check_disable(_Config) -> ok. t_sys_mem_check_alarm(Config) -> - case emqx_os_mon:is_sysmem_check_supported() of + case emqx_os_mon:is_os_check_supported() of true -> do_sys_mem_check_alarm(Config); false -> skip end. @@ -167,7 +185,7 @@ t_cpu_check_alarm(_) -> util, fun() -> CpuUtil end, fun() -> - timer:sleep(500), + timer:sleep(1000), Alarms = emqx_alarm:get_alarms(activated), ?assert( emqx_vm_mon_SUITE:is_existing(high_cpu_usage, emqx_alarm:get_alarms(activated)) @@ -193,7 +211,7 @@ t_cpu_check_alarm(_) -> ?assert(is_binary(Msg)), emqx_config:put([sysmon, os, cpu_high_watermark], 1), emqx_config:put([sysmon, os, cpu_low_watermark], 0.96), - timer:sleep(500), + timer:sleep(800), ?assertNot( emqx_vm_mon_SUITE:is_existing(high_cpu_usage, emqx_alarm:get_alarms(activated)) ) diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 09c7fba37..0e2ecb799 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -17,7 +17,8 @@ asn1, syntax_tools, ssl, - os_mon, + %% started temporary in emqx to prevent crash vm when permanent. + {os_mon, load}, inets, compiler, runtime_tools, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 059c323ff..e00044add 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -197,13 +197,16 @@ vm_stats() -> ]. vm_stats('cpu.idle') -> - case cpu_sup:util([detailed]) of - %% Not support for Windows - {_, 0, 0, _} -> 0; - {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) + case emqx_vm:cpu_util([detailed]) of + {_Num, _Use, List, _} when is_list(List) -> proplists:get_value(idle, List, 0); + %% return {all, 0, 0, []} when cpu_sup is not started + _ -> 0 end; vm_stats('cpu.use') -> - 100 - vm_stats('cpu.idle'); + case vm_stats('cpu.idle') of + 0 -> 0; + Idle -> 100 - Idle + end; vm_stats('total.memory') -> {_, MemTotal} = get_sys_memory(), MemTotal; diff --git a/changes/ce/fix-11445.en.md b/changes/ce/fix-11445.en.md new file mode 100644 index 000000000..589846db2 --- /dev/null +++ b/changes/ce/fix-11445.en.md @@ -0,0 +1,2 @@ +Removed os_mon application monitor support on Windows platforms to prevent VM crashes. +Functionality remains on non-Windows platforms. From 8b23ee86b32bd36dcecf5c346c1f03e989078e9c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 15:19:18 +0800 Subject: [PATCH 33/37] fix: hide cpu_status if os_check is not supported --- apps/emqx/src/emqx_vm.erl | 11 ++++------- apps/emqx_management/src/emqx_mgmt.erl | 27 ++++++++++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index d3f98e06c..79ad9905c 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -390,18 +390,15 @@ compat_windows(Fun) -> compat_windows(Fun, Args) -> try - case is_windows() of - true -> 0.0; - false when Args =:= [] -> Fun(); - false -> Fun(Args) + case emqx_os_mon:is_os_check_supported() of + false -> 0.0; + true when Args =:= [] -> Fun(); + true -> Fun(Args) end catch _:_ -> 0.0 end. -is_windows() -> - os:type() =:= {win32, nt}. - load(Avg) -> floor((Avg / 256) * 100) / 100. diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index e00044add..5417bd4b9 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -185,16 +185,27 @@ node_info(Nodes) -> stopped_node_info(Node) -> {Node, #{node => Node, node_status => 'stopped', role => core}}. +%% Hide cpu stats if os_check is not supported. vm_stats() -> - Idle = vm_stats('cpu.idle'), {MemUsedRatio, MemTotal} = get_sys_memory(), - [ - {run_queue, vm_stats('run.queue')}, - {cpu_idle, Idle}, - {cpu_use, 100 - Idle}, - {total_memory, MemTotal}, - {used_memory, erlang:round(MemTotal * MemUsedRatio)} - ]. + cpu_stats() ++ + [ + {run_queue, vm_stats('run.queue')}, + {total_memory, MemTotal}, + {used_memory, erlang:round(MemTotal * MemUsedRatio)} + ]. + +cpu_stats() -> + case emqx_os_mon:is_os_check_supported() of + false -> + []; + true -> + Idle = vm_stats('cpu.idle'), + [ + {cpu_idle, Idle}, + {cpu_use, 100 - Idle} + ] + end. vm_stats('cpu.idle') -> case emqx_vm:cpu_util([detailed]) of From c0d14bb112f437c3216317d47c7c916961068b47 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 20:52:44 +0800 Subject: [PATCH 34/37] chore: don't include os_mon on windows elixir release --- mix.exs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 2580cc3d8..e6047249e 100644 --- a/mix.exs +++ b/mix.exs @@ -403,7 +403,8 @@ defmodule EMQXUmbrella.MixProject do quicer: enable_quicer?(), bcrypt: enable_bcrypt?(), jq: enable_jq?(), - observer: is_app?(:observer) + observer: is_app?(:observer), + os_mon: enable_os_mon?() } |> Enum.reject(&elem(&1, 1)) |> Enum.map(&elem(&1, 0)) @@ -835,6 +836,10 @@ defmodule EMQXUmbrella.MixProject do not win32?() end + defp enable_os_mon?() do + not win32?() + end + defp enable_jq?() do not Enum.any?([ build_without_jq?(), From 78356d26259dc14674311a199691f784875dd0f6 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 16 Aug 2023 20:54:53 +0800 Subject: [PATCH 35/37] chore: start os_mon in handle_continue/2 --- apps/emqx/src/emqx_os_mon.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index ec2fb1d5f..4114a0b17 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -82,11 +82,11 @@ current_sysmem_percent() -> %%-------------------------------------------------------------------- init([]) -> - %% start os_mon temporarily - {ok, _} = application:ensure_all_started(os_mon), {ok, undefined, {continue, setup}}. handle_continue(setup, undefined) -> + %% start os_mon temporarily + {ok, _} = application:ensure_all_started(os_mon), %% memsup is not reliable, ignore memsup:set_sysmem_high_watermark(1.0), SysHW = init_os_monitor(), From a6a023eff77bc5c112aa80350546592f927e4fff Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 16 Aug 2023 11:48:47 -0300 Subject: [PATCH 36/37] ci: add `integration_test` dir to `emqx` application This gives us a place where to put tests that exercise multiple umbrella application, which is more closely related to `emqx`, but in a way that doesn't affect the standalone app tests. --- .github/workflows/run_emqx_app_tests.yaml | 2 +- apps/emqx/integration_test/.gitkeep | 0 apps/emqx/rebar.config | 10 ++++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 apps/emqx/integration_test/.gitkeep diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 5479a86ab..72fe2b0d5 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -56,7 +56,7 @@ jobs: ./rebar3 xref ./rebar3 dialyzer ./rebar3 eunit -v - ./rebar3 ct --name 'test@127.0.0.1' -v --readable=true + ./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true ./rebar3 proper -d test/props - uses: actions/upload-artifact@v3 if: failure() diff --git a/apps/emqx/integration_test/.gitkeep b/apps/emqx/integration_test/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 355f005a8..730155805 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -41,6 +41,16 @@ {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [ + {deps, [ + {meck, "0.9.2"}, + {proper, "1.4.0"}, + {bbmustache, "1.10.0"}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.6"}}} + ]}, + {extra_src_dirs, [{"test", [recursive]}, + {"integration_test", [recursive]}]} + ]}, + {standalone_test, [ {deps, [ {meck, "0.9.2"}, {proper, "1.4.0"}, From a64386ef8201c6359a50545a9d30a45582283124 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 14 Aug 2023 18:11:01 -0300 Subject: [PATCH 37/37] fix(kinesis): return error message on access denied Fixes https://emqx.atlassian.net/browse/EMQX-10764 --- .../src/emqx_bridge_kinesis.app.src | 2 +- .../emqx_bridge_kinesis_connector_client.erl | 9 ++- .../src/emqx_bridge_kinesis_impl_producer.erl | 7 ++- ...mqx_bridge_kinesis_impl_producer_SUITE.erl | 55 +++++++++++++++++-- changes/ee/fix-11444.en.md | 1 + 5 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 changes/ee/fix-11444.en.md diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index 36f6c8b0b..3eb923b5d 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kinesis, [ {description, "EMQX Enterprise Amazon Kinesis Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl index bb1000e5f..d9dc0220f 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl @@ -111,7 +111,14 @@ init(#{ erlcloud_config:configure( to_str(AwsAccessKey), to_str(AwsSecretAccessKey), Host, Port, Scheme, New ), - {ok, State}. + % check the connection + case erlcloud_kinesis:list_streams() of + {ok, _} -> + {ok, State}; + {error, Reason} -> + ?tp(kinesis_init_failed, #{instance_id => InstanceId, reason => Reason}), + {stop, Reason} + end. handle_call(connection_status, _From, #{stream_name := StreamName} = State) -> Status = diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 7948581b5..1e07ae96e 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -114,7 +114,12 @@ on_get_status(_InstanceId, #{pool_name := Pool} = State) -> false -> disconnected end end; - {error, _} -> + {error, Reason} -> + ?SLOG(error, #{ + msg => "kinesis_producer_get_status_failed", + state => State, + reason => Reason + }), disconnected end. diff --git a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl index 114f324a9..d0fe4a1b4 100644 --- a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl @@ -796,7 +796,9 @@ t_publish_connection_down(Config0) -> ok. t_wrong_server(Config) -> + TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kinesis_name, Config), + KinesisConfig0 = ?config(kinesis_config, Config), ResourceId = ?config(resource_id, Config), Overrides = #{ @@ -806,12 +808,57 @@ t_wrong_server(Config) -> <<"health_check_interval">> => <<"60s">> } }, + % probe + KinesisConfig = emqx_utils_maps:deep_merge(KinesisConfig0, Overrides), + Params = KinesisConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + ProbePath = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(post, ProbePath, "", AuthHeader, Params) + ), + % create ?wait_async_action( create_bridge(Config, Overrides), - #{?snk_kind := emqx_bridge_kinesis_impl_producer_start_ok}, + #{?snk_kind := start_pool_failed}, 30_000 ), - ?assertEqual({error, timeout}, emqx_resource_manager:health_check(ResourceId)), - emqx_bridge_resource:stop(?BRIDGE_TYPE, Name), - emqx_bridge_resource:remove(?BRIDGE_TYPE, Name), + ?assertMatch( + {ok, _, #{error := {start_pool_failed, ResourceId, _}}}, + emqx_resource_manager:lookup_cached(ResourceId) + ), + ok. + +t_access_denied(Config) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(kinesis_name, Config), + KinesisConfig = ?config(kinesis_config, Config), + ResourceId = ?config(resource_id, Config), + AccessError = {<<"AccessDeniedException">>, <<>>}, + Params = KinesisConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + ProbePath = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + emqx_common_test_helpers:with_mock( + erlcloud_kinesis, + list_streams, + fun() -> {error, AccessError} end, + fun() -> + % probe + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(post, ProbePath, "", AuthHeader, Params) + ), + % create + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := kinesis_init_failed}, + 30_000 + ), + ?assertMatch( + {ok, _, #{error := {start_pool_failed, ResourceId, AccessError}}}, + emqx_resource_manager:lookup_cached(ResourceId) + ), + ok + end + ), ok. diff --git a/changes/ee/fix-11444.en.md b/changes/ee/fix-11444.en.md new file mode 100644 index 000000000..c8e80946d --- /dev/null +++ b/changes/ee/fix-11444.en.md @@ -0,0 +1 @@ +Fixed error information when Kinesis bridge fails to connect to endpoint.