From 7567d211daa3826bcdb0e7613b55c33976938327 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 7 Aug 2023 17:40:59 +0800 Subject: [PATCH 1/8] fix(placeholder): porting fix to support utf8 key in placeholder --- apps/emqx_utils/src/emqx_placeholder.erl | 8 ++- .../test/emqx_placeholder_SUITE.erl | 50 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl index edf4123e4..0f677236d 100644 --- a/apps/emqx_utils/src/emqx_placeholder.erl +++ b/apps/emqx_utils/src/emqx_placeholder.erl @@ -48,9 +48,13 @@ -define(PH_VAR_THIS, '$this'). --define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})"). +%% To match any pattern starts with '$' and followed by '{', and closed by a '}' char: +%% e.g. for string "a${abc}bb", "${abc}" will be matched. +%% Note this is non-greedy matching +%% e.g. if "${{abc}}" is given, the "${{abc}" should be matched, NOT "${{abc}}". +-define(EX_PLACE_HOLDER, "(\\$\\{[^}]+\\})"). --define(EX_PLACE_HOLDER_DOUBLE_QUOTE, "(\\$\\{[a-zA-Z0-9\\._]+\\}|\"\\$\\{[a-zA-Z0-9\\._]+\\}\")"). +-define(EX_PLACE_HOLDER_DOUBLE_QUOTE, "(\\$\\{[^}]+\\}|\"\\$\\{[^}]+\\}\")"). %% Space and CRLF -define(EX_WITHE_CHARS, "\\s"). diff --git a/apps/emqx_utils/test/emqx_placeholder_SUITE.erl b/apps/emqx_utils/test/emqx_placeholder_SUITE.erl index 81bf0853a..f813656f2 100644 --- a/apps/emqx_utils/test/emqx_placeholder_SUITE.erl +++ b/apps/emqx_utils/test/emqx_placeholder_SUITE.erl @@ -206,3 +206,53 @@ t_preproc_tmpl_deep(_) -> #{<<"${a}">> => [<<"1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>], 0}]}, emqx_placeholder:proc_tmpl_deep(Tmpl1, Selected) ). + +t_proc_tmpl_arbitrary_var_name(_) -> + Selected = #{ + <<"中"/utf8>> => <<"1">>, + <<"中-1"/utf8>> => <<"1-1">>, + <<"-_+=<>,/?:;\"'\\[]|">> => 1, + <<"-_+=<>,">> => #{<<"/?:;\"'\\[]|">> => 2}, + <<"!@#$%^&*()">> => 1.0, + <<"d">> => #{ + <<"$ff">> => <<"oo">>, + <<"${f">> => <<"hi">>, + <<"${f}">> => <<"qq">> + } + }, + Tks = emqx_placeholder:preproc_tmpl( + << + "a:${中},a:${中-1},b:${-_+=<>,/?:;\"'\\[]|}," + "b:${-_+=<>,./?:;\"'\\[]|},c:${!@#$%^&*()},d:${d.$ff},d1:${d.${f}}"/utf8 + >> + ), + ?assertEqual( + <<"a:1,a:1-1,b:1,b:2,c:1.0,d:oo,d1:hi}">>, + emqx_placeholder:proc_tmpl(Tks, Selected) + ). + +t_proc_tmpl_arbitrary_var_name_double_quote(_) -> + Selected = #{ + <<"中"/utf8>> => <<"1">>, + <<"中-1"/utf8>> => <<"1-1">>, + <<"-_+=<>,/?:;\"'\\[]|">> => 1, + <<"-_+=<>,">> => #{<<"/?:;\"'\\[]|">> => 2}, + <<"!@#$%^&*()">> => 1.0, + <<"d">> => #{ + <<"$ff">> => <<"oo">>, + <<"${f">> => <<"hi">>, + <<"${f}">> => <<"qq">> + } + }, + Tks = emqx_placeholder:preproc_tmpl( + << + "a:\"${中}\",a:\"${中-1}\",b:\"${-_+=<>,/?:;\"'\\[]|}\"," + "b:\"${-_+=<>,./?:;\"'\\[]|}\",c:\"${!@#$%^&*()}\",d:\"${d.$ff}\",d1:\"${d.${f}\"}"/utf8 + >>, + #{strip_double_quote => true} + ), + ct:print("TKs:~p~n", [Tks]), + ?assertEqual( + <<"a:1,a:1-1,b:1,b:2,c:1.0,d:oo,d1:hi}">>, + emqx_placeholder:proc_tmpl(Tks, Selected) + ). From 3e9155fdb1c287a0b565c602a178a5b3d2f190f7 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 7 Aug 2023 17:53:28 +0800 Subject: [PATCH 2/8] chore: bump emqx_utils version && changes --- apps/emqx_utils/src/emqx_utils.app.src | 2 +- changes/ce/perf-11399.en.md | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 changes/ce/perf-11399.en.md diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 5900514dc..f8905b513 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/changes/ce/perf-11399.en.md b/changes/ce/perf-11399.en.md new file mode 100644 index 000000000..42dac80bc --- /dev/null +++ b/changes/ce/perf-11399.en.md @@ -0,0 +1,8 @@ +Improved the placeholder syntax of rule engine. + +The parameters of actions support using placeholder syntax to +dynamically fill in the content of strings. The format of the +placeholder syntax is `${key}`. +Before this improvement, the `key` in `${key}` could only contain +letters, numbers, and underscores. Now the `key` supports any UTF8 +characters. From 3933227636f005d425cd6ea6933e733e0f5f2553 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 8 Aug 2023 20:59:43 +0800 Subject: [PATCH 3/8] feat: support opentelemetry metrics --- apps/emqx/src/emqx_stats.erl | 25 ++- apps/emqx_conf/src/emqx_conf_schema.erl | 1 + apps/emqx_machine/priv/reboot_lists.eterm | 8 +- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt.erl | 30 ++- apps/emqx_opentelemetry/README.md | 4 + apps/emqx_opentelemetry/etc/emqx_otel.conf | 0 apps/emqx_opentelemetry/rebar.config | 29 +++ .../src/emqx_opentelemetry.app.src | 15 ++ apps/emqx_opentelemetry/src/emqx_otel.erl | 207 ++++++++++++++++++ apps/emqx_opentelemetry/src/emqx_otel_api.erl | 112 ++++++++++ apps/emqx_opentelemetry/src/emqx_otel_app.erl | 29 +++ .../src/emqx_otel_config.erl | 58 +++++ .../src/emqx_otel_schema.erl | 82 +++++++ apps/emqx_opentelemetry/src/emqx_otel_sup.erl | 67 ++++++ .../src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 26 +-- mix.exs | 28 ++- rebar.config | 8 + rel/i18n/emqx_otel_schema.hocon | 15 ++ 20 files changed, 710 insertions(+), 38 deletions(-) create mode 100644 apps/emqx_opentelemetry/README.md create mode 100644 apps/emqx_opentelemetry/etc/emqx_otel.conf create mode 100644 apps/emqx_opentelemetry/rebar.config create mode 100644 apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src create mode 100644 apps/emqx_opentelemetry/src/emqx_otel.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_api.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_app.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_config.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_schema.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_sup.erl create mode 100644 rel/i18n/emqx_otel_schema.hocon diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index ef9109e33..e590577da 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -37,7 +37,8 @@ setstat/2, setstat/3, statsfun/1, - statsfun/2 + statsfun/2, + names/0 ]). -export([ @@ -157,6 +158,28 @@ getstats() -> _ -> ets:tab2list(?TAB) end. +names() -> + [ + emqx_connections_count, + emqx_connections_max, + emqx_live_connections_count, + emqx_live_connections_max, + emqx_sessions_count, + emqx_sessions_max, + emqx_topics_count, + emqx_topics_max, + emqx_suboptions_count, + emqx_suboptions_max, + emqx_subscribers_count, + emqx_subscribers_max, + emqx_subscriptions_count, + emqx_subscriptions_max, + emqx_subscriptions_shared_count, + emqx_subscriptions_shared_max, + emqx_retained_count, + emqx_retained_max + ]. + %% @doc Get stats by name. -spec getstat(atom()) -> non_neg_integer(). getstat(Name) -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index eea2bf1b8..246f36f41 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -63,6 +63,7 @@ emqx_psk_schema, emqx_limiter_schema, emqx_slow_subs_schema, + emqx_otel_schema, emqx_mgmt_api_key_schema ]). %% 1 million default ports counter diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 51c2d2274..ac738ae3c 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -24,7 +24,12 @@ redbug, xmerl, {hocon, load}, - telemetry + telemetry, + {opentelemetry, load}, + {opentelemetry_api, load}, + {opentelemetry_experimental, load}, + {opentelemetry_api_experimental, load}, + {opentelemetry_exporter, load} ], %% must always be of type `load' common_business_apps => @@ -68,6 +73,7 @@ emqx_redis, emqx_mysql, emqx_plugins, + emqx_opentelemetry, quicer, bcrypt, jq, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 9e22cd375..0e2d37285 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.26"}, + {vsn, "5.0.27"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 2f261c0d5..059c323ff 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -107,7 +107,8 @@ %% Common Table API -export([ default_row_limit/0, - vm_stats/0 + vm_stats/0, + vm_stats/1 ]). -elvis([{elvis_style, god_modules, disable}]). @@ -185,22 +186,33 @@ stopped_node_info(Node) -> {Node, #{node => Node, node_status => 'stopped', role => core}}. vm_stats() -> - Idle = - case cpu_sup:util([detailed]) of - %% Not support for Windows - {_, 0, 0, _} -> 0; - {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) - end, - RunQueue = erlang:statistics(run_queue), + Idle = vm_stats('cpu.idle'), {MemUsedRatio, MemTotal} = get_sys_memory(), [ - {run_queue, RunQueue}, + {run_queue, vm_stats('run.queue')}, {cpu_idle, Idle}, {cpu_use, 100 - Idle}, {total_memory, MemTotal}, {used_memory, erlang:round(MemTotal * MemUsedRatio)} ]. +vm_stats('cpu.idle') -> + case cpu_sup:util([detailed]) of + %% Not support for Windows + {_, 0, 0, _} -> 0; + {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) + end; +vm_stats('cpu.use') -> + 100 - vm_stats('cpu.idle'); +vm_stats('total.memory') -> + {_, MemTotal} = get_sys_memory(), + MemTotal; +vm_stats('used.memory') -> + {MemUsedRatio, MemTotal} = get_sys_memory(), + erlang:round(MemTotal * MemUsedRatio); +vm_stats('run.queue') -> + erlang:statistics(run_queue). + %%-------------------------------------------------------------------- %% Brokers %%-------------------------------------------------------------------- diff --git a/apps/emqx_opentelemetry/README.md b/apps/emqx_opentelemetry/README.md new file mode 100644 index 000000000..d5d0b97ea --- /dev/null +++ b/apps/emqx_opentelemetry/README.md @@ -0,0 +1,4 @@ +emqx_opentelemetry +===== + +OpenTelemetry metric log trace framework for EMQX. diff --git a/apps/emqx_opentelemetry/etc/emqx_otel.conf b/apps/emqx_opentelemetry/etc/emqx_otel.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config new file mode 100644 index 000000000..7086a2f29 --- /dev/null +++ b/apps/emqx_opentelemetry/rebar.config @@ -0,0 +1,29 @@ +%% -*- mode: erlang -*- + +{deps, [ + {emqx, {path, "../emqx"}} +]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [ + warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform} +]}. + +{xref_checks, [ + undefined_function_calls, + undefined_functions, + locals_not_used, + deprecated_function_calls, + warnings_as_errors, + deprecated_functions +]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src new file mode 100644 index 000000000..adffd4c88 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -0,0 +1,15 @@ +{application, emqx_opentelemetry, [ + {description, "OpenTelemetry for EMQX Broker"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_otel_app, []}}, + {applications, [kernel, stdlib, emqx]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel.erl new file mode 100644 index 000000000..e17850b59 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel.erl @@ -0,0 +1,207 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_otel). +-include_lib("emqx/include/logger.hrl"). + +-export([start_link/1]). +-export([get_cluster_gauge/1, get_stats_gauge/1, get_vm_gauge/1, get_metric_counter/1]). +-export([init/1, handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +start_link(Conf) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). + +init(Conf) -> + erlang:process_flag(trap_exit, true), + {ok, #{}, {continue, {setup, Conf}}}. + +handle_continue({setup, Conf}, State) -> + setup(Conf), + {noreply, State, hibernate}. + +handle_call(_Msg, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Msg, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + cleanup(), + ok. + +setup(Conf = #{enable := true}) -> + ensure_apps(Conf), + create_metric_views(); +setup(_Conf) -> + cleanup(), + ok. + +ensure_apps(Conf) -> + #{exporter := #{interval := ExporterInterval}} = Conf, + {ok, _} = application:ensure_all_started(opentelemetry_exporter), + _ = application:stop(opentelemetry_experimental), + ok = application:set_env( + opentelemetry_experimental, + readers, + [ + #{ + module => otel_metric_reader, + config => #{ + exporter => {opentelemetry_exporter, #{}}, + export_interval_ms => ExporterInterval + } + } + ] + ), + {ok, _} = application:ensure_all_started(opentelemetry_experimental), + {ok, _} = application:ensure_all_started(opentelemetry_api_experimental), + ok. + +cleanup() -> + _ = application:stop(opentelemetry_experimental), + _ = application:stop(opentelemetry_experimental_api), + _ = application:stop(opentelemetry_exporter), + ok. + +create_metric_views() -> + Meter = opentelemetry_experimental:get_meter(), + StatsGauge = emqx_stats:getstats(), + create_gauge(Meter, StatsGauge, fun ?MODULE:get_stats_gauge/1), + VmGauge = lists:map(fun({K, V}) -> {normalize_name(K), V} end, emqx_mgmt:vm_stats()), + create_gauge(Meter, VmGauge, fun ?MODULE:get_vm_gauge/1), + ClusterGauge = [{'node.running', 0}, {'node.stopped', 0}], + create_gauge(Meter, ClusterGauge, fun ?MODULE:get_cluster_gauge/1), + Metrics = lists:map(fun({K, V}) -> {K, V, unit(K)} end, emqx_metrics:all()), + create_counter(Meter, Metrics, fun ?MODULE:get_metric_counter/1), + ok. + +unit(K) -> + case lists:member(K, bytes_metrics()) of + true -> kb; + false -> '1' + end. + +bytes_metrics() -> + [ + 'bytes.received', + 'bytes.sent', + 'packets.received', + 'packets.sent', + 'packets.connect', + 'packets.connack.sent', + 'packets.connack.error', + 'packets.connack.auth_error', + 'packets.publish.received', + 'packets.publish.sent', + 'packets.publish.inuse', + 'packets.publish.error', + 'packets.publish.auth_error', + 'packets.publish.dropped', + 'packets.puback.received', + 'packets.puback.sent', + 'packets.puback.inuse', + 'packets.puback.missed', + 'packets.pubrec.received', + 'packets.pubrec.sent', + 'packets.pubrec.inuse', + 'packets.pubrec.missed', + 'packets.pubrel.received', + 'packets.pubrel.sent', + 'packets.pubrel.missed', + 'packets.pubcomp.received', + 'packets.pubcomp.sent', + 'packets.pubcomp.inuse', + 'packets.pubcomp.missed', + 'packets.subscribe.received', + 'packets.subscribe.error', + 'packets.subscribe.auth_error', + 'packets.suback.sent', + 'packets.unsubscribe.received', + 'packets.unsubscribe.error', + 'packets.unsuback.sent', + 'packets.pingreq.received', + 'packets.pingresp.sent', + 'packets.disconnect.received', + 'packets.disconnect.sent', + 'packets.auth.received', + 'packets.auth.sent' + ]. + +get_stats_gauge(Name) -> + [{emqx_stats:getstat(Name), #{}}]. + +get_vm_gauge(Name) -> + [{emqx_mgmt:vm_stats(Name), #{}}]. + +get_cluster_gauge('node.running') -> + length(emqx:cluster_nodes(running)); +get_cluster_gauge('node.stopped') -> + length(emqx:cluster_nodes(stopped)). + +get_metric_counter(Name) -> + [{emqx_metrics:val(Name), #{}}]. + +create_gauge(Meter, Names, CallBack) -> + lists:foreach( + fun({Name, _}) -> + true = otel_meter_server:add_view( + #{instrument_name => Name}, + #{aggregation_module => otel_aggregation_last_value} + ), + otel_meter:create_observable_gauge( + Meter, + Name, + CallBack, + Name, + #{ + description => iolist_to_binary([ + <<"observable ">>, atom_to_binary(Name), <<" gauge">> + ]), + unit => '1' + } + ) + end, + Names + ). + +create_counter(Meter, Counters, CallBack) -> + lists:foreach( + fun({Name, _, Unit}) -> + true = otel_meter_server:add_view( + #{instrument_name => Name}, + #{aggregation_module => otel_aggregation_sum} + ), + otel_meter:create_observable_counter( + Meter, + Name, + CallBack, + Name, + #{ + description => iolist_to_binary([ + <<"observable ">>, atom_to_binary(Name), <<" counter">> + ]), + unit => Unit + } + ) + end, + Counters + ). + +normalize_name(Name) -> + list_to_existing_atom(lists:flatten(string:replace(atom_to_list(Name), "_", ".", all))). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_api.erl b/apps/emqx_opentelemetry/src/emqx_otel_api.erl new file mode 100644 index 000000000..7478859f8 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_api.erl @@ -0,0 +1,112 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_otel_api). + +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/http_api.hrl"). + +-import(hoconsc, [ref/2]). + +-export([ + api_spec/0, + paths/0, + schema/1 +]). + +-export([config/2]). + +-define(TAGS, [<<"Monitor">>]). + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/opentelemetry" + ]. + +schema("/opentelemetry") -> + #{ + 'operationId' => config, + get => + #{ + description => "Get opentelmetry configuration", + tags => ?TAGS, + responses => + #{200 => otel_config_schema()} + }, + put => + #{ + description => "Update opentelmetry configuration", + tags => ?TAGS, + 'requestBody' => otel_config_schema(), + responses => + #{ + 200 => otel_config_schema(), + 400 => + emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Update Config Failed">> + ) + } + } + }. + +%%-------------------------------------------------------------------- +%% API Handler funcs +%%-------------------------------------------------------------------- + +config(get, _Params) -> + {200, get_raw()}; +config(put, #{body := Body}) -> + case emqx_otel_config:update(Body) of + {ok, NewConfig} -> + {200, NewConfig}; + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), + {400, ?BAD_REQUEST, Message} + end. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +get_raw() -> + Path = <<"opentelemetry">>, + #{Path := Conf} = + emqx_config:fill_defaults( + #{Path => emqx_conf:get_raw([Path])}, + #{obfuscate_sensitive_values => true} + ), + Conf. + +otel_config_schema() -> + emqx_dashboard_swagger:schema_with_example( + ref(emqx_otel_schema, "opentelemetry"), + otel_config_example() + ). + +otel_config_example() -> + #{ + enable => true, + exporter => + #{ + endpoint => "http://localhost:4317", + interval => "10s" + } + }. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_app.erl b/apps/emqx_opentelemetry/src/emqx_otel_app.erl new file mode 100644 index 000000000..f028a000a --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_app.erl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_otel_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_otel_config:add_handler(), + emqx_otel_sup:start_link(). + +stop(_State) -> + emqx_otel_config:remove_handler(), + ok. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl new file mode 100644 index 000000000..3df535890 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_config). + +-behaviour(emqx_config_handler). + +-define(OPTL, [opentelemetry]). + +-export([add_handler/0, remove_handler/0]). +-export([post_config_update/5]). +-export([update/1]). + +update(Config) -> + case + emqx_conf:update( + ?OPTL, + Config, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + {ok, NewConfigRows}; + {error, Reason} -> + {error, Reason} + end. + +add_handler() -> + ok = emqx_config_handler:add_handler(?OPTL, ?MODULE), + ok. + +remove_handler() -> + ok = emqx_config_handler:remove_handler(?OPTL), + ok. + +post_config_update(?OPTL, _Req, New, _Old, AppEnvs) -> + application:set_env(AppEnvs), + ensure_otel(New); +post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> + ok. + +ensure_otel(#{enable := true} = Conf) -> + _ = emqx_otel_sup:stop_otel(), + emqx_otel_sup:start_otel(Conf); +ensure_otel(#{enable := false}) -> + emqx_otel_sup:stop_otel(). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl new file mode 100644 index 000000000..1479009a2 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -0,0 +1,82 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_schema). + +-include_lib("hocon/include/hoconsc.hrl"). + +-export([ + roots/0, + fields/1, + namespace/0, + desc/1 +]). + +namespace() -> opentelemetry. +roots() -> ["opentelemetry"]. + +fields("opentelemetry") -> + [ + {exporter, + ?HOCON( + ?R_REF("exporter"), + #{desc => ?DESC(exporter)} + )}, + {enable, + ?HOCON( + boolean(), + #{ + default => false, + required => true, + desc => ?DESC(enable) + } + )} + ]; +fields("exporter") -> + [ + {"protocol", + ?HOCON( + %% http_protobuf is not support for metrics yet. + ?ENUM([grpc]), + #{ + mapping => "opentelemetry_exporter.otlp_protocol", + desc => ?DESC(protocol), + default => grpc, + importance => ?IMPORTANCE_HIDDEN + } + )}, + {"endpoint", + ?HOCON( + emqx_schema:url(), + #{ + mapping => "opentelemetry_exporter.otlp_endpoint", + default => "http://localhost:4317", + desc => ?DESC(endpoint) + } + )}, + {"interval", + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"10s">>, + required => true, + desc => ?DESC(interval) + } + )} + ]. + +desc("opentelemetry") -> ?DESC(opentelemetry); +desc("exporter") -> ?DESC(exporter); +desc(_) -> undefined. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl new file mode 100644 index 000000000..2240cca03 --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl @@ -0,0 +1,67 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_sup). + +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). +-export([start_otel/1]). +-export([stop_otel/0]). + +-define(CHILD(Mod, Opts), #{ + id => Mod, + start => {Mod, start_link, [Opts]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod] +}). + +-define(WORKER, emqx_otel). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start_otel(map()) -> ok. +start_otel(Conf) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(?WORKER, Conf))). + +-spec stop_otel() -> ok | {error, term()}. +stop_otel() -> + case supervisor:terminate_child(?MODULE, ?WORKER) of + ok -> supervisor:delete_child(?MODULE, ?WORKER); + {error, not_found} -> ok; + Error -> Error + end. + +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 512 + }, + Children = + case emqx_conf:get([opentelemetry]) of + #{enable := false} -> []; + #{enable := true} = Conf -> [?CHILD(?WORKER, Conf)] + end, + {ok, {SupFlags, Children}}. + +assert_started({ok, _Pid}) -> ok; +assert_started({ok, _Pid, _Info}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; +assert_started({error, Reason}) -> {error, Reason}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index e6ee145ff..10fd75e98 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index d999f294e..ac902ca55 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -160,7 +160,7 @@ collect_mf(_Registry, Callback) -> Stats = emqx_stats:getstats(), VMData = emqx_vm_data(), ClusterData = emqx_cluster_data(), - _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats()], + _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats:names()], _ = [add_collect_family(Name, VMData, Callback, gauge) || Name <- emqx_vm()], _ = [add_collect_family(Name, ClusterData, Callback, gauge) || Name <- emqx_cluster()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_packets()], @@ -176,7 +176,7 @@ collect(<<"json">>) -> Stats = emqx_stats:getstats(), VMData = emqx_vm_data(), #{ - stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats()]), + stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats:names()]), metrics => maps:from_list([collect_stats(Name, VMData) || Name <- emqx_vm()]), packets => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]), messages => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]), @@ -460,28 +460,6 @@ emqx_collect(emqx_cluster_nodes_stopped, ClusterData) -> %% Indicators %%-------------------------------------------------------------------- -emqx_stats() -> - [ - emqx_connections_count, - emqx_connections_max, - emqx_live_connections_count, - emqx_live_connections_max, - emqx_sessions_count, - emqx_sessions_max, - emqx_topics_count, - emqx_topics_max, - emqx_suboptions_count, - emqx_suboptions_max, - emqx_subscribers_count, - emqx_subscribers_max, - emqx_subscriptions_count, - emqx_subscriptions_max, - emqx_subscriptions_shared_count, - emqx_subscriptions_shared_max, - emqx_retained_count, - emqx_retained_max - ]. - emqx_metrics_packets() -> [ emqx_bytes_received, diff --git a/mix.exs b/mix.exs index 00d190136..956e5545d 100644 --- a/mix.exs +++ b/mix.exs @@ -98,7 +98,32 @@ defmodule EMQXUmbrella.MixProject do # set by hackney (dependency) {:ssl_verify_fun, "1.1.6", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, - {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true} + {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, + {:opentelemetry_api, + github: "emqx/opentelemetry-erlang", + sparse: "apps/opentelemetry_api", + override: true, + runtime: false}, + {:opentelemetry, + github: "emqx/opentelemetry-erlang", + sparse: "apps/opentelemetry", + override: true, + runtime: false}, + {:opentelemetry_api_experimental, + github: "emqx/opentelemetry-erlang", + sparse: "apps/opentelemetry_api_experimental", + override: true, + runtime: false}, + {:opentelemetry_experimental, + github: "emqx/opentelemetry-erlang", + sparse: "apps/opentelemetry_experimental", + override: true, + runtime: false}, + {:opentelemetry_exporter, + github: "emqx/opentelemetry-erlang", + sparse: "apps/opentelemetry_exporter", + override: true, + runtime: false} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() @@ -324,6 +349,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_plugins, :emqx_ft, :emqx_s3, + :emqx_opentelemetry, :emqx_durable_storage, :rabbit_common ], diff --git a/rebar.config b/rebar.config index 131149f47..dd1e139f8 100644 --- a/rebar.config +++ b/rebar.config @@ -84,6 +84,14 @@ %% in conflict by erlavro and rocketmq , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} +%% trace + , {opentelemetry_api, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry"}} + %% log metrics + , {opentelemetry_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api_experimental"}} + %% export + , {opentelemetry_exporter, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_exporter"}} ]}. {xref_ignores, diff --git a/rel/i18n/emqx_otel_schema.hocon b/rel/i18n/emqx_otel_schema.hocon new file mode 100644 index 000000000..f662598b9 --- /dev/null +++ b/rel/i18n/emqx_otel_schema.hocon @@ -0,0 +1,15 @@ +emqx_otel_schema { + +opentelemetry.desc: "Open Telemetry Toolkit configuration" + +exporter.desc: "Open Telemetry Exporter" + +enable.desc: "Enable or disable open telemetry metrics" + +protocol.desc: "Open Telemetry Exporter Protocol" + +endpoint.desc: "Open Telemetry Exporter Endpoint" + +interval.desc: "The interval of sending metrics to Open Telemetry Endpoint" + +} From d5fe89196172a145162fd8d280b88cae4fc9bf63 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 17 Jul 2023 22:21:30 +0800 Subject: [PATCH 4/8] fix: don't be kill when send large payload when log is debug --- apps/emqx/include/emqx_mqtt.hrl | 3 + apps/emqx/src/emqx_packet.erl | 87 +++---------------- .../src/emqx_trace/emqx_trace_formatter.erl | 19 +++- apps/emqx/test/emqx_trace_SUITE.erl | 1 - .../test/emqx_mgmt_api_trace_SUITE.erl | 1 - 5 files changed, 32 insertions(+), 79 deletions(-) diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 2bd27339c..d8922fb98 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -679,4 +679,7 @@ end). -define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})). -define(THROW_SERIALIZE_ERROR(Reason), erlang:throw({?FRAME_SERIALIZE_ERROR, Reason})). +-define(MAX_PAYLOAD_FORMAT_SIZE, 1024). +-define(MAX_PAYLOAD_FORMAT_LIMIT(Bin), (byte_size(Bin) =< ?MAX_PAYLOAD_FORMAT_SIZE)). + -endif. diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 96eacc5a9..e3e2229d9 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -55,8 +55,6 @@ format/2 ]). --export([encode_hex/1]). - -define(TYPE_NAMES, {'CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL', 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK', 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH'} @@ -616,9 +614,20 @@ format_password(undefined) -> ""; format_password(<<>>) -> ""; format_password(_Password) -> "******". -format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])]; -format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)]; -format_payload(_, hidden) -> "Payload=******". +format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> + ["Payload=", unicode:characters_to_list(Payload)]; +format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> + ["Payload(hex)=", binary:encode_hex(Payload)]; +format_payload(_, hidden) -> + "Payload=******"; +format_payload(<> = Payload, _) -> + [ + "Payload=", + Part, + "...The ", + integer_to_list(byte_size(Payload) - ?MAX_PAYLOAD_FORMAT_SIZE), + "bytes of this log are truncated" + ]. i(true) -> 1; i(false) -> 0; @@ -641,71 +650,3 @@ format_topic_filters(Filters) -> ), "]" ]. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Hex encoding functions -%% Copy from binary:encode_hex/1 (was only introduced in OTP24). -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --define(HEX(X), (hex(X)):16). --compile({inline, [hex/1]}). --spec encode_hex(Bin) -> Bin2 when - Bin :: binary(), - Bin2 :: <<_:_*16>>. -encode_hex(Data) when byte_size(Data) rem 8 =:= 0 -> - << - <> - || <> <= Data - >>; -encode_hex(Data) when byte_size(Data) rem 7 =:= 0 -> - << - <> - || <> <= Data - >>; -encode_hex(Data) when byte_size(Data) rem 6 =:= 0 -> - <<<> || <> <= Data>>; -encode_hex(Data) when byte_size(Data) rem 5 =:= 0 -> - <<<> || <> <= Data>>; -encode_hex(Data) when byte_size(Data) rem 4 =:= 0 -> - <<<> || <> <= Data>>; -encode_hex(Data) when byte_size(Data) rem 3 =:= 0 -> - <<<> || <> <= Data>>; -encode_hex(Data) when byte_size(Data) rem 2 =:= 0 -> - <<<> || <> <= Data>>; -encode_hex(Data) when is_binary(Data) -> - <<<> || <> <= Data>>; -encode_hex(Bin) -> - erlang:error(badarg, [Bin]). - -hex(X) -> - element( - X + 1, - {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, - 16#3041, 16#3042, 16#3043, 16#3044, 16#3045, 16#3046, 16#3130, 16#3131, 16#3132, - 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141, 16#3142, - 16#3143, 16#3144, 16#3145, 16#3146, 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, - 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241, 16#3242, 16#3243, 16#3244, - 16#3245, 16#3246, 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, - 16#3337, 16#3338, 16#3339, 16#3341, 16#3342, 16#3343, 16#3344, 16#3345, 16#3346, - 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, - 16#3439, 16#3441, 16#3442, 16#3443, 16#3444, 16#3445, 16#3446, 16#3530, 16#3531, - 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541, - 16#3542, 16#3543, 16#3544, 16#3545, 16#3546, 16#3630, 16#3631, 16#3632, 16#3633, - 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641, 16#3642, 16#3643, - 16#3644, 16#3645, 16#3646, 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, - 16#3736, 16#3737, 16#3738, 16#3739, 16#3741, 16#3742, 16#3743, 16#3744, 16#3745, - 16#3746, 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, - 16#3838, 16#3839, 16#3841, 16#3842, 16#3843, 16#3844, 16#3845, 16#3846, 16#3930, - 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, - 16#3941, 16#3942, 16#3943, 16#3944, 16#3945, 16#3946, 16#4130, 16#4131, 16#4132, - 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141, 16#4142, - 16#4143, 16#4144, 16#4145, 16#4146, 16#4230, 16#4231, 16#4232, 16#4233, 16#4234, - 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241, 16#4242, 16#4243, 16#4244, - 16#4245, 16#4246, 16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, - 16#4337, 16#4338, 16#4339, 16#4341, 16#4342, 16#4343, 16#4344, 16#4345, 16#4346, - 16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, - 16#4439, 16#4441, 16#4442, 16#4443, 16#4444, 16#4445, 16#4446, 16#4530, 16#4531, - 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541, - 16#4542, 16#4543, 16#4544, 16#4545, 16#4546, 16#4630, 16#4631, 16#4632, 16#4633, - 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641, 16#4642, 16#4643, - 16#4644, 16#4645, 16#4646} - ). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index a44237bd0..843b502d5 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -14,6 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_trace_formatter). +-include("emqx_mqtt.hrl"). -export([format/2]). -export([format_meta_map/1]). @@ -68,10 +69,20 @@ weight({K, _}) -> {1, K}. format_packet(undefined, _) -> ""; format_packet(Packet, Encode) -> emqx_packet:format(Packet, Encode). -format_payload(undefined, _) -> ""; -format_payload(Payload, text) -> io_lib:format("~ts", [Payload]); -format_payload(Payload, hex) -> emqx_packet:encode_hex(Payload); -format_payload(_, hidden) -> "******". +format_payload(undefined, _) -> + ""; +format_payload(_, hidden) -> + "******"; +format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> + unicode:characters_to_list(Payload); +format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> binary:encode_hex(Payload); +format_payload(<> = Payload, _) -> + [ + Part, + "...The ", + integer_to_list(byte_size(Payload) - ?MAX_PAYLOAD_FORMAT_SIZE), + "bytes of this log are truncated" + ]. to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); to_iolist(Int) when is_integer(Int) -> integer_to_list(Int); diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index 0166613a4..1bbe084fd 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -274,7 +274,6 @@ t_load_state(_Config) -> ok. t_client_event(_Config) -> - application:set_env(emqx, allow_anonymous, true), ClientId = <<"client-test">>, Now = erlang:system_time(second), Name = <<"test_client_id_event">>, diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 0102eb56c..8f9a4a5ca 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -269,7 +269,6 @@ create_trace(Name, ClientId, Start) -> ). t_stream_log(_Config) -> - application:set_env(emqx, allow_anonymous, true), emqx_trace:clear(), load(), ClientId = <<"client-stream">>, From 5a4dd3a5e523378ae9718c086940a674220eca17 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 10 Aug 2023 09:35:12 +0800 Subject: [PATCH 5/8] fix: truncate large payload --- apps/emqx/src/emqx_packet.erl | 8 ++++---- apps/emqx/src/emqx_trace/emqx_trace_formatter.erl | 8 ++++---- changes/ce/fix-11279.en.md | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changes/ce/fix-11279.en.md diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index e3e2229d9..1f7aaa4c9 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -620,13 +620,13 @@ format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> ["Payload(hex)=", binary:encode_hex(Payload)]; format_payload(_, hidden) -> "Payload=******"; -format_payload(<> = Payload, _) -> +format_payload(<> = Payload, _) -> [ "Payload=", Part, - "...The ", - integer_to_list(byte_size(Payload) - ?MAX_PAYLOAD_FORMAT_SIZE), - "bytes of this log are truncated" + "... The ", + integer_to_list(byte_size(Payload) - 100), + " bytes of this log are truncated" ]. i(true) -> 1; diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 843b502d5..42623e91a 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -76,12 +76,12 @@ format_payload(_, hidden) -> format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> unicode:characters_to_list(Payload); format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> binary:encode_hex(Payload); -format_payload(<> = Payload, _) -> +format_payload(<> = Payload, _) -> [ Part, - "...The ", - integer_to_list(byte_size(Payload) - ?MAX_PAYLOAD_FORMAT_SIZE), - "bytes of this log are truncated" + "... The ", + integer_to_list(byte_size(Payload) - 100), + " bytes of this log are truncated" ]. to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom); diff --git a/changes/ce/fix-11279.en.md b/changes/ce/fix-11279.en.md new file mode 100644 index 000000000..9f56bf543 --- /dev/null +++ b/changes/ce/fix-11279.en.md @@ -0,0 +1 @@ +Prevent client disconnected when sending large payloads with debug/trace logging is enabled. From d6476481c9e3a0e78e71c6b0d69f3e4dc3fae197 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 10 Aug 2023 07:58:57 +0200 Subject: [PATCH 6/8] ci(packages): run publish_artifacts on github hosted runner to get sudo --- .github/workflows/build_packages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cbe95b974..d33d46f11 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -264,7 +264,7 @@ jobs: path: _packages/${{ matrix.profile }}/ publish_artifacts: - runs-on: ${{ inputs.runner }} + runs-on: ubuntu-latest needs: - mac - linux @@ -280,7 +280,7 @@ jobs: name: ${{ matrix.profile }} path: packages/${{ matrix.profile }} - name: install dos2unix - run: apt-get update && apt install -y dos2unix + run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | set -eu @@ -300,7 +300,7 @@ jobs: env: PROFILE: ${{ matrix.profile }} run: | - set -e -u + set -eu if [ $PROFILE = 'emqx' ]; then s3dir='emqx-ce' elif [ $PROFILE = 'emqx-enterprise' ]; then From 1c5b42806bd6938e4a28269dd505f6a94d361e6c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 10 Aug 2023 08:49:49 +0200 Subject: [PATCH 7/8] build: correctly parse package version --- build | 12 ++++++------ scripts/semver.sh | 29 +++++++++++++++++++++++++++++ scripts/shelltest/run_tests.sh | 4 ++-- scripts/shelltest/semver.test | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) create mode 100755 scripts/semver.sh create mode 100644 scripts/shelltest/semver.test diff --git a/build b/build index 50b3fd861..34f7e8edb 100755 --- a/build +++ b/build @@ -378,11 +378,11 @@ make_docker() { local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}" local PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" # shellcheck disable=SC2155 - local VSN_MAJOR="$(echo "$PKG_VSN" | cut -d . -f 1)" + local VSN_MAJOR="$(scripts/semver.sh "$PKG_VSN" --major)" # shellcheck disable=SC2155 - local VSN_MINOR="$(echo "$PKG_VSN" | cut -d . -f 2)" + local VSN_MINOR="$(scripts/semver.sh "$PKG_VSN" --minor)" # shellcheck disable=SC2155 - local VSN_PATCH="$(echo "$PKG_VSN" | cut -d . -f 3)" + local VSN_MINOR="$(scripts/semver.sh "$PKG_VSN" --patch)" local SUFFIX='' if [[ "$PROFILE" = *-elixir ]]; then SUFFIX="-elixir" @@ -430,8 +430,6 @@ make_docker() { --label org.opencontainers.image.licenses="${LICENSE}" \ --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \ --tag "${EMQX_IMAGE_TAG}" \ - --tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" \ - --tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" \ --provenance false \ --pull ) @@ -442,7 +440,9 @@ make_docker() { DOCKER_BUILDX_ARGS+=(--label org.opencontainers.image.elixir.version="${EMQX_BUILDER_ELIXIR}") fi if [ "${DOCKER_LATEST:-false}" = true ]; then - DOCKER_BUILDX_ARGS+=(--tag "${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE}:latest${SUFFIX}") + DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}") + DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}") + DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}") fi if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then DOCKER_BUILDX_ARGS+=(--platform "${DOCKER_PLATFORMS}") diff --git a/scripts/semver.sh b/scripts/semver.sh new file mode 100755 index 000000000..065241355 --- /dev/null +++ b/scripts/semver.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +function parseSemver() { + local RE='^([0-9]+)\.([0-9]+)\.([0-9]+)(-([a-z]+\.[0-9]+))?$' + echo "$1" | grep -qE "$RE" || exit 1 + #shellcheck disable=SC2155 + local MAJOR=$( echo "$1" | sed -r "s#$RE#\1#") + #shellcheck disable=SC2155 + local MINOR=$( echo "$1" | sed -r "s#$RE#\2#") + #shellcheck disable=SC2155 + local PATCH=$( echo "$1" | sed -r "s#$RE#\3#") + #shellcheck disable=SC2155 + local SPECIAL=$(echo "$1" | sed -r "s#$RE#\5#") + case "${2}" in + --major) echo "${MAJOR}" ;; + --minor) echo "${MINOR}" ;; + --patch) echo "${PATCH}" ;; + --special) echo "${SPECIAL}" ;; + *) + cat <>>= 1 + +./semver.sh 5.1.0 +>>> +{"major": 5, "minor": 1, "patch": 0, "special": ""} +>>>= 0 + +./semver.sh 5.1.0-patch.3 +>>> +{"major": 5, "minor": 1, "patch": 0, "special": "patch.3"} +>>>= 0 + +./semver.sh 5.1.0-patch.3 --major +>>> +5 +>>>= 0 + +./semver.sh 5.1.0-patch.3 --minor +>>> +1 +>>>= 0 + +./semver.sh 5.1.0-patch.3 --patch +>>> +0 +>>>= 0 + +./semver.sh 5.1.0-patch.3 --special +>>> +patch.3 +>>>= 0 From 9ad44f1bba3f34ded7f6a0076b4d0f81e40de8b0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 10 Aug 2023 09:57:53 +0200 Subject: [PATCH 8/8] ci: do not run tests on tag push --- .github/workflows/_push-entrypoint.yaml | 32 ++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index a65df1234..32f3ae15a 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -82,20 +82,8 @@ jobs: echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT - build_slim_packages: - if: ${{ needs.prepare.outputs.release != 'true' }} - needs: - - prepare - uses: ./.github/workflows/build_slim_packages.yaml - with: - runner: ${{ needs.prepare.outputs.runner }} - builder: ${{ needs.prepare.outputs.builder }} - builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} - otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} - elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} - build_packages: - if: ${{ needs.prepare.outputs.release == 'true' }} + if: needs.prepare.outputs.release == 'true' needs: - prepare uses: ./.github/workflows/build_packages.yaml @@ -109,7 +97,7 @@ jobs: secrets: inherit build_and_push_docker_images: - if: ${{ needs.prepare.outputs.release == 'true' }} + if: needs.prepare.outputs.release == 'true' needs: - prepare uses: ./.github/workflows/build_and_push_docker_images.yaml @@ -124,7 +112,20 @@ jobs: runner: ${{ needs.prepare.outputs.runner }} secrets: inherit + build_slim_packages: + if: needs.prepare.outputs.release != 'true' + needs: + - prepare + uses: ./.github/workflows/build_slim_packages.yaml + with: + runner: ${{ needs.prepare.outputs.runner }} + builder: ${{ needs.prepare.outputs.builder }} + builder_vsn: ${{ needs.prepare.outputs.builder_vsn }} + otp_vsn: ${{ needs.prepare.outputs.otp_vsn }} + elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }} + compile: + if: needs.prepare.outputs.release != 'true' runs-on: ${{ needs.prepare.outputs.runner }} container: ${{ needs.prepare.outputs.builder }} needs: @@ -157,6 +158,7 @@ jobs: retention-days: 1 run_test_cases: + if: needs.prepare.outputs.release != 'true' needs: - prepare - compile @@ -169,6 +171,7 @@ jobs: ct-docker: ${{ needs.prepare.outputs.ct-docker }} run_conf_tests: + if: needs.prepare.outputs.release != 'true' needs: - prepare - compile @@ -178,6 +181,7 @@ jobs: builder: ${{ needs.prepare.outputs.builder }} static_checks: + if: needs.prepare.outputs.release != 'true' needs: - prepare - compile