From 9b4d9608885d039fa4203ddc3e7d893721b45b1a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 28 Mar 2024 13:52:48 +0100 Subject: [PATCH 01/86] chore: prepare for 5.7.x releases --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- scripts/rel/cut.sh | 6 ++++++ scripts/rel/sync-remotes.sh | 8 ++++++-- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 9679510d9..18e059422 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.6.0"). +-define(EMQX_RELEASE_CE, "5.7.0-alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.6.0"). +-define(EMQX_RELEASE_EE, "5.7.0-alpha.1"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 0fd47100b..2f363bdfc 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.6.0 +version: 5.7.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.6.0 +appVersion: 5.7.0-alpha.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index b1ae7ff66..60498d5d7 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.6.0 +version: 5.7.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.6.0 +appVersion: 5.7.0-alpha.1 diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 95cd7a1d4..724b0cab2 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -129,6 +129,12 @@ rel_branch() { e5.6.*) echo 'release-56' ;; + v5.7.*) + echo 'release-57' + ;; + e5.7.*) + echo 'release-57' + ;; *) logerr "Unsupported version tag $TAG" exit 1 diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index 6492c90c0..430021a79 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -5,7 +5,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." -BASE_BRANCHES=( 'release-56' 'release-55' 'master' ) +BASE_BRANCHES=( 'release-57' 'release-56' 'release-55' 'master' ) usage() { cat < Date: Wed, 24 Apr 2024 21:25:47 +0200 Subject: [PATCH 02/86] ci: refine dashboard_test --- scripts/ui-tests/dashboard_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/ui-tests/dashboard_test.py b/scripts/ui-tests/dashboard_test.py index 5f7ac8e15..7003802ab 100644 --- a/scripts/ui-tests/dashboard_test.py +++ b/scripts/ui-tests/dashboard_test.py @@ -9,6 +9,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.common import utils +from selenium.common.exceptions import NoSuchElementException @pytest.fixture def driver(): @@ -31,11 +32,13 @@ def dashboard_url(dashboard_host, dashboard_port): @pytest.fixture def login(driver, dashboard_url): + # admin is set in CI jobs, hence as default value + password = os.getenv("EMQX_DASHBOARD__DEFAULT_PASSWORD", "admin") driver.get(dashboard_url) assert "EMQX Dashboard" == driver.title assert f"{dashboard_url}/#/login?to=/dashboard/overview" == driver.current_url driver.find_element(By.XPATH, "//div[@class='login']//form[1]//input[@type='text']").send_keys("admin") - driver.find_element(By.XPATH, "//div[@class='login']//form[1]//input[@type='password']").send_keys("admin") + driver.find_element(By.XPATH, "//div[@class='login']//form[1]//input[@type='password']").send_keys(password) driver.find_element(By.XPATH, "//div[@class='login']//form[1]//button[1]").click() dest_url = urljoin(dashboard_url, "/#/dashboard/overview") driver.get(dest_url) @@ -91,8 +94,12 @@ def test_docs_link(driver, login, dashboard_url): else: emqx_version = f"v{emqx_community_version}" docs_base_url = "https://www.emqx.io/docs/en" - + emqx_version = ".".join(emqx_version.split(".")[:2]) docs_url = f"{docs_base_url}/{emqx_version}" xpath = f"//div[@id='app']//div[@class='nav-header']//a[@href[starts-with(.,'{docs_url}')]]" - assert driver.find_element(By.XPATH, xpath) + + try: + driver.find_element(By.XPATH, xpath) + except NoSuchElementException: + raise AssertionError(f"Cannot find the doc URL for {emqx_name} version {emqx_version}, please make sure the dashboard package is up to date.") From eaeaeb57d64718667711cd623f651665804b6d25 Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Apr 2024 09:05:54 +0200 Subject: [PATCH 03/86] chore: update dashboard versions --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2f01ad16e..b899017b4 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.8.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.6.1 +export EMQX_DASHBOARD_VERSION ?= v1.9.0-beta.1 +export EMQX_EE_DASHBOARD_VERSION ?= e1.7.0-beta.1 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise From 5ca90ccced49556ff9b842cb45f7aeab8b0656b2 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 09:43:11 +0200 Subject: [PATCH 04/86] fix: improve structure of log trace entries for HTTP action Fixes: https://emqx.atlassian.net/browse/EMQX-12025 --- .../emqx_trace/emqx_trace_json_formatter.erl | 15 +++++++++++++++ .../src/emqx_bridge_http_connector.erl | 17 +++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl index 35b09b9b0..8f748ed9f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl @@ -48,6 +48,21 @@ prepare_log_map(LogMap, PEncode) -> NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)], maps:from_list(NewKeyValuePairs). +prepare_key_value(K, {Formatter, V}, PEncode) when is_function(Formatter, 1) -> + %% A cusom formatter is provided with the value + try + NewV = Formatter(V), + prepare_key_value(K, NewV, PEncode) + catch + _:_ -> + {K, V} + end; +prepare_key_value(K, {ok, Status, Headers, Body}, PEncode) when + is_integer(Status), is_list(Headers), is_binary(Body) +-> + %% This is unlikely anything else then info about a HTTP request so we make + %% it more structured + prepare_key_value(K, #{status => Status, headers => Headers, body => Body}, PEncode); prepare_key_value(payload = K, V, PEncode) -> NewV = try diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 9be7457e1..76f2686a1 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -359,7 +359,7 @@ on_query(InstId, {Method, Request, Timeout}, State) -> on_query( InstId, {ActionId, KeyOrNum, Method, Request, Timeout, Retry}, - #{base_path := BasePath} = State + #{base_path := BasePath, host := Host} = State ) -> ?TRACE( "QUERY", @@ -373,7 +373,7 @@ on_query( } ), NRequest = formalize_request(Method, BasePath, Request), - trace_rendered_action_template(ActionId, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), Worker = resolve_pool_worker(State, KeyOrNum), Result0 = ehttpc:request( Worker, @@ -469,7 +469,7 @@ on_query_async( InstId, {ActionId, KeyOrNum, Method, Request, Timeout}, ReplyFunAndArgs, - #{base_path := BasePath} = State + #{base_path := BasePath, host := Host} = State ) -> Worker = resolve_pool_worker(State, KeyOrNum), ?TRACE( @@ -483,7 +483,7 @@ on_query_async( } ), NRequest = formalize_request(Method, BasePath, Request), - trace_rendered_action_template(ActionId, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), MaxAttempts = maps:get(max_attempts, State, 3), Context = #{ attempt => 1, @@ -503,12 +503,13 @@ on_query_async( ), {ok, Worker}. -trace_rendered_action_template(ActionId, Method, NRequest, Timeout) -> +trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> case NRequest of {Path, Headers} -> emqx_trace:rendered_action_template( ActionId, #{ + host => Host, path => Path, method => Method, headers => emqx_utils_redact:redact_headers(Headers), @@ -519,15 +520,19 @@ trace_rendered_action_template(ActionId, Method, NRequest, Timeout) -> emqx_trace:rendered_action_template( ActionId, #{ + host => Host, path => Path, method => Method, headers => emqx_utils_redact:redact_headers(Headers), timeout => Timeout, - body => Body + body => {fun log_format_body/1, Body} } ) end. +log_format_body(Body) -> + unicode:characters_to_binary(Body). + resolve_pool_worker(State, undefined) -> resolve_pool_worker(State, self()); resolve_pool_worker(#{pool_name := PoolName} = State, Key) -> From 1f676ce035e8be9b731ce7d7f35f0370341436be Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 10:36:05 +0200 Subject: [PATCH 05/86] feat: add stop after render and after render trace to mqtt action --- .../src/emqx_bridge_mqtt_connector.erl | 29 ++++++++-- .../src/emqx_bridge_mqtt_egress.erl | 55 ++++++++++++++----- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 900f6143f..d61950513 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -264,7 +264,7 @@ on_query( ), Channels = maps:get(installed_channels, State), ChannelConfig = maps:get(ChannelId, Channels), - handle_send_result(with_egress_client(PoolName, send, [Msg, ChannelConfig])); + handle_send_result(with_egress_client(ChannelId, PoolName, send, [Msg, ChannelConfig])); on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", @@ -283,7 +283,7 @@ on_query_async( Callback = {fun on_async_result/2, [CallbackIn]}, Channels = maps:get(installed_channels, State), ChannelConfig = maps:get(ChannelId, Channels), - Result = with_egress_client(PoolName, send_async, [Msg, Callback, ChannelConfig]), + Result = with_egress_client(ChannelId, PoolName, send_async, [Msg, Callback, ChannelConfig]), case Result of ok -> ok; @@ -300,8 +300,25 @@ on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> reason => "Egress is not configured" }). -with_egress_client(ResourceId, Fun, Args) -> - ecpool:pick_and_do(ResourceId, {emqx_bridge_mqtt_egress, Fun, Args}, no_handover). +with_egress_client(ActionID, ResourceId, Fun, Args) -> + LogMetaData = logger:get_process_metadata(), + TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ActionID}, + TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + ecpool:pick_and_do( + ResourceId, {emqx_bridge_mqtt_egress, Fun, [TraceRenderedFunc | Args]}, no_handover + ). + +trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> + OldMetaData = logger:get_process_metadata(), + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. on_async_result(Callback, Result) -> apply_callback_function(Callback, handle_send_result(Result)). @@ -322,7 +339,9 @@ handle_send_result({ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}}) -> handle_send_result({ok, Reply}) -> {error, classify_reply(Reply)}; handle_send_result({error, Reason}) -> - {error, classify_error(Reason)}. + {error, classify_error(Reason)}; +handle_send_result({unrecoverable_error, Reason}) -> + {error, {unrecoverable_error, Reason}}. classify_reply(Reply = #{reason_code := _}) -> {unrecoverable_error, Reply}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index d23899ef1..80d38bc78 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -22,13 +22,16 @@ -export([ config/1, - send/3, - send_async/4 + send/4, + send_async/5 ]). -type message() :: emqx_types:message() | map(). -type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. -type remote_message() :: #mqtt_msg{}. +-type trace_rendered_func() :: { + fun((RenderResult :: any(), CTX :: map()) -> any()), TraceCTX :: map() +}. -type egress() :: #{ local => #{ @@ -42,25 +45,37 @@ config(#{remote := RC = #{}} = Conf) -> Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}. --spec send(pid(), message(), egress()) -> ok. -send(Pid, MsgIn, Egress) -> - emqtt:publish(Pid, export_msg(MsgIn, Egress)). +-spec send(pid(), trace_rendered_func(), message(), egress()) -> ok. +send(Pid, TraceRenderedFunc, MsgIn, Egress) -> + try + emqtt:publish(Pid, export_msg(MsgIn, Egress, TraceRenderedFunc)) + catch + error:{unrecoverable_error, Reason} -> + {unrecoverable_error, Reason} + end. --spec send_async(pid(), message(), callback(), egress()) -> +-spec send_async(pid(), trace_rendered_func(), message(), callback(), egress()) -> ok | {ok, pid()}. -send_async(Pid, MsgIn, Callback, Egress) -> - ok = emqtt:publish_async(Pid, export_msg(MsgIn, Egress), _Timeout = infinity, Callback), - {ok, Pid}. +send_async(Pid, TraceRenderedFunc, MsgIn, Callback, Egress) -> + try + ok = emqtt:publish_async( + Pid, export_msg(MsgIn, Egress, TraceRenderedFunc), _Timeout = infinity, Callback + ), + {ok, Pid} + catch + error:{unrecoverable_error, Reason} -> + {unrecoverable_error, Reason} + end. -export_msg(Msg, #{remote := Remote}) -> - to_remote_msg(Msg, Remote). +export_msg(Msg, #{remote := Remote}, TraceRenderedFunc) -> + to_remote_msg(Msg, Remote, TraceRenderedFunc). --spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars()) -> +-spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars(), trace_rendered_func()) -> remote_message(). -to_remote_msg(#message{flags = Flags} = Msg, Vars) -> +to_remote_msg(#message{flags = Flags} = Msg, Vars, TraceRenderedFunc) -> {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), - to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars); -to_remote_msg(Msg = #{}, Remote) -> + to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars, TraceRenderedFunc); +to_remote_msg(Msg = #{}, Remote, {TraceRenderedFun, TraceRenderedCTX}) -> #{ topic := Topic, payload := Payload, @@ -68,6 +83,16 @@ to_remote_msg(Msg = #{}, Remote) -> retain := Retain } = emqx_bridge_mqtt_msg:render(Msg, Remote), PubProps = maps:get(pub_props, Msg, #{}), + TraceRenderedFun( + #{ + qos => QoS, + retain => Retain, + topic => Topic, + props => PubProps, + payload => Payload + }, + TraceRenderedCTX + ), #mqtt_msg{ qos = QoS, retain = Retain, From b02ed4e6ec0fe97c4bc04186d663560ffc113648 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 11:09:08 +0200 Subject: [PATCH 06/86] feat: add stop after render and after render trace to pgsql action --- apps/emqx/src/emqx_trace/emqx_trace.erl | 7 +++++-- apps/emqx_postgresql/src/emqx_postgresql.erl | 14 +++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 7bbe59b2b..4152fdbaa 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -87,7 +87,7 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -rendered_action_template(ActionID, RenderResult) -> +rendered_action_template(ActionID, RenderResult) when is_binary(ActionID) -> TraceResult = ?TRACE( "QUERY_RENDER", "action_template_rendered", @@ -111,7 +111,10 @@ rendered_action_template(ActionID, RenderResult) -> _ -> ok end, - TraceResult. + TraceResult; +rendered_action_template(_ActionID, _RenderResult) -> + %% We do nothing if we don't get a valid Action ID + ok. log(List, Msg, Meta) -> log(debug, List, Msg, Meta). diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index f27ec8615..761c9c0f6 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -304,7 +304,7 @@ on_query( }), Type = pgsql_query_type(TypeOrKey), {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), - Res = on_sql_query(InstId, PoolName, Type, NameOrSQL2, Data), + Res = on_sql_query(TypeOrKey, InstId, PoolName, Type, NameOrSQL2, Data), ?tp(postgres_bridge_connector_on_query_return, #{instance_id => InstId, result => Res}), handle_result(Res). @@ -337,7 +337,7 @@ on_batch_query( {_Statement, RowTemplate} -> PrepStatement = get_prepared_statement(BinKey, State), Rows = [render_prepare_sql_row(RowTemplate, Data) || {_Key, Data} <- BatchReq], - case on_sql_query(InstId, PoolName, execute_batch, PrepStatement, Rows) of + case on_sql_query(Key, InstId, PoolName, execute_batch, PrepStatement, Rows) of {error, _Error} = Result -> handle_result(Result); {_Column, Results} -> @@ -386,7 +386,15 @@ get_prepared_statement(Key, #{prepares := PrepStatements}) -> BinKey = to_bin(Key), maps:get(BinKey, PrepStatements). -on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> +on_sql_query(Key, InstId, PoolName, Type, NameOrSQL, Data) -> + emqx_trace:rendered_action_template( + Key, + #{ + statement_type => Type, + statement_or_name => NameOrSQL, + data => Data + } + ), try ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, no_handover) of {error, Reason} -> ?tp( From 32c27f171140f878810b696e8db2e8491b44b365 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 12:16:18 +0200 Subject: [PATCH 07/86] feat: add stop after render and after render trace to kafka action --- .../src/emqx_bridge_kafka_impl_producer.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 6bb1690ff..16bca153a 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -319,6 +319,9 @@ on_query( emqx_bridge_kafka_impl_producer_sync_query, #{headers_config => KafkaHeaders, instance_id => InstId} ), + emqx_trace:rendered_action_template(MessageTag, #{ + message => KafkaMessage, send_type => sync + }), do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) catch throw:{bad_kafka_header, _} = Error -> @@ -376,6 +379,9 @@ on_query_async( emqx_bridge_kafka_impl_producer_async_query, #{headers_config => KafkaHeaders, instance_id => InstId} ), + emqx_trace:rendered_action_template(MessageTag, #{ + message => KafkaMessage, send_type => async + }), do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) catch error:{invalid_partition_count, _Count, _Partitioner} -> From 7ad354f41210aeae82106f45d6ebb9d0d5603495 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 12:18:28 +0200 Subject: [PATCH 08/86] feat: add stop after render and after render trace to clickhouse action --- .../src/emqx_bridge_clickhouse_connector.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index 942f7590b..2c824aa95 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -386,7 +386,7 @@ on_query( SimplifiedRequestType = query_type(RequestType), Templates = get_templates(RequestType, State), SQL = get_sql(SimplifiedRequestType, Templates, DataOrSQL), - ClickhouseResult = execute_sql_in_clickhouse_server(PoolName, SQL), + ClickhouseResult = execute_sql_in_clickhouse_server(RequestType, PoolName, SQL), transform_and_log_clickhouse_result(ClickhouseResult, ResourceID, SQL). get_templates(ChannId, State) -> @@ -398,7 +398,7 @@ get_templates(ChannId, State) -> end. get_sql(channel_message, #{send_message_template := PreparedSQL}, Data) -> - emqx_placeholder:proc_tmpl(PreparedSQL, Data); + emqx_placeholder:proc_tmpl(PreparedSQL, Data, #{return => full_binary}); get_sql(_, _, SQL) -> SQL. @@ -425,7 +425,7 @@ on_batch_query(ResourceID, BatchReq, #{pool_name := PoolName} = State) -> %% Create batch insert SQL statement SQL = objects_to_sql(ObjectsToInsert, Templates), %% Do the actual query in the database - ResultFromClickhouse = execute_sql_in_clickhouse_server(PoolName, SQL), + ResultFromClickhouse = execute_sql_in_clickhouse_server(ChannId, PoolName, SQL), %% Transform the result to a better format transform_and_log_clickhouse_result(ResultFromClickhouse, ResourceID, SQL). @@ -464,7 +464,8 @@ objects_to_sql(_, _) -> %% This function is used by on_query/3 and on_batch_query/3 to send a query to %% the database server and receive a result -execute_sql_in_clickhouse_server(PoolName, SQL) -> +execute_sql_in_clickhouse_server(Id, PoolName, SQL) -> + emqx_trace:rendered_action_template(Id, #{rendered_sql => SQL}), ecpool:pick_and_do( PoolName, {?MODULE, execute_sql_in_clickhouse_server_using_connection, [SQL]}, From b2811f96b252c001bdfdf006ff3bd176993425a1 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 15:27:16 +0200 Subject: [PATCH 09/86] refactor(rule trace): simplify function for setting trace meta data This commit simplifies a function to set trace meta data in line with a suggestion from @zmstone: https://github.com/emqx/emqx/pull/12912#discussion_r1576053856 --- .../src/emqx_rule_runtime.erl | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index f99341a9b..e2f01321a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -141,22 +141,24 @@ apply_rule(Rule = #{id := RuleID}, Columns, Envs) -> set_process_trace_metadata(RuleID, #{clientid := ClientID} = Columns) -> logger:update_process_metadata(#{ - clientid => ClientID - }), - set_process_trace_metadata(RuleID, maps:remove(clientid, Columns)); + clientid => ClientID, + rule_id => RuleID, + rule_trigger_time => rule_trigger_time(Columns) + }); set_process_trace_metadata(RuleID, Columns) -> - EventTimestamp = - case Columns of - #{timestamp := Timestamp} -> - Timestamp; - _ -> - erlang:system_time(millisecond) - end, logger:update_process_metadata(#{ rule_id => RuleID, - rule_trigger_time => EventTimestamp + rule_trigger_time => rule_trigger_time(Columns) }). +rule_trigger_time(Columns) -> + case Columns of + #{timestamp := Timestamp} -> + Timestamp; + _ -> + erlang:system_time(millisecond) + end. + reset_process_trace_metadata(#{clientid := _ClientID}) -> Meta = logger:get_process_metadata(), Meta1 = maps:remove(clientid, Meta), From 120b35ac7521c34181915633f4b6d5e462429fe0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 15:54:46 +0200 Subject: [PATCH 10/86] feat: add stop after render and after render trace to mysql action --- .../emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl | 8 ++++++-- apps/emqx_mysql/src/emqx_mysql.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl index 6720e1fb7..da9377814 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -104,10 +104,12 @@ on_query( #{channels := Channels, connector_state := ConnectorState} ) when is_binary(Channel) -> ChannelConfig = maps:get(Channel, Channels), + MergedState0 = maps:merge(ConnectorState, ChannelConfig), + MergedState1 = MergedState0#{channel_id => Channel}, Result = emqx_mysql:on_query( InstanceId, Request, - maps:merge(ConnectorState, ChannelConfig) + MergedState1 ), ?tp(mysql_connector_on_query_return, #{instance_id => InstanceId, result => Result}), Result; @@ -121,10 +123,12 @@ on_batch_query( ) when is_binary(element(1, Req)) -> Channel = element(1, Req), ChannelConfig = maps:get(Channel, Channels), + MergedState0 = maps:merge(ConnectorState, ChannelConfig), + MergedState1 = MergedState0#{channel_id => Channel}, Result = emqx_mysql:on_batch_query( InstanceId, BatchRequest, - maps:merge(ConnectorState, ChannelConfig) + MergedState1 ), ?tp(mysql_connector_on_batch_query_return, #{instance_id => InstanceId, result => Result}), Result; diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index f23d7b092..9637cc473 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index e77965d67..ff851558a 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -498,6 +498,8 @@ on_sql_query( ) -> LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, ?TRACE("QUERY", "mysql_connector_received", LogMeta), + ChannelID = maps:get(channel_id, State, no_channel), + emqx_trace:rendered_action_template(ChannelID, #{sql => SQLOrKey}), Worker = ecpool:get_client(PoolName), case ecpool_worker:client(Worker) of {ok, Conn} -> From 810aa68b02ed1a3ca1646b28b20e74e4676532a9 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 17:58:55 +0200 Subject: [PATCH 11/86] feat: add stop after render and after render trace to dynamo action --- .../src/emqx_bridge_dynamo_connector.erl | 25 ++++++++++++++++++- .../emqx_bridge_dynamo_connector_client.erl | 17 +++++++------ .../src/emqx_bridge_mqtt_connector.erl | 6 ++++- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 372472dda..f9a87ccf7 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -246,12 +246,17 @@ do_query( table := Table, templates := Templates } = ChannelState, + LogMetaData = logger:get_process_metadata(), + TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, + TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, Result = case ensuare_dynamo_keys(Query, ChannelState) of true -> ecpool:pick_and_do( PoolName, - {emqx_bridge_dynamo_connector_client, query, [Table, QueryTuple, Templates]}, + {emqx_bridge_dynamo_connector_client, query, [ + Table, QueryTuple, Templates, TraceRenderedFunc + ]}, no_handover ); _ -> @@ -259,6 +264,8 @@ do_query( end, case Result of + {error, {unrecoverable_error, {action_stopped_after_template_rendering, _}}} = Error -> + Error; {error, Reason} -> ?tp( dynamo_connector_query_return, @@ -291,6 +298,22 @@ do_query( Result end. +trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. + get_channel_id([{ChannelId, _Req} | _]) -> ChannelId; get_channel_id({ChannelId, _Req}) -> diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 4f924ef67..1d1ad3760 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -10,7 +10,7 @@ -export([ start_link/1, is_connected/2, - query/4 + query/5 ]). %% gen_server callbacks @@ -40,8 +40,8 @@ is_connected(Pid, Timeout) -> {false, Error} end. -query(Pid, Table, Query, Templates) -> - gen_server:call(Pid, {query, Table, Query, Templates}, infinity). +query(Pid, Table, Query, Templates, TraceRenderedFunc) -> + gen_server:call(Pid, {query, Table, Query, Templates, TraceRenderedFunc}, infinity). %%-------------------------------------------------------------------- %% @doc @@ -77,14 +77,14 @@ handle_call(is_connected, _From, State) -> {false, Error} end, {reply, IsConnected, State}; -handle_call({query, Table, Query, Templates}, _From, State) -> - Result = do_query(Table, Query, Templates), +handle_call({query, Table, Query, Templates, TraceRenderedFunc}, _From, State) -> + Result = do_query(Table, Query, Templates, TraceRenderedFunc), {reply, Result, State}; handle_call(_Request, _From, State) -> {reply, ok, State}. handle_cast({query, Table, Query, Templates, {ReplyFun, [Context]}}, State) -> - Result = do_query(Table, Query, Templates), + Result = do_query(Table, Query, Templates, {fun(_, _) -> ok end, none}), ReplyFun(Context, Result), {noreply, State}; handle_cast(_Request, State) -> @@ -102,11 +102,14 @@ code_change(_OldVsn, State, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== -do_query(Table, Query0, Templates) -> +do_query(Table, Query0, Templates, {TraceRenderedFun, TraceRenderedCTX}) -> try Query = apply_template(Query0, Templates), + TraceRenderedFun(#{table => Table, query => Query}, TraceRenderedCTX), execute(Query, Table) catch + error:{unrecoverable_error, Reason} -> + {error, {unrecoverable_error, Reason}}; _Type:Reason -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index d61950513..d3ffd6c92 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -309,7 +309,11 @@ with_egress_client(ActionID, ResourceId, Fun, Args) -> ). trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> - OldMetaData = logger:get_process_metadata(), + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, try logger:set_process_metadata(LogMetaData), emqx_trace:rendered_action_template( From a2dd8f5aee920bc7bb5878d1b705cf17243a1105 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 23 Apr 2024 18:25:53 +0200 Subject: [PATCH 12/86] feat: add stop after render and after render trace to cassandra action --- apps/emqx/src/emqx_trace/emqx_trace.erl | 2 +- .../src/emqx_bridge_cassandra_connector.erl | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 4152fdbaa..329e5f696 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -87,7 +87,7 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -rendered_action_template(ActionID, RenderResult) when is_binary(ActionID) -> +rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) -> TraceResult = ?TRACE( "QUERY_RENDER", "action_template_rendered", diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index eb12cbaae..ef79f78fe 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -223,6 +223,11 @@ do_single_query(InstId, Request, Async, #{pool_name := PoolName} = State) -> } ), {PreparedKeyOrCQL1, Data} = proc_cql_params(Type, PreparedKeyOrCQL, Params, State), + emqx_trace:rendered_action_template(PreparedKeyOrCQL, #{ + type => Type, + key_or_cql => PreparedKeyOrCQL1, + data => Data + }), Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrCQL1, Data), handle_result(Res). @@ -261,6 +266,14 @@ do_batch_query(InstId, Requests, Async, #{pool_name := PoolName} = State) -> state => State } ), + ChannelID = + case Requests of + [{CID, _} | _] -> CID; + _ -> none + end, + emqx_trace:rendered_action_template(ChannelID, #{ + cqls => CQLs + }), Res = exec_cql_batch_query(InstId, PoolName, Async, CQLs), handle_result(Res). From 7922d5d4220e5cd379633724a1c3b9169dca7ad6 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 10:36:24 +0200 Subject: [PATCH 13/86] feat: add stop after render and after render trace to gcp action --- .../src/emqx_bridge_gcp_pubsub_impl_producer.erl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 13040dccf..12d5d1f2f 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -284,6 +284,13 @@ do_send_requests_sync(ConnectorState, Requests, InstanceId) -> Method = post, ReqOpts = #{request_ttl => RequestTTL}, Request = {prepared_request, {Method, Path, Body}, ReqOpts}, + emqx_trace:rendered_action_template(MessageTag, #{ + method => Method, + path => Path, + body => Body, + options => ReqOpts, + is_async => false + }), Result = emqx_bridge_gcp_pubsub_client:query_sync(Request, Client), QueryMode = sync, handle_result(Result, Request, QueryMode, InstanceId). @@ -312,6 +319,13 @@ do_send_requests_async(ConnectorState, Requests, ReplyFunAndArgs0) -> ReqOpts = #{request_ttl => RequestTTL}, Request = {prepared_request, {Method, Path, Body}, ReqOpts}, ReplyFunAndArgs = {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs0]}, + emqx_trace:rendered_action_template(MessageTag, #{ + method => Method, + path => Path, + body => Body, + options => ReqOpts, + is_async => true + }), emqx_bridge_gcp_pubsub_client:query_async( Request, ReplyFunAndArgs, Client ). From 7c7590fbc8a9f780e09c0eb633a7f1acad3146dd Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 10:37:52 +0200 Subject: [PATCH 14/86] feat: add stop after render and after render trace to greptimedb action --- .../src/emqx_bridge_greptimedb_connector.erl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 6bdd8a4cd..97eedf3f6 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -128,7 +128,7 @@ on_query(InstId, {Channel, Message}, State) -> greptimedb_connector_send_query, #{points => Points, batch => false, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, ErrorPoints} -> ?tp( greptimedb_connector_send_query_error, @@ -152,7 +152,7 @@ on_batch_query(InstId, [{Channel, _} | _] = BatchData, State) -> greptimedb_connector_send_query, #{points => Points, batch => true, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, Reason} -> ?tp( greptimedb_connector_send_query_error, @@ -173,7 +173,7 @@ on_query_async(InstId, {Channel, Message}, {ReplyFun, Args}, State) -> greptimedb_connector_send_query, #{points => Points, batch => false, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, ErrorPoints} = Err -> ?tp( greptimedb_connector_send_query_error, @@ -195,7 +195,7 @@ on_batch_query_async(InstId, [{Channel, _} | _] = BatchData, {ReplyFun, Args}, S greptimedb_connector_send_query, #{points => Points, batch => true, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, Reason} -> ?tp( greptimedb_connector_send_query_error, @@ -420,7 +420,8 @@ is_auth_key(_) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, Client, Points) -> +do_query(InstId, Channel, Client, Points) -> + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => false}), case greptimedb:write_batch(Client, Points) of {ok, #{response := {affected_rows, #{value := Rows}}}} -> ?SLOG(debug, #{ @@ -452,12 +453,13 @@ do_query(InstId, Client, Points) -> end end. -do_async_query(InstId, Client, Points, ReplyFunAndArgs) -> +do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) -> ?SLOG(info, #{ msg => "greptimedb_write_point_async", connector => InstId, points => Points }), + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => true}), WrappedReplyFunAndArgs = {fun ?MODULE:reply_callback/2, [ReplyFunAndArgs]}, ok = greptimedb:async_write_batch(Client, Points, WrappedReplyFunAndArgs). From 9d6655bc30639bec7a3d554b17650d5b851aa2ef Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 13:53:01 +0200 Subject: [PATCH 15/86] feat: add stop after render and after render trace to hstreamdb action --- .../src/emqx_bridge_hstreamdb_connector.erl | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index cf195fc9f..cf53291b2 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -134,8 +134,11 @@ on_query( #{ producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate } = maps:get(ChannelID, Channels), - try to_record(PartitionKey, HRecordTemplate, Data) of - Record -> append_record(InstId, Producer, Record, false) + try + KeyAndRawRecord = to_key_and_raw_record(PartitionKey, HRecordTemplate, Data), + emqx_trace:rendered_action_template(ChannelID, #{record => KeyAndRawRecord}), + Record = to_record(KeyAndRawRecord), + append_record(InstId, Producer, Record, false) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -148,8 +151,13 @@ on_batch_query( #{ producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate } = maps:get(ChannelID, Channels), - try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of - Records -> append_record(InstId, Producer, Records, true) + try + KeyAndRawRecordList = to_multi_part_key_and_partition_key( + PartitionKey, HRecordTemplate, BatchList + ), + emqx_trace:rendered_action_template(ChannelID, #{records => KeyAndRawRecordList}), + Records = [to_record(Item) || Item <- KeyAndRawRecordList], + append_record(InstId, Producer, Records, true) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -348,20 +356,20 @@ ensure_start_producer(ProducerName, ProducerOptions) -> produce_name(ActionId) -> list_to_binary("backend_hstream_producer:" ++ to_string(ActionId)). -to_record(PartitionKeyTmpl, HRecordTmpl, Data) -> +to_key_and_raw_record(PartitionKeyTmpl, HRecordTmpl, Data) -> PartitionKey = emqx_placeholder:proc_tmpl(PartitionKeyTmpl, Data), RawRecord = emqx_placeholder:proc_tmpl(HRecordTmpl, Data), - to_record(PartitionKey, RawRecord). + #{partition_key => PartitionKey, raw_record => RawRecord}. -to_record(PartitionKey, RawRecord) when is_binary(PartitionKey) -> - to_record(binary_to_list(PartitionKey), RawRecord); -to_record(PartitionKey, RawRecord) -> +to_record(#{partition_key := PartitionKey, raw_record := RawRecord}) when is_binary(PartitionKey) -> + to_record(#{partition_key => binary_to_list(PartitionKey), raw_record => RawRecord}); +to_record(#{partition_key := PartitionKey, raw_record := RawRecord}) -> hstreamdb:to_record(PartitionKey, raw, RawRecord). -to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) -> +to_multi_part_key_and_partition_key(PartitionKeyTmpl, HRecordTmpl, BatchList) -> lists:map( fun({_, Data}) -> - to_record(PartitionKeyTmpl, HRecordTmpl, Data) + to_key_and_raw_record(PartitionKeyTmpl, HRecordTmpl, Data) end, BatchList ). From e2b35ea2425d3d39575d4a4c232960987f68807b Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 13:53:58 +0200 Subject: [PATCH 16/86] feat: add stop after render and after render trace to opents action --- .../emqx_bridge_opents/src/emqx_bridge_opents_connector.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index abef958ff..509d53284 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -167,9 +167,10 @@ on_batch_query( BatchReq, #{channels := Channels} = State ) -> + [{ChannelId, _} | _] = BatchReq, case try_render_messages(BatchReq, Channels) of {ok, Datas} -> - do_query(InstanceId, Datas, State); + do_query(InstanceId, ChannelId, Datas, State); Error -> Error end. @@ -222,12 +223,13 @@ on_get_channel_status(InstanceId, ChannelId, #{channels := Channels} = State) -> %% Helper fns %%======================================================================================== -do_query(InstanceId, Query, #{pool_name := PoolName} = State) -> +do_query(InstanceId, ChannelID, Query, #{pool_name := PoolName} = State) -> ?TRACE( "QUERY", "opents_connector_received", #{connector => InstanceId, query => Query, state => State} ), + emqx_trace:rendered_action_template(ChannelID, #{query => Query}), ?tp(opents_bridge_on_query, #{instance_id => InstanceId}), From beedc72be47d5cdee9b28871f79608c20120cdcb Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 13:54:52 +0200 Subject: [PATCH 17/86] feat: add stop after render and after render trace to mongodb action --- .../src/emqx_bridge_mongodb_connector.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index a0d53d454..69c2242e4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -66,10 +66,15 @@ on_query(InstanceId, {Channel, Message0}, #{channels := Channels, connector_stat payload_template := PayloadTemplate, collection_template := CollectionTemplate } = ChannelState0 = maps:get(Channel, Channels), + Collection = emqx_placeholder:proc_tmpl(CollectionTemplate, Message0), ChannelState = ChannelState0#{ - collection => emqx_placeholder:proc_tmpl(CollectionTemplate, Message0) + collection => Collection }, Message = render_message(PayloadTemplate, Message0), + emqx_trace:rendered_action_template(Channel, #{ + collection => Collection, + data => Message + }), Res = emqx_mongodb:on_query( InstanceId, {Channel, Message}, From 279ad186f796bfb899a2b377049b653f77be9b87 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 13:55:43 +0200 Subject: [PATCH 18/86] feat: add stop after render and after render trace to kinesis action --- .../src/emqx_bridge_kinesis_impl_producer.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index c8a522e01..8744dfd71 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -261,6 +261,11 @@ do_send_requests_sync( stream_name := StreamName } = maps:get(ChannelId, InstalledChannels), Records = render_records(Requests, Templates), + StructuredRecords = [ + #{data => Data, partition_key => PartitionKey} + || {Data, PartitionKey} <- Records + ], + emqx_trace:rendered_action_template(ChannelId, StructuredRecords), Result = ecpool:pick_and_do( PoolName, {emqx_bridge_kinesis_connector_client, query, [Records, StreamName]}, From d27f05fa604e131ba1e3ba4400775b4bd0b91ce0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 13:56:41 +0200 Subject: [PATCH 19/86] feat: add stop after render and after render trace to influxdb action --- .../src/emqx_bridge_influxdb_connector.erl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 94419c7d9..f239d3735 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -130,7 +130,7 @@ on_query(InstId, {Channel, Message}, #{channels := ChannelConf}) -> influxdb_connector_send_query, #{points => Points, batch => false, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, ErrorPoints} -> ?tp( influxdb_connector_send_query_error, @@ -152,7 +152,7 @@ on_batch_query(InstId, BatchData, #{channels := ChannelConf}) -> influxdb_connector_send_query, #{points => Points, batch => true, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, Reason} -> ?tp( influxdb_connector_send_query_error, @@ -175,7 +175,7 @@ on_query_async( influxdb_connector_send_query, #{points => Points, batch => false, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, ErrorPoints} = Err -> ?tp( influxdb_connector_send_query_error, @@ -200,7 +200,7 @@ on_batch_query_async( influxdb_connector_send_query, #{points => Points, batch => true, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, Reason} -> ?tp( influxdb_connector_send_query_error, @@ -496,7 +496,8 @@ is_auth_key(_) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, Client, Points) -> +do_query(InstId, Channel, Client, Points) -> + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => false}), case influxdb:write(Client, Points) of ok -> ?SLOG(debug, #{ @@ -527,12 +528,13 @@ do_query(InstId, Client, Points) -> end end. -do_async_query(InstId, Client, Points, ReplyFunAndArgs) -> +do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) -> ?SLOG(info, #{ msg => "influxdb_write_point_async", connector => InstId, points => Points }), + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => true}), WrappedReplyFunAndArgs = {fun ?MODULE:reply_callback/2, [ReplyFunAndArgs]}, {ok, _WorkerPid} = influxdb:write_async(Client, Points, WrappedReplyFunAndArgs). From 9c37c99b623d4562eeec4cb37d8f1308906f1191 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 14:11:04 +0200 Subject: [PATCH 20/86] feat: add stop after render and after render trace to oracle action --- apps/emqx_oracle/src/emqx_oracle.erl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 4e88a3d96..e90665cc4 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -210,7 +210,7 @@ on_query( }), Type = query, {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), - Res = on_sql_query(InstId, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), + Res = on_sql_query(InstId, TypeOrKey, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), handle_result(Res). on_batch_query( @@ -244,7 +244,9 @@ on_batch_query( Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas], St = maps:get(BinKey, Sts), case - on_sql_query(InstId, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2) + on_sql_query( + InstId, BinKey, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2 + ) of {ok, Results} -> handle_batch_result(Results, 0); @@ -281,7 +283,13 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{ end end. -on_sql_query(InstId, PoolName, Type, ApplyMode, NameOrSQL, Data) -> +on_sql_query(InstId, ChannelID, PoolName, Type, ApplyMode, NameOrSQL, Data) -> + emqx_trace:rendered_action_template(ChannelID, #{ + type => Type, + apply_mode => ApplyMode, + name_or_sql => NameOrSQL, + data => Data + }), case ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, ApplyMode) of {error, Reason} = Result -> ?tp( From 74fac80e7e056cafa30a4161b7ce3f3dd815b8fe Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 14:24:52 +0200 Subject: [PATCH 21/86] feat: add stop after render and after render trace to pulsar action --- .../src/emqx_bridge_pulsar_connector.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 8c39c3671..0cddfab66 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -196,6 +196,11 @@ on_query(_InstanceId, {ChannelId, Message}, State) -> {error, channel_not_found}; {ok, #{message := MessageTmpl, sync_timeout := SyncTimeout, producers := Producers}} -> PulsarMessage = render_message(Message, MessageTmpl), + emqx_trace:rendered_action_template(ChannelId, #{ + message => PulsarMessage, + sync_timeout => SyncTimeout, + is_async => false + }), try pulsar:send_sync(Producers, [PulsarMessage], SyncTimeout) catch @@ -217,12 +222,16 @@ on_query_async(_InstanceId, {ChannelId, Message}, AsyncReplyFn, State) -> ?tp_span( pulsar_producer_on_query_async, #{instance_id => _InstanceId, message => Message}, - on_query_async2(Producers, Message, MessageTmpl, AsyncReplyFn) + on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) ) end. -on_query_async2(Producers, Message, MessageTmpl, AsyncReplyFn) -> +on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) -> PulsarMessage = render_message(Message, MessageTmpl), + emqx_trace:rendered_action_template(ChannelId, #{ + message => PulsarMessage, + is_async => true + }), pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}). %%------------------------------------------------------------------------------------- From 2abc1b1141fb7f480333954be1c166e5aa9ee8bf Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 14:48:23 +0200 Subject: [PATCH 22/86] feat: add stop after render and after render trace to redis action --- .../src/emqx_bridge_redis_connector.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index b7477b385..3f7c4897c 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -107,7 +107,7 @@ on_query(InstId, {cmd, Cmd}, #{conn_st := RedisConnSt}) -> Result; on_query( InstId, - {_MessageTag, _Data} = Msg, + {MessageTag, _Data} = Msg, #{channels := Channels, conn_st := RedisConnSt} ) -> case try_render_message([Msg], Channels) of @@ -116,6 +116,10 @@ on_query( redis_bridge_connector_cmd, #{cmd => Cmd, batch => false, mode => sync} ), + emqx_trace:rendered_action_template( + MessageTag, + #{command => Cmd, batch => false, mode => sync} + ), Result = query(InstId, {cmd, Cmd}, RedisConnSt), ?tp( redis_bridge_connector_send_done, @@ -135,6 +139,11 @@ on_batch_query( redis_bridge_connector_send, #{batch_data => BatchData, batch => true, mode => sync} ), + [{ChannelID, _} | _] = BatchData, + emqx_trace:rendered_action_template( + ChannelID, + #{commands => Cmds, batch => ture, mode => sync} + ), Result = query(InstId, {cmds, Cmds}, RedisConnSt), ?tp( redis_bridge_connector_send_done, From 22c7224267edf838efb6f78950fd67571e52facc Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 15:09:05 +0200 Subject: [PATCH 23/86] feat: add stop after render and after render trace to rocketmq action --- .../src/emqx_bridge_rocketmq_connector.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index f9b4ec5d4..314afb350 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -264,7 +264,11 @@ do_query( TopicKey = get_topic_key(Query, TopicTks), Data = apply_template(Query, Templates, DispatchStrategy), - + emqx_trace:rendered_action_template(ChannelId, #{ + topic_key => TopicKey, + data => Data, + request_timeout => RequestTimeout + }), Result = safe_do_produce( ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout ), From 03e3ac19a98ebe2a3f7c8c38506ec2fbfe7cfb27 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 15:21:33 +0200 Subject: [PATCH 24/86] feat: add stop after render and after render trace to s3 action --- apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index 4407222d5..5d3ed19f8 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -168,13 +168,14 @@ init_channel_state(#{parameters := Parameters}) -> on_query(InstId, {Tag, Data}, #{client_config := Config, channels := Channels}) -> case maps:get(Tag, Channels, undefined) of ChannelState = #{} -> - run_simple_upload(InstId, Data, ChannelState, Config); + run_simple_upload(InstId, Tag, Data, ChannelState, Config); undefined -> {error, {unrecoverable_error, {invalid_message_tag, Tag}}} end. run_simple_upload( InstId, + ChannelID, Data, #{ bucket := BucketTemplate, @@ -188,6 +189,11 @@ run_simple_upload( Client = emqx_s3_client:create(Bucket, Config), Key = render_key(KeyTemplate, Data), Content = render_content(ContentTemplate, Data), + emqx_trace:rendered_action_template(ChannelID, #{ + bucket => Bucket, + key => Key, + content => Content + }), case emqx_s3_client:put_object(Client, Key, UploadOpts, Content) of ok -> ?tp(s3_bridge_connector_upload_ok, #{ From a0b2357abbf82fa956ce454c2ccff51d227147c3 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 15:30:15 +0200 Subject: [PATCH 25/86] feat: add stop after render and after render trace to sqlserver action --- .../src/emqx_bridge_sqlserver_connector.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 1eb9746dc..683551316 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -413,6 +413,9 @@ do_query( %% only insert sql statement for single query and batch query case apply_template(QueryTuple, Templates) of {?ACTION_SEND_MESSAGE, SQL} -> + emqx_trace:rendered_action_template(ChannelId, #{ + sql => SQL + }), Result = ecpool:pick_and_do( PoolName, {?MODULE, worker_do_insert, [SQL, State]}, From 11d9d30fc00a1058276fb328042375420d245842 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 15:51:52 +0200 Subject: [PATCH 26/86] feat: add stop after render and after render trace to syskeeper action --- .../src/emqx_bridge_syskeeper_connector.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 045af348f..a6d47229c 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -273,6 +273,8 @@ do_query( Result = case try_render_message(Query, Channels) of {ok, Msg} -> + [{ChannelID, _} | _] = Query, + emqx_trace:rendered_action_template(ChannelID, #{message => Msg}), ecpool:pick_and_do( PoolName, {emqx_bridge_syskeeper_client, forward, [Msg, AckTimeout + ?EXTRA_CALL_TIMEOUT]}, From d6ceeb3b30f099378525cd6dca3d813dde4b4c55 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 16:38:35 +0200 Subject: [PATCH 27/86] feat: add stop after render and after render trace to rabbitmq action --- .../src/emqx_bridge_rabbitmq_connector.erl | 65 ++++++++++++++++--- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index 1ef1c6617..1637743b5 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -41,7 +41,7 @@ -export([connect/1]). %% Internal callbacks --export([publish_messages/4]). +-export([publish_messages/5]). namespace() -> "rabbitmq". @@ -214,9 +214,12 @@ on_query(ResourceID, {ChannelId, Data} = MsgReq, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> + LogMetaData = logger:get_process_metadata(), + TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, + TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq]]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq], TraceRenderedFunc]}, no_handover ), handle_result(Res); @@ -234,9 +237,12 @@ on_batch_query(ResourceID, [{ChannelId, _Data} | _] = Batch, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> + LogMetaData = logger:get_process_metadata(), + TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, + TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch, TraceRenderedFunc]}, no_handover ), handle_result(Res); @@ -244,6 +250,22 @@ on_batch_query(ResourceID, [{ChannelId, _Data} | _] = Batch, State) -> {error, {unrecoverable_error, {invalid_message_tag, ChannelId}}} end. +trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. + publish_messages( Conn, RabbitMQ, @@ -255,7 +277,8 @@ publish_messages( wait_for_publish_confirmations := WaitForPublishConfirmations, publish_confirmation_timeout := PublishConfirmationTimeout }, - Messages + Messages, + TraceRenderedFunc ) -> try publish_messages( @@ -267,15 +290,18 @@ publish_messages( PayloadTmpl, Messages, WaitForPublishConfirmations, - PublishConfirmationTimeout + PublishConfirmationTimeout, + TraceRenderedFunc ) catch + error:{unrecoverable_error, {action_stopped_after_template_rendering, _}} = Reason -> + {error, Reason}; %% if send a message to a non-existent exchange, RabbitMQ client will crash %% {shutdown,{server_initiated_close,404,<<"NOT_FOUND - no exchange 'xyz' in vhost '/'">>} %% so we catch and return {recoverable_error, Reason} to increase metrics _Type:Reason -> Msg = iolist_to_binary(io_lib:format("RabbitMQ: publish_failed: ~p", [Reason])), - erlang:error({recoverable_error, Msg}) + {error, {recoverable_error, Msg}} end. publish_messages( @@ -287,7 +313,8 @@ publish_messages( PayloadTmpl, Messages, WaitForPublishConfirmations, - PublishConfirmationTimeout + PublishConfirmationTimeout, + {TraceRenderedFun, TraceRenderedFuncCTX} ) -> case maps:find(Conn, RabbitMQ) of {ok, Channel} -> @@ -299,18 +326,36 @@ publish_messages( exchange = Exchange, routing_key = RoutingKey }, + FormattedMsgs = [ + format_data(PayloadTmpl, M) + || {_, M} <- Messages + ], + TraceRenderedFun( + #{ + messages => FormattedMsgs, + properties => #{ + headers => [], + delivery_mode => DeliveryMode + }, + method => #{ + exchange => Exchange, + routing_key => RoutingKey + } + }, + TraceRenderedFuncCTX + ), lists:foreach( - fun({_, MsgRaw}) -> + fun(Msg) -> amqp_channel:cast( Channel, Method, #amqp_msg{ - payload = format_data(PayloadTmpl, MsgRaw), + payload = Msg, props = MessageProperties } ) end, - Messages + FormattedMsgs ), case WaitForPublishConfirmations of true -> From 0dbaef431671ada97cc4a9030a8a3e42e791a7fa Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 24 Apr 2024 17:03:04 +0200 Subject: [PATCH 28/86] feat: add stop after render and after render trace to tdengine action --- .../src/emqx_bridge_tdengine_connector.erl | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 7bb342ed1..383ceabf7 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -32,7 +32,7 @@ -export([connector_examples/1]). --export([connect/1, do_get_status/1, execute/3, do_batch_insert/4]). +-export([connect/1, do_get_status/1, execute/3, do_batch_insert/5]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -186,6 +186,7 @@ on_query(InstanceId, {ChannelId, Data}, #{channels := Channels} = State) -> case maps:find(ChannelId, Channels) of {ok, #{insert := Tokens, opts := Opts}} -> Query = emqx_placeholder:proc_tmpl(Tokens, Data), + emqx_trace:rendered_action_template(ChannelId, #{query => Query}), do_query_job(InstanceId, {?MODULE, execute, [Query, Opts]}, State); _ -> {error, {unrecoverable_error, {invalid_channel_id, InstanceId}}} @@ -199,9 +200,12 @@ on_batch_query( ) -> case maps:find(ChannelId, Channels) of {ok, #{batch := Tokens, opts := Opts}} -> + LogMetaData = logger:get_process_metadata(), + TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, + TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, do_query_job( InstanceId, - {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts]}, + {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts, TraceRenderedFunc]}, State ); _ -> @@ -212,6 +216,22 @@ on_batch_query(InstanceId, BatchReq, State) -> ?SLOG(error, LogMeta#{msg => "invalid_request"}), {error, {unrecoverable_error, invalid_request}}. +trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. + on_get_status(_InstanceId, #{pool_name := PoolName} = State) -> case emqx_resource_pool:health_check_workers( @@ -338,9 +358,15 @@ do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) -> execute(Conn, Query, Opts) -> tdengine:insert(Conn, Query, Opts). -do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> +do_batch_insert(Conn, Tokens, BatchReqs, Opts, {TraceRenderedFun, TraceRenderedFunCTX}) -> SQL = aggregate_query(Tokens, BatchReqs, <<"INSERT INTO">>), - execute(Conn, SQL, Opts). + try + TraceRenderedFun(#{query => SQL}, TraceRenderedFunCTX), + execute(Conn, SQL, Opts) + catch + error:{unrecoverable_error, {action_stopped_after_template_rendering, _}} = Reason -> + {error, Reason} + end. aggregate_query(BatchTks, BatchReqs, Acc) -> lists:foldl( From ef9884cf47727ef334af65f469779f6345a325d2 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 25 Apr 2024 11:39:20 +0200 Subject: [PATCH 29/86] refactor(rule trace): templates rendered trace to increase code reuse * The code for passing the trace context to a sub process has been improved to increase code reuse. This code is used when the action templates are rendered in a sub process. * A macro has also been added for the error term that is thrown when the action shall be stopped after the templates has been rendered. This is also done to reduce code duplication and to reduce the risk of introducing bugs due to typos. * Fix incorrect type spec Thanks to @zmstone for suggesting these improvements in comments to a PR (https://github.com/emqx/emqx/pull/12916). --- apps/emqx/include/emqx_trace.hrl | 6 ++ apps/emqx/src/emqx_trace/emqx_trace.erl | 55 ++++++++++++++++- .../src/emqx_bridge_dynamo_connector.erl | 26 ++------ .../emqx_bridge_dynamo_connector_client.erl | 23 +++++-- .../src/emqx_bridge_mqtt_connector.erl | 28 ++------- .../src/emqx_bridge_mqtt_egress.erl | 53 ++++++++-------- .../src/emqx_bridge_rabbitmq_connector.erl | 60 ++++++------------- .../src/emqx_bridge_tdengine_connector.erl | 32 +++------- .../src/emqx_rule_runtime.erl | 3 +- 9 files changed, 140 insertions(+), 146 deletions(-) diff --git a/apps/emqx/include/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl index d1e70b184..27dd8b6c8 100644 --- a/apps/emqx/include/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -38,4 +38,10 @@ -define(SHARD, ?COMMON_SHARD). -define(MAX_SIZE, 30). +-define(EMQX_TRACE_STOP_ACTION(REASON), + {unrecoverable_error, {action_stopped_after_template_rendering, REASON}} +). + +-define(EMQX_TRACE_STOP_ACTION_MATCH, ?EMQX_TRACE_STOP_ACTION(_)). + -endif. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 329e5f696..91de65b39 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -29,7 +29,9 @@ unsubscribe/2, log/3, log/4, - rendered_action_template/2 + rendered_action_template/2, + make_rendered_action_template_trace_context/1, + rendered_action_template_with_ctx/2 ]). -export([ @@ -70,6 +72,12 @@ -export_type([ruleid/0]). -type ruleid() :: binary(). +-export_type([rendered_action_template_ctx/0]). +-opaque rendered_action_template_ctx() :: #{ + trace_ctx := map(), + action_id := any() +}. + publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when @@ -107,15 +115,56 @@ rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) -> ) ), MsgBin = unicode:characters_to_binary(StopMsg), - error({unrecoverable_error, {action_stopped_after_template_rendering, MsgBin}}); + error(?EMQX_TRACE_STOP_ACTION(MsgBin)); _ -> ok end, TraceResult; rendered_action_template(_ActionID, _RenderResult) -> - %% We do nothing if we don't get a valid Action ID + %% We do nothing if we don't get a valid Action ID. This can happen when + %% called from connectors that are used for actions as well as authz and + %% authn. ok. +%% The following two functions are used for connectors that don't do the +%% rendering in the main process (the one that called on_*query). In this case +%% we need to pass the trace context to the sub process that do the rendering +%% so that the result of the rendering can be traced correctly. It is also +%% important to ensure that the error that can be thrown from +%% rendered_action_template_with_ctx is handled in the appropriate way in the +%% sub process. +-spec make_rendered_action_template_trace_context(any()) -> rendered_action_template_ctx(). +make_rendered_action_template_trace_context(ActionID) -> + MetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + #{trace_ctx => MetaData, action_id => ActionID}. + +-spec rendered_action_template_with_ctx(rendered_action_template_ctx(), Result :: term()) -> term(). +rendered_action_template_with_ctx( + #{ + trace_ctx := LogMetaData, + action_id := ActionID + }, + RenderResult +) -> + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. + log(List, Msg, Meta) -> log(debug, List, Msg, Meta). diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index f9a87ccf7..598b3342d 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -9,6 +9,7 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -246,16 +247,15 @@ do_query( table := Table, templates := Templates } = ChannelState, - LogMetaData = logger:get_process_metadata(), - TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, - TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + TraceRenderedCTX = + emqx_trace:make_rendered_action_template_trace_context(ChannelId), Result = case ensuare_dynamo_keys(Query, ChannelState) of true -> ecpool:pick_and_do( PoolName, {emqx_bridge_dynamo_connector_client, query, [ - Table, QueryTuple, Templates, TraceRenderedFunc + Table, QueryTuple, Templates, TraceRenderedCTX ]}, no_handover ); @@ -264,7 +264,7 @@ do_query( end, case Result of - {error, {unrecoverable_error, {action_stopped_after_template_rendering, _}}} = Error -> + {error, ?EMQX_TRACE_STOP_ACTION(_)} = Error -> Error; {error, Reason} -> ?tp( @@ -298,22 +298,6 @@ do_query( Result end. -trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> - OldMetaData = - case logger:get_process_metadata() of - undefined -> #{}; - M -> M - end, - try - logger:set_process_metadata(LogMetaData), - emqx_trace:rendered_action_template( - ActionID, - RenderResult - ) - after - logger:set_process_metadata(OldMetaData) - end. - get_channel_id([{ChannelId, _Req} | _]) -> ChannelId; get_channel_id({ChannelId, _Req}) -> diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 1d1ad3760..f257ae389 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -40,8 +40,8 @@ is_connected(Pid, Timeout) -> {false, Error} end. -query(Pid, Table, Query, Templates, TraceRenderedFunc) -> - gen_server:call(Pid, {query, Table, Query, Templates, TraceRenderedFunc}, infinity). +query(Pid, Table, Query, Templates, TraceRenderedCTX) -> + gen_server:call(Pid, {query, Table, Query, Templates, TraceRenderedCTX}, infinity). %%-------------------------------------------------------------------- %% @doc @@ -77,8 +77,8 @@ handle_call(is_connected, _From, State) -> {false, Error} end, {reply, IsConnected, State}; -handle_call({query, Table, Query, Templates, TraceRenderedFunc}, _From, State) -> - Result = do_query(Table, Query, Templates, TraceRenderedFunc), +handle_call({query, Table, Query, Templates, TraceRenderedCTX}, _From, State) -> + Result = do_query(Table, Query, Templates, TraceRenderedCTX), {reply, Result, State}; handle_call(_Request, _From, State) -> {reply, ok, State}. @@ -102,10 +102,13 @@ code_change(_OldVsn, State, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== -do_query(Table, Query0, Templates, {TraceRenderedFun, TraceRenderedCTX}) -> +do_query(Table, Query0, Templates, TraceRenderedCTX) -> try Query = apply_template(Query0, Templates), - TraceRenderedFun(#{table => Table, query => Query}, TraceRenderedCTX), + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + table => Table, + query => {fun trace_format_query/1, Query} + }), execute(Query, Table) catch error:{unrecoverable_error, Reason} -> @@ -114,6 +117,14 @@ do_query(Table, Query0, Templates, {TraceRenderedFun, TraceRenderedCTX}) -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. +trace_format_query({Type, Data}) -> + #{type => Type, data => Data}; +trace_format_query([_ | _] = Batch) -> + BatchData = [trace_format_query(Q) || Q <- Batch], + #{type => batch, data => BatchData}; +trace_format_query(Query) -> + Query. + %% some simple query commands for authn/authz or test execute({insert_item, Msg}, Table) -> Item = convert_to_item(Msg), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index d3ffd6c92..f133bf334 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -301,29 +301,11 @@ on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> }). with_egress_client(ActionID, ResourceId, Fun, Args) -> - LogMetaData = logger:get_process_metadata(), - TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ActionID}, - TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ActionID), ecpool:pick_and_do( - ResourceId, {emqx_bridge_mqtt_egress, Fun, [TraceRenderedFunc | Args]}, no_handover + ResourceId, {emqx_bridge_mqtt_egress, Fun, [TraceRenderedCTX | Args]}, no_handover ). -trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> - OldMetaData = - case logger:get_process_metadata() of - undefined -> #{}; - M -> M - end, - try - logger:set_process_metadata(LogMetaData), - emqx_trace:rendered_action_template( - ActionID, - RenderResult - ) - after - logger:set_process_metadata(OldMetaData) - end. - on_async_result(Callback, Result) -> apply_callback_function(Callback, handle_send_result(Result)). @@ -343,9 +325,7 @@ handle_send_result({ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}}) -> handle_send_result({ok, Reply}) -> {error, classify_reply(Reply)}; handle_send_result({error, Reason}) -> - {error, classify_error(Reason)}; -handle_send_result({unrecoverable_error, Reason}) -> - {error, {unrecoverable_error, Reason}}. + {error, classify_error(Reason)}. classify_reply(Reply = #{reason_code := _}) -> {unrecoverable_error, Reply}. @@ -360,6 +340,8 @@ classify_error({shutdown, _} = Reason) -> {recoverable_error, Reason}; classify_error(shutdown = Reason) -> {recoverable_error, Reason}; +classify_error({unrecoverable_error, _Reason} = Error) -> + Error; classify_error(Reason) -> {unrecoverable_error, Reason}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 80d38bc78..a4a0b0d37 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -29,9 +29,6 @@ -type message() :: emqx_types:message() | map(). -type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. -type remote_message() :: #mqtt_msg{}. --type trace_rendered_func() :: { - fun((RenderResult :: any(), CTX :: map()) -> any()), TraceCTX :: map() -}. -type egress() :: #{ local => #{ @@ -45,37 +42,40 @@ config(#{remote := RC = #{}} = Conf) -> Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}. --spec send(pid(), trace_rendered_func(), message(), egress()) -> ok. -send(Pid, TraceRenderedFunc, MsgIn, Egress) -> +-spec send(pid(), emqx_trace:rendered_action_template_ctx(), message(), egress()) -> + ok | {error, {unrecoverable_error, term()}}. +send(Pid, TraceRenderedCTX, MsgIn, Egress) -> try - emqtt:publish(Pid, export_msg(MsgIn, Egress, TraceRenderedFunc)) + emqtt:publish(Pid, export_msg(MsgIn, Egress, TraceRenderedCTX)) catch error:{unrecoverable_error, Reason} -> - {unrecoverable_error, Reason} + {error, {unrecoverable_error, Reason}} end. --spec send_async(pid(), trace_rendered_func(), message(), callback(), egress()) -> - ok | {ok, pid()}. -send_async(Pid, TraceRenderedFunc, MsgIn, Callback, Egress) -> +-spec send_async(pid(), emqx_trace:rendered_action_template_ctx(), message(), callback(), egress()) -> + {ok, pid()} | {error, {unrecoverable_error, term()}}. +send_async(Pid, TraceRenderedCTX, MsgIn, Callback, Egress) -> try ok = emqtt:publish_async( - Pid, export_msg(MsgIn, Egress, TraceRenderedFunc), _Timeout = infinity, Callback + Pid, export_msg(MsgIn, Egress, TraceRenderedCTX), _Timeout = infinity, Callback ), {ok, Pid} catch error:{unrecoverable_error, Reason} -> - {unrecoverable_error, Reason} + {error, {unrecoverable_error, Reason}} end. -export_msg(Msg, #{remote := Remote}, TraceRenderedFunc) -> - to_remote_msg(Msg, Remote, TraceRenderedFunc). +export_msg(Msg, #{remote := Remote}, TraceRenderedCTX) -> + to_remote_msg(Msg, Remote, TraceRenderedCTX). --spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars(), trace_rendered_func()) -> +-spec to_remote_msg( + message(), emqx_bridge_mqtt_msg:msgvars(), emqx_trace:rendered_action_template_ctx() +) -> remote_message(). -to_remote_msg(#message{flags = Flags} = Msg, Vars, TraceRenderedFunc) -> +to_remote_msg(#message{flags = Flags} = Msg, Vars, TraceRenderedCTX) -> {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), - to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars, TraceRenderedFunc); -to_remote_msg(Msg = #{}, Remote, {TraceRenderedFun, TraceRenderedCTX}) -> + to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars, TraceRenderedCTX); +to_remote_msg(Msg = #{}, Remote, TraceRenderedCTX) -> #{ topic := Topic, payload := Payload, @@ -83,16 +83,13 @@ to_remote_msg(Msg = #{}, Remote, {TraceRenderedFun, TraceRenderedCTX}) -> retain := Retain } = emqx_bridge_mqtt_msg:render(Msg, Remote), PubProps = maps:get(pub_props, Msg, #{}), - TraceRenderedFun( - #{ - qos => QoS, - retain => Retain, - topic => Topic, - props => PubProps, - payload => Payload - }, - TraceRenderedCTX - ), + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + qos => QoS, + retain => Retain, + topic => Topic, + props => PubProps, + payload => Payload + }), #mqtt_msg{ qos = QoS, retain = Retain, diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index 1637743b5..dacb47a57 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -9,6 +9,7 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -214,12 +215,10 @@ on_query(ResourceID, {ChannelId, Data} = MsgReq, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> - LogMetaData = logger:get_process_metadata(), - TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, - TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq], TraceRenderedFunc]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq], TraceRenderedCTX]}, no_handover ), handle_result(Res); @@ -237,12 +236,10 @@ on_batch_query(ResourceID, [{ChannelId, _Data} | _] = Batch, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> - LogMetaData = logger:get_process_metadata(), - TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, - TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch, TraceRenderedFunc]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch, TraceRenderedCTX]}, no_handover ), handle_result(Res); @@ -250,22 +247,6 @@ on_batch_query(ResourceID, [{ChannelId, _Data} | _] = Batch, State) -> {error, {unrecoverable_error, {invalid_message_tag, ChannelId}}} end. -trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> - OldMetaData = - case logger:get_process_metadata() of - undefined -> #{}; - M -> M - end, - try - logger:set_process_metadata(LogMetaData), - emqx_trace:rendered_action_template( - ActionID, - RenderResult - ) - after - logger:set_process_metadata(OldMetaData) - end. - publish_messages( Conn, RabbitMQ, @@ -278,7 +259,7 @@ publish_messages( publish_confirmation_timeout := PublishConfirmationTimeout }, Messages, - TraceRenderedFunc + TraceRenderedCTX ) -> try publish_messages( @@ -291,10 +272,10 @@ publish_messages( Messages, WaitForPublishConfirmations, PublishConfirmationTimeout, - TraceRenderedFunc + TraceRenderedCTX ) catch - error:{unrecoverable_error, {action_stopped_after_template_rendering, _}} = Reason -> + error:?EMQX_TRACE_STOP_ACTION_MATCH = Reason -> {error, Reason}; %% if send a message to a non-existent exchange, RabbitMQ client will crash %% {shutdown,{server_initiated_close,404,<<"NOT_FOUND - no exchange 'xyz' in vhost '/'">>} @@ -314,7 +295,7 @@ publish_messages( Messages, WaitForPublishConfirmations, PublishConfirmationTimeout, - {TraceRenderedFun, TraceRenderedFuncCTX} + TraceRenderedCTX ) -> case maps:find(Conn, RabbitMQ) of {ok, Channel} -> @@ -330,20 +311,17 @@ publish_messages( format_data(PayloadTmpl, M) || {_, M} <- Messages ], - TraceRenderedFun( - #{ - messages => FormattedMsgs, - properties => #{ - headers => [], - delivery_mode => DeliveryMode - }, - method => #{ - exchange => Exchange, - routing_key => RoutingKey - } + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + messages => FormattedMsgs, + properties => #{ + headers => [], + delivery_mode => DeliveryMode }, - TraceRenderedFuncCTX - ), + method => #{ + exchange => Exchange, + routing_key => RoutingKey + } + }), lists:foreach( fun(Msg) -> amqp_channel:cast( diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 383ceabf7..67b0e77bc 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -10,6 +10,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -200,12 +201,10 @@ on_batch_query( ) -> case maps:find(ChannelId, Channels) of {ok, #{batch := Tokens, opts := Opts}} -> - LogMetaData = logger:get_process_metadata(), - TraceRenderedFuncContext = #{trace_ctx => LogMetaData, action_id => ChannelId}, - TraceRenderedFunc = {fun trace_render_result/2, TraceRenderedFuncContext}, + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), do_query_job( InstanceId, - {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts, TraceRenderedFunc]}, + {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts, TraceRenderedCTX]}, State ); _ -> @@ -216,22 +215,6 @@ on_batch_query(InstanceId, BatchReq, State) -> ?SLOG(error, LogMeta#{msg => "invalid_request"}), {error, {unrecoverable_error, invalid_request}}. -trace_render_result(RenderResult, #{trace_ctx := LogMetaData, action_id := ActionID}) -> - OldMetaData = - case logger:get_process_metadata() of - undefined -> #{}; - M -> M - end, - try - logger:set_process_metadata(LogMetaData), - emqx_trace:rendered_action_template( - ActionID, - RenderResult - ) - after - logger:set_process_metadata(OldMetaData) - end. - on_get_status(_InstanceId, #{pool_name := PoolName} = State) -> case emqx_resource_pool:health_check_workers( @@ -358,13 +341,16 @@ do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) -> execute(Conn, Query, Opts) -> tdengine:insert(Conn, Query, Opts). -do_batch_insert(Conn, Tokens, BatchReqs, Opts, {TraceRenderedFun, TraceRenderedFunCTX}) -> +do_batch_insert(Conn, Tokens, BatchReqs, Opts, TraceRenderedCTX) -> SQL = aggregate_query(Tokens, BatchReqs, <<"INSERT INTO">>), try - TraceRenderedFun(#{query => SQL}, TraceRenderedFunCTX), + emqx_trace:rendered_action_template_with_ctx( + TraceRenderedCTX, + #{query => SQL} + ), execute(Conn, SQL, Opts) catch - error:{unrecoverable_error, {action_stopped_after_template_rendering, _}} = Reason -> + error:?EMQX_TRACE_STOP_ACTION_MATCH = Reason -> {error, Reason} end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index e2f01321a..5ec4bdc6e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -18,6 +18,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("emqx_resource/include/emqx_resource_errors.hrl"). -export([ @@ -724,7 +725,7 @@ inc_action_metrics(TraceCtx, Result) -> do_inc_action_metrics( #{rule_id := RuleId, action_id := ActId} = TraceContext, - {error, {unrecoverable_error, {action_stopped_after_template_rendering, Explanation}} = _Reason} + {error, ?EMQX_TRACE_STOP_ACTION(Explanation) = _Reason} ) -> TraceContext1 = maps:remove(action_id, TraceContext), trace_action( From ff09f1419142e5dad089bcb8301710be8d836da6 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 25 Apr 2024 13:27:14 +0200 Subject: [PATCH 30/86] fix(http connector): remove sensitive info from headers lazily In production code we don't need to redact the headers for a trace that will never appear anywhere so we can improve performance by doing removal of sensitive information lazily. --- apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 76f2686a1..88e4d9c36 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -512,7 +512,7 @@ trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> host => Host, path => Path, method => Method, - headers => emqx_utils_redact:redact_headers(Headers), + headers => {fun emqx_utils_redact:redact_headers/1, Headers}, timeout => Timeout } ); @@ -523,7 +523,7 @@ trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> host => Host, path => Path, method => Method, - headers => emqx_utils_redact:redact_headers(Headers), + headers => {fun emqx_utils_redact:redact_headers/1, Headers}, timeout => Timeout, body => {fun log_format_body/1, Body} } From 15594b4db64e44d1473fd0b62f53827087644d4e Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 25 Apr 2024 16:54:09 +0200 Subject: [PATCH 31/86] fix(HTTP connector): retry on 503 Service Unavailable response Previously, if an HTTP request received a 503 (Service Unavailable) status, it was marked as a failure without retrying. This has now been fixed so that the request is retried a configurable number of times. Fixes: https://emqx.atlassian.net/browse/EMQX-12217 https://github.com/emqx/emqx/issues/12869 (partly) --- .../src/emqx_bridge_http_connector.erl | 7 +++++ .../test/emqx_bridge_http_SUITE.erl | 31 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 88e4d9c36..ef9c6c70d 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -836,6 +836,8 @@ transform_result(Result) -> Result; {ok, _TooManyRequests = StatusCode = 429, Headers} -> {error, {recoverable_error, #{status_code => StatusCode, headers => Headers}}}; + {ok, _ServiceUnavailable = StatusCode = 503, Headers} -> + {error, {recoverable_error, #{status_code => StatusCode, headers => Headers}}}; {ok, StatusCode, Headers} -> {error, {unrecoverable_error, #{status_code => StatusCode, headers => Headers}}}; {ok, _TooManyRequests = StatusCode = 429, Headers, Body} -> @@ -843,6 +845,11 @@ transform_result(Result) -> {recoverable_error, #{ status_code => StatusCode, headers => Headers, body => Body }}}; + {ok, _ServiceUnavailable = StatusCode = 503, Headers, Body} -> + {error, + {recoverable_error, #{ + status_code => StatusCode, headers => Headers, body => Body + }}}; {ok, StatusCode, Headers, Body} -> {error, {unrecoverable_error, #{ diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 9d215d815..7f9418bd3 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -93,6 +93,14 @@ init_per_testcase(t_too_many_requests, Config) -> ), ok = emqx_bridge_http_connector_test_server:set_handler(too_many_requests_http_handler()), [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; +init_per_testcase(t_service_unavailable, Config) -> + HTTPPath = <<"/path">>, + ServerSSLOpts = false, + {ok, {HTTPPort, _Pid}} = emqx_bridge_http_connector_test_server:start_link( + _Port = random, HTTPPath, ServerSSLOpts + ), + ok = emqx_bridge_http_connector_test_server:set_handler(service_unavailable_http_handler()), + [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; init_per_testcase(t_rule_action_expired, Config) -> [ {bridge_name, ?BRIDGE_NAME} @@ -115,6 +123,7 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(TestCase, _Config) when TestCase =:= t_path_not_found; TestCase =:= t_too_many_requests; + TestCase =:= t_service_unavailable; TestCase =:= t_rule_action_expired; TestCase =:= t_bridge_probes_header_atoms; TestCase =:= t_send_async_connection_timeout; @@ -260,6 +269,12 @@ not_found_http_handler() -> end. too_many_requests_http_handler() -> + fail_then_success_http_handler(429). + +service_unavailable_http_handler() -> + fail_then_success_http_handler(503). + +fail_then_success_http_handler(FailStatusCode) -> GetAndBump = fun() -> NCalled = persistent_term:get({?MODULE, times_called}, 0), @@ -272,7 +287,7 @@ too_many_requests_http_handler() -> {ok, Body, Req} = cowboy_req:read_body(Req0), TestPid ! {http, cowboy_req:headers(Req), Body}, Rep = - case N >= 2 of + case N >= 3 of true -> cowboy_req:reply( 200, @@ -282,9 +297,13 @@ too_many_requests_http_handler() -> ); false -> cowboy_req:reply( - 429, + FailStatusCode, #{<<"content-type">> => <<"text/plain">>}, - <<"slow down, buddy">>, + %% Body and no body to trigger different code paths + case N of + 1 -> <<"slow down, buddy">>; + _ -> <<>> + end, Req ) end, @@ -570,6 +589,12 @@ t_path_not_found(Config) -> ok. t_too_many_requests(Config) -> + check_send_is_retried(Config). + +t_service_unavailable(Config) -> + check_send_is_retried(Config). + +check_send_is_retried(Config) -> ?check_trace( begin #{port := Port, path := Path} = ?config(http_server, Config), From 6ab9460c1657b9b51c1e1785a0b5e8dabad66f5b Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 25 Apr 2024 17:10:30 +0200 Subject: [PATCH 32/86] docs: add change log entry --- changes/ce/fix-12932.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-12932.en.md diff --git a/changes/ce/fix-12932.en.md b/changes/ce/fix-12932.en.md new file mode 100644 index 000000000..fee7078a7 --- /dev/null +++ b/changes/ce/fix-12932.en.md @@ -0,0 +1 @@ +Previously, if a HTTP action request received a 503 (Service Unavailable) status, it was marked as a failure and the request was not retried. This has now been fixed so that the request is retried a configurable number of times. From d30b52f0f9ea08a2f74aea4628d140ab8c48f638 Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Apr 2024 14:04:50 +0200 Subject: [PATCH 33/86] docs: refine acl.conf comments --- apps/emqx_auth/etc/acl.conf | 137 +++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 24 deletions(-) diff --git a/apps/emqx_auth/etc/acl.conf b/apps/emqx_auth/etc/acl.conf index dbeec6852..b23714558 100644 --- a/apps/emqx_auth/etc/acl.conf +++ b/apps/emqx_auth/etc/acl.conf @@ -1,27 +1,4 @@ -%%-------------------------------------------------------------------- -%% -type(ipaddr() :: {ipaddr, string()}). -%% -%% -type(ipaddrs() :: {ipaddrs, [string()]}). -%% -%% -type(username() :: {user | username, string()} | {user | username, {re, regex()}}). -%% -%% -type(clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}). -%% -%% -type(who() :: ipaddr() | ipaddrs() | username() | clientid() | -%% {'and', [ipaddr() | ipaddrs() | username() | clientid()]} | -%% {'or', [ipaddr() | ipaddrs() | username() | clientid()]} | -%% all). -%% -%% -type(action() :: subscribe | publish | all). -%% -%% -type(topic_filters() :: string()). -%% -%% -type(topics() :: [topic_filters() | {eq, topic_filters()}]). -%% -%% -type(permission() :: allow | deny). -%% -%% -type(rule() :: {permission(), who(), action(), topics()} | {permission(), all}). -%%-------------------------------------------------------------------- +%%-------------- Default ACL rules ------------------------------------------------------- {allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. @@ -29,4 +6,116 @@ {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. +%% NOTE: change to {deny, all} in production! {allow, all}. + +%% See docs below +%% +%% ------------ The formal spec ---------------------------------------------------------- +%% +%% -type ipaddr() :: {ipaddr, string()}. +%% -type ipaddrs() :: {ipaddrs, [string()]}. +%% -type username() :: {user | username, string()} | {user | username, {re, regex()}}. +%% -type clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}. +%% -type who() :: ipaddr() | ipaddrs() | username() | clientid() | +%% {'and', [ipaddr() | ipaddrs() | username() | clientid()]} | +%% {'or', [ipaddr() | ipaddrs() | username() | clientid()]} | +%% all. +%% -type simple_action() :: subscribe | publish | all. +%% -type complex_action() :: {simple_action(), [{qos, 0..2}, {retain, true|false|all}]}. +%% -type action() :: simple_action() | complex_action(). +%% -type topic() :: string(). +%% -type topic_filter() :: string(). +%% -type topic_match() :: topic() | topic_filter() | {eq, topic() | topic_filter()}. +%% -type perm() :: allow | deny. +%% -type rule() :: {perm(), who(), action(), [topic_match()]} | {perm(), all}. + +%%-------------- Viusal aid for the spec ------------------------------------------------- +%% +%% rule() +%% ├── {perm(), who(), action(), [topic_match()]} +%% │ │ │ │ ├── topic() :: string() +%% │ │ │ │ ├── topic_filter() :: string() +%% │ │ │ │ └── {eq, topic() | topic_filter()} +%% │ │ │ │ +%% │ │ │ ├── simple_action() +%% │ │ │ │ ├── publish +%% │ │ │ │ ├── subscribe +%% │ │ │ │ └── all +%% │ │ │ └── {simple_action(), [{qos,0..2},{retain,true|false|all}]} +%% │ │ │ +%% │ │ ├── ipaddr() +%% │ │ │ └── {ipaddr, string()} +%% │ │ ├── ipaddrs() +%% │ │ │ └── {ipaddrs, [string()]} +%% │ │ ├── username() +%% │ │ │ ├── {user | username, string()} +%% │ │ │ └── {user | username, {re, regex()}} +%% │ │ ├── clientid() +%% │ │ │ ├── {client | clientid, string()} +%% │ │ │ └── {client | clientid, {re, regex()}} +%% │ │ ├── {'and', [ipaddr() | ipaddrs() | username() | clientid()]} +%% │ │ ├── {'or', [ipaddr() | ipaddrs() | username() | clientid()]} +%% │ │ └── all +%% │ │ +%% │ ├── allow +%% │ └── deny +%% │ +%% └── {perm(), all} +%% + +%% This file defines a set of ACL rules for MQTT client pub/sub authorization. +%% The content is of Erlang-term format. +%% Each Erlang-term is a tuple `{...}` terminated by dot `.` +%% +%% NOTE: When deploy to production, the last rule should be changed to {deny, all}. +%% +%% NOTE: It's a good practice to keep the nubmer of rules small, because in worst case +%% scenarios, all rules have to be traversed for each message publish. +%% +%% A rule is a 4-element tuple. +%% For example, `{allow, {username, "Jon"}, subscribe, ["#"]}` allows Jon to subscribe to +%% any topic they want. +%% +%% Below is an explanation: +%% +%% - `perm()`: The permission. +%% Defines whether this is an `allow` or `deny` rule. +%% +%% - `who()`: The MQTT client matching condition. +%% - `all`: A rule which applies to all clients. +%% - `{ipaddr, IpAddress}`: Matches a client by source IP address. CIDR notation is allowed. +%% - `{ipaddrs, [IpAddress]}`: Matches clients by a set of IP addresses. CIDR notation is allowed. +%% - `{clientid, ClientID}`: Matches a client by ID. +%% - `{username, Username}`: Matches a client by username. +%% - `{..., {re, ..}}`: Regular expression to match either clientid or username. +%% - `{'and', [...]}`: Combines a list of matching conditions. +%% - `{'or', [...]}`: Combines a list of matching conditions. +%% +%% - `action()`: Matches publish or subscribe actions (or both). +%% Applies the rule to `publish` or `subscribe` actions. +%% The special value `all` denotes allowing or denying both `publish` and `subscribe`. +%% It can also be associated with `qos` and `retain` flags to match the action with +%% more specifics. For example, `{publish, [{qos,0},{retain,false}]}` should only +%% match the `publish` action when the message has QoS 0, and without retained flag set. +%% +%% - `[topic_match()]`: +%% A list of topics, topic-filters, or template rendering to match the topic being +%% subscribed to or published. +%% For example, `{allow, {username, "Jan"}, publish, ["jan/#"]}` permits Jan to publish +%% to any topic matching the wildcard pattern "jan/#". +%% A special tuple `{eq, topic_match()}` is useful to allow or deny the specific wildcard +%% subscription instead of performing a topic match. +%% A `topic_match()` can also contain a placeholder rendered with actual value at runtime, +%% for example, `{allow, all, publish, "${clientid}/#"}` allows all clients to publish to +%% topics prefixed by their own client ID. +%% +%% Supported placeholders are: +%% - `${cn}`: TLS certificate common name. +%% - `${clientid}`: The client ID. +%% - `${username}`: The username. +%% - `${client_attrs.NAME}`: A client attribute named `NAME`, which can be initialized by +%% `mqtt.client_attrs_init` config or extended by certain authentication backends. +%% NOTE: Placeholder is not rendered as empty string if the referencing value is not +%% foud. For example, `${client_attrs.group}/#` is not rendered as `/#` if the +%% client does not have a `group` attribute. From 01923147a20500a2767122ca25aead86cc975d3d Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Apr 2024 16:20:47 +0200 Subject: [PATCH 34/86] fix(variform and authz): do not initialize empty client_attrs field when client_attrs_init expression renders to empty string, do not initialize the attribute. also fixed an ACL error: a template render failure for a topic would stop the ACL checks for the following topics if more than one topic is configured. --- apps/emqx/src/emqx_channel.erl | 9 ++++ apps/emqx_auth/etc/acl.conf | 4 +- .../src/emqx_authz/emqx_authz_rule.erl | 18 +++++++- .../test/emqx_authz/emqx_authz_SUITE.erl | 41 ++++++++++++++++++- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index efb5133bc..ec49d165c 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1638,6 +1638,15 @@ initialize_client_attrs(Inits, ClientInfo) -> fun(#{expression := Variform, set_as_attr := Name}, Acc) -> Attrs = maps:get(client_attrs, ClientInfo, #{}), case emqx_variform:render(Variform, ClientInfo) of + {ok, <<>>} -> + ?SLOG( + debug, + #{ + msg => "client_attr_rednered_to_empty_string", + set_as_attr => Name + } + ), + Acc; {ok, Value} -> ?SLOG( debug, diff --git a/apps/emqx_auth/etc/acl.conf b/apps/emqx_auth/etc/acl.conf index b23714558..3cc0ed5b8 100644 --- a/apps/emqx_auth/etc/acl.conf +++ b/apps/emqx_auth/etc/acl.conf @@ -6,8 +6,10 @@ {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. -%% NOTE: change to {deny, all} in production! {allow, all}. +%% NOTE! when deploy in production: +%% - Change the last rule to `{deny, all}.` +%% - Set config `authorization.no_match = deny` %% See docs below %% diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 9a877b5c8..b3bddada4 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -351,8 +351,9 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = bin(emqx_template:render_strict(PatternFilter, ClientInfo)), - match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse + TopicFilter = render_topic(PatternFilter, ClientInfo), + (is_binary(TopicFilter) andalso + match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter))) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> match_topic(emqx_topic:words(Topic), TopicFilter) orelse @@ -362,3 +363,16 @@ match_topic(Topic, {'eq', TopicFilter}) -> Topic =:= TopicFilter; match_topic(Topic, TopicFilter) -> emqx_topic:match(Topic, TopicFilter). + +render_topic(Topic, ClientInfo) -> + try + bin(emqx_template:render_strict(Topic, ClientInfo)) + catch + error:Reason -> + ?SLOG(debug, #{ + msg => "failed_to_render_topic_template", + template => Topic, + reason => Reason + }), + error + end. diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl index 70dd0bbb6..575eb4109 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl @@ -173,7 +173,16 @@ end_per_testcase(_TestCase, _Config) -> -define(SOURCE_FILE_CLIENT_ATTR, ?SOURCE_FILE( << - "{allow,all,all,[\"${client_attrs.alias}/#\"]}.\n" + "{allow,all,all,[\"${client_attrs.alias}/#\",\"client_attrs_backup\"]}.\n" + "{deny, all}." + >> + ) +). + +-define(SOURCE_FILE_CLIENT_NO_SUCH_ATTR, + ?SOURCE_FILE( + << + "{allow,all,all,[\"${client_attrs.nonexist}/#\",\"client_attrs_backup\"]}.\n" "{deny, all}." >> ) @@ -572,11 +581,41 @@ t_alias_prefix(_Config) -> ?assertMatch({ok, _}, emqtt:connect(C)), ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C, SubTopic)), ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, SubTopicNotAllowed)), + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, <<"/#">>)), unlink(C), emqtt:stop(C), + NonMatching = <<"clientid_which_has_no_dash">>, + {ok, C2} = emqtt:start_link([{clientid, NonMatching}, {proto_ver, v5}]), + ?assertMatch({ok, _}, emqtt:connect(C2)), + ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C2, <<"client_attrs_backup">>)), + %% assert '${client_attrs.alias}/#' is not rendered as '/#' + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C2, <<"/#">>)), + unlink(C2), + emqtt:stop(C2), emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []), ok. +t_non_existing_attr(_Config) -> + {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE_FILE_CLIENT_NO_SUCH_ATTR]), + %% '^.*-(.*)$': extract the suffix after the last '-' + {ok, Compiled} = emqx_variform:compile("concat(regex_extract(clientid,'^.*-(.*)$'))"), + emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [ + #{ + expression => Compiled, + %% this is intended to be different from 'nonexist' + set_as_attr => <<"existing">> + } + ]), + ClientId = <<"org1-name3">>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + ?assertMatch({ok, _}, emqtt:connect(C)), + ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C, <<"client_attrs_backup">>)), + %% assert '${client_attrs.nonexist}/#' is not rendered as '/#' + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, <<"/#">>)), + unlink(C), + emqtt:stop(C), + ok. + %% client is allowed by ACL to publish to its LWT topic, is connected, %% and then gets banned and kicked out while connected. Should not %% publish LWT. From 71cdcc860aba12e4ab95e30e1d4f8b53999fda92 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 17 Apr 2024 16:22:08 +0800 Subject: [PATCH 35/86] fix(plugin): plugin's mgmt api schema codes --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 14 +++++++------- .../test/emqx_mgmt_api_plugins_SUITE.erl | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 9e05d39a7..49905540d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -105,7 +105,7 @@ schema("/plugins/install") -> } }, responses => #{ - 200 => <<"OK">>, + 204 => <<"Install plugin successfully">>, 400 => emqx_dashboard_swagger:error_codes( ['UNEXPECTED_ERROR', 'ALREADY_INSTALLED', 'BAD_PLUGIN_INFO'] ) @@ -117,7 +117,7 @@ schema("/plugins/:name") -> 'operationId' => plugin, get => #{ summary => <<"Get a plugin description">>, - description => "Describs plugin according to its `release.json` and `README.md`.", + description => "Describe a plugin according to its `release.json` and `README.md`.", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -152,7 +152,7 @@ schema("/plugins/:name/:action") -> {action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})} ], responses => #{ - 200 => <<"OK">>, + 204 => <<"Trigger action successfully">>, 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } } @@ -161,13 +161,13 @@ schema("/plugins/:name/move") -> #{ 'operationId' => update_boot_order, post => #{ - summary => <<"Move plugin within plugin hiearchy">>, + summary => <<"Move plugin within plugin hierarchy">>, description => "Setting the boot order of plugins.", tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => move_request_body(), responses => #{ - 200 => <<"OK">>, + 204 => <<"Boot order changed successfully">>, 400 => emqx_dashboard_swagger:error_codes(['MOVE_FAILED'], <<"Move failed">>) } } @@ -382,7 +382,7 @@ do_install_package(FileName, Bin) -> {[_ | _] = Res, []} = emqx_mgmt_api_plugins_proto_v2:install_package(Nodes, FileName, Bin), case lists:filter(fun(R) -> R =/= ok end, Res) of [] -> - {200}; + {204}; Filtered -> %% crash if we have unexpected errors or results [] = lists:filter( @@ -425,7 +425,7 @@ update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> Position -> case emqx_plugins:ensure_enabled(Name, Position, _ConfLocation = global) of ok -> - {200}; + {204}; {error, Reason} -> {400, #{ code => 'MOVE_FAILED', diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 8106afc4a..4e5dacc7a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -271,7 +271,7 @@ install_plugin(FilePath) -> Token ) of - {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok; + {ok, {{"HTTP/1.1", 204, "No Content"}, _Headers, <<>>}} -> ok; Error -> Error end. @@ -288,7 +288,7 @@ install_plugin(Config, FilePath) -> Auth ) of - {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok; + {ok, {{"HTTP/1.1", 204, "No Content"}, _Headers, <<>>}} -> ok; Error -> Error end. From 8db5e515926184fa656008ded224cec71c76c2dd Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 19 Apr 2024 16:14:59 +0800 Subject: [PATCH 36/86] feat: plugin config with avro schema and apis --- apps/emqx/priv/bpapi.versions | 1 + .../src/emqx_mgmt_api_plugins.erl | 162 +++- .../proto/emqx_mgmt_api_plugins_proto_v2.erl | 1 + .../proto/emqx_mgmt_api_plugins_proto_v3.erl | 65 ++ apps/emqx_plugins/include/emqx_plugins.hrl | 21 + apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins.erl | 725 +++++++++++------- apps/emqx_plugins/src/emqx_plugins_serde.erl | 274 +++++++ apps/emqx_plugins/src/emqx_plugins_sup.erl | 12 +- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 24 +- apps/emqx_plugins/test/emqx_plugins_tests.erl | 4 +- 11 files changed, 980 insertions(+), 311 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl create mode 100644 apps/emqx_plugins/src/emqx_plugins_serde.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index a2913c37a..10d27fc63 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -46,6 +46,7 @@ {emqx_metrics,2}. {emqx_mgmt_api_plugins,1}. {emqx_mgmt_api_plugins,2}. +{emqx_mgmt_api_plugins,3}. {emqx_mgmt_cluster,1}. {emqx_mgmt_cluster,2}. {emqx_mgmt_cluster,3}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 49905540d..63a8ba517 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -19,7 +19,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -%%-include_lib("emqx_plugins/include/emqx_plugins.hrl"). +-include_lib("emqx_plugins/include/emqx_plugins.hrl"). -export([ api_spec/0, @@ -34,6 +34,8 @@ upload_install/2, plugin/2, update_plugin/2, + plugin_config/2, + plugin_schema/2, update_boot_order/2 ]). @@ -43,7 +45,8 @@ install_package/2, delete_package/1, describe_package/1, - ensure_action/2 + ensure_action/2, + do_update_plugin_config/3 ]). -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$"). @@ -52,7 +55,11 @@ %% app_name must be a snake_case (no '-' allowed). -define(VSN_WILDCARD, "-*.tar.gz"). -namespace() -> "plugins". +-define(CONTENT_PLUGIN, plugin). +-define(CONTENT_CONFIG, config). + +namespace() -> + "plugins". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -64,6 +71,8 @@ paths() -> "/plugins/:name", "/plugins/install", "/plugins/:name/:action", + "/plugins/:name/config", + "/plugins/:name/schema", "/plugins/:name/move" ]. @@ -97,10 +106,10 @@ schema("/plugins/install") -> schema => #{ type => object, properties => #{ - plugin => #{type => string, format => binary} + ?CONTENT_PLUGIN => #{type => string, format => binary} } }, - encoding => #{plugin => #{'contentType' => 'application/gzip'}} + encoding => #{?CONTENT_PLUGIN => #{'contentType' => 'application/gzip'}} } } }, @@ -157,6 +166,70 @@ schema("/plugins/:name/:action") -> } } }; +schema("/plugins/:name/config") -> + #{ + 'operationId' => plugin_config, + get => #{ + summary => + <<"Get plugin config">>, + description => + "Get plugin config by avro encoded binary config. Schema defined by user's schema.avsc file.
", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + %% binary avro encoded config + 200 => hoconsc:mk(binary()), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) + } + }, + put => #{ + summary => + <<"Update plugin config">>, + description => + "Update plugin config by avro encoded binary config. Schema defined by user's schema.avsc file.
", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + type => object, + properties => #{ + ?CONTENT_CONFIG => #{type => string, format => binary} + } + }, + encoding => #{?CONTENT_CONFIG => #{'contentType' => 'application/gzip'}} + } + } + }, + responses => #{ + 204 => <<"Config updated successfully">>, + 400 => emqx_dashboard_swagger:error_codes( + ['UNEXPECTED_ERROR'], <<"Update plugin config failed">> + ), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) + } + } + }; +schema("/plugins/:name/schema") -> + #{ + 'operationId' => plugin_schema, + get => #{ + summary => <<"Get installed plugin's avro schema">>, + description => + "Get plugin's config avro schema.", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + %% avro schema and i18n json object + 200 => hoconsc:mk(binary()), + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND', 'FILE_NOT_EXISTED'], + <<"Plugin Not Found or Plugin not given a schema file">> + ) + } + } + }; schema("/plugins/:name/move") -> #{ 'operationId' => update_boot_order, @@ -338,7 +411,7 @@ upload_install(post, #{body := #{<<"plugin">> := Plugin}}) when is_map(Plugin) - %% File bin is too large, we use rpc:multicall instead of cluster_rpc:multicall NameVsn = string:trim(FileName, trailing, ".tar.gz"), case emqx_plugins:describe(NameVsn) of - {error, #{error := "bad_info_file", return := {enoent, _}}} -> + {error, #{error_msg := "bad_info_file", reason := {enoent, _}}} -> case emqx_plugins:parse_name_vsn(FileName) of {ok, AppName, _Vsn} -> AppDir = filename:join(emqx_plugins:install_dir(), AppName), @@ -394,7 +467,7 @@ do_install_package(FileName, Bin) -> ), Reason = case hd(Filtered) of - {error, #{error := Reason0}} -> Reason0; + {error, #{error_msg := Reason0}} -> Reason0; {error, #{reason := Reason0}} -> Reason0 end, {400, #{ @@ -418,6 +491,42 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> Res = emqx_mgmt_api_plugins_proto_v2:ensure_action(Name, Action), return(204, Res). +plugin_config(get, #{bindings := #{name := Name}}) -> + case emqx_plugins:get_plugin_config(Name, #{format => ?CONFIG_FORMAT_AVRO}) of + {ok, AvroBin} -> + {200, #{<<"content-type">> => <<"application/octet-stream">>}, AvroBin}; + {error, _} -> + {400, #{ + code => 'BAD_CONFIG', + message => <<"Failed to get plugin config">> + }} + end; +plugin_config(put, #{bindings := #{name := Name}, body := #{<<"config">> := RawAvro}}) -> + case emqx_plugins:decode_plugin_avro_config(Name, RawAvro) of + {ok, Config} -> + Nodes = emqx:running_nodes(), + _Res = emqx_mgmt_api_plugins_proto_v3:update_plugin_config( + Nodes, Name, RawAvro, Config + ), + {204}; + {error, Reason} -> + {400, #{ + code => 'BAD_CONFIG', + message => readable_error_msg(Reason) + }} + end. + +plugin_schema(get, #{bindings := #{name := NameVsn}}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _Plugin} -> + {200, format_plugin_schema_with_i18n(NameVsn)}; + _ -> + {404, #{ + code => 'NOT_FOUND', + message => <<"Plugin Not Found">> + }} + end. + update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> case parse_position(Body, Name) of {error, Reason} -> @@ -429,7 +538,7 @@ update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> {error, Reason} -> {400, #{ code => 'MOVE_FAILED', - message => iolist_to_binary(io_lib:format("~p", [Reason])) + message => readable_error_msg(Reason) }} end end. @@ -443,7 +552,7 @@ install_package(FileName, Bin) -> ok = file:write_file(File, Bin), PackageName = string:trim(FileName, trailing, ".tar.gz"), case emqx_plugins:ensure_installed(PackageName) of - {error, #{return := not_found}} = NotFound -> + {error, #{reason := not_found}} = NotFound -> NotFound; {error, Reason} = Error -> ?SLOG(error, Reason#{msg => "failed_to_install_plugin"}), @@ -454,9 +563,9 @@ install_package(FileName, Bin) -> end. %% For RPC plugin get -describe_package(Name) -> +describe_package(NameVsn) -> Node = node(), - case emqx_plugins:describe(Name) of + case emqx_plugins:describe(NameVsn) of {ok, Plugin} -> {Node, [Plugin]}; _ -> {Node, []} end. @@ -487,12 +596,25 @@ ensure_action(Name, restart) -> _ = emqx_plugins:restart(Name), ok. +%% for RPC plugin avro encoded config update +do_update_plugin_config(Name, Avro, PluginConfig) -> + emqx_plugins:put_plugin_config(Name, Avro, PluginConfig). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + return(Code, ok) -> {Code}; -return(_, {error, #{error := "bad_info_file", return := {enoent, _} = Reason}}) -> - {404, #{code => 'NOT_FOUND', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}; +return(_, {error, #{error_msg := "bad_info_file", reason := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => readable_error_msg(Reason)}}; +return(_, {error, #{error_msg := "bad_avro_config_file", reason := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => readable_error_msg(Reason)}}; return(_, {error, Reason}) -> - {400, #{code => 'PARAM_ERROR', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}. + {400, #{code => 'PARAM_ERROR', message => readable_error_msg(Reason)}}. + +readable_error_msg(Msg) -> + emqx_utils:readable_error_msg(Msg). parse_position(#{<<"position">> := <<"front">>}, _) -> front; @@ -563,6 +685,18 @@ aggregate_status([{Node, Plugins} | List], Acc) -> ), aggregate_status(List, NewAcc). +format_plugin_schema_with_i18n(NameVsn) -> + #{ + avsc => try_read_file(fun() -> emqx_plugins:plugin_avsc(NameVsn) end), + i18n => try_read_file(fun() -> emqx_plugins:plugin_i18n(NameVsn) end) + }. + +try_read_file(Fun) -> + case Fun() of + {ok, Bin} -> Bin; + _ -> null + end. + % running_status: running loaded, stopped %% config_status: not_configured disable enable plugin_status(#{running_status := running}) -> running; diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl index 781096af0..91d4674f9 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl @@ -24,6 +24,7 @@ describe_package/2, delete_package/1, ensure_action/2 + %% plugin_config/2 ]). -include_lib("emqx/include/bpapi.hrl"). diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl new file mode 100644 index 000000000..13c13bae7 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_plugins_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + get_plugins/1, + install_package/3, + describe_package/2, + delete_package/1, + ensure_action/2, + update_plugin_config/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.7.0". + +-spec get_plugins([node()]) -> emqx_rpc:multicall_result(). +get_plugins(Nodes) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, get_plugins, [], 15000). + +-spec install_package([node()], binary() | string(), binary()) -> emqx_rpc:multicall_result(). +install_package(Nodes, Filename, Bin) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, install_package, [Filename, Bin], 25000). + +-spec describe_package([node()], binary() | string()) -> emqx_rpc:multicall_result(). +describe_package(Nodes, Name) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, describe_package, [Name], 10000). + +-spec delete_package(binary() | string()) -> ok | {error, any()}. +delete_package(Name) -> + emqx_cluster_rpc:multicall(emqx_mgmt_api_plugins, delete_package, [Name], all, 10000). + +-spec ensure_action(binary() | string(), 'restart' | 'start' | 'stop') -> ok | {error, any()}. +ensure_action(Name, Action) -> + emqx_cluster_rpc:multicall(emqx_mgmt_api_plugins, ensure_action, [Name, Action], all, 10000). + +-spec update_plugin_config( + [node()], + binary() | string(), + binary(), + map() +) -> + emqx_rpc:multicall_result(). +update_plugin_config(Nodes, Name, RawAvro, PluginConfig) -> + rpc:multicall( + Nodes, emqx_mgmt_api_plugins, do_update_plugin_config, [Name, RawAvro, PluginConfig], 10000 + ). diff --git a/apps/emqx_plugins/include/emqx_plugins.hrl b/apps/emqx_plugins/include/emqx_plugins.hrl index 05959e46f..95dc50e4f 100644 --- a/apps/emqx_plugins/include/emqx_plugins.hrl +++ b/apps/emqx_plugins/include/emqx_plugins.hrl @@ -19,4 +19,25 @@ -define(CONF_ROOT, plugins). +-define(PLUGIN_SERDE_TAB, emqx_plugins_schema_serde_tab). + +-define(CONFIG_FORMAT_AVRO, config_format_avro). +-define(CONFIG_FORMAT_MAP, config_format_map). + +-type schema_name() :: binary(). +-type avsc() :: binary(). + +-type encoded_data() :: iodata(). +-type decoded_data() :: map(). + +-record(plugin_schema_serde, { + name :: schema_name(), + eval_context :: term(), + %% TODO: fields to mark schema import status + %% scheam_imported :: boolean(), + %% for future use + extra = [] +}). +-type plugin_schema_serde() :: #plugin_schema_serde{}. + -endif. diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index b26836475..0e0945c7f 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.8"}, + {vsn, "0.2.0"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index ebe5b7932..ce66fc94d 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -16,7 +16,6 @@ -module(emqx_plugins). --include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl"). -include("emqx_plugins.hrl"). @@ -24,6 +23,15 @@ -include_lib("eunit/include/eunit.hrl"). -endif. +-export([ + describe/1, + plugin_avsc/1, + plugin_i18n/1, + plugin_avro/1, + parse_name_vsn/1 +]). + +%% Package operations -export([ ensure_installed/1, ensure_uninstalled/1, @@ -35,21 +43,26 @@ delete_package/1 ]). +%% Plugin runtime management -export([ ensure_started/0, ensure_started/1, ensure_stopped/0, ensure_stopped/1, + get_plugin_config/1, + get_plugin_config/2, + put_plugin_config/3, restart/1, - list/0, - describe/1, - parse_name_vsn/1 + list/0 ]). +%% Package utils -export([ + decode_plugin_avro_config/2, get_config/2, put_config/2, - get_tar/1 + get_tar/1, + install_dir/0 ]). %% `emqx_config_handler' API @@ -57,21 +70,26 @@ post_config_update/5 ]). -%% internal +%% Internal export -export([do_ensure_started/1]). --export([ - install_dir/0 -]). -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. +%% Defines +-define(PLUGIN_PERSIS_CONFIG_KEY(NameVsn), {?MODULE, NameVsn}). + +%% Types %% "my_plugin-0.1.0" -type name_vsn() :: binary() | string(). %% the parse result of the JSON info file -type plugin() :: map(). +-type schema_json() :: map(). +-type i18n_json() :: map(). +-type avro_binary() :: binary(). +-type plugin_config() :: map(). -type position() :: no_move | front | rear | {before, name_vsn()} | {behind, name_vsn()}. %%-------------------------------------------------------------------- @@ -80,12 +98,36 @@ %% @doc Describe a plugin. -spec describe(name_vsn()) -> {ok, plugin()} | {error, any()}. -describe(NameVsn) -> read_plugin(NameVsn, #{fill_readme => true}). +describe(NameVsn) -> + read_plugin_info(NameVsn, #{fill_readme => true}). + +-spec plugin_avsc(name_vsn()) -> {ok, schema_json()} | {error, any()}. +plugin_avsc(NameVsn) -> + read_plugin_avsc(NameVsn). + +-spec plugin_i18n(name_vsn()) -> {ok, i18n_json()} | {error, any()}. +plugin_i18n(NameVsn) -> + read_plugin_i18n(NameVsn). + +-spec plugin_avro(name_vsn()) -> {ok, avro_binary()} | {error, any()}. +plugin_avro(NameVsn) -> + read_plugin_avro(NameVsn). + +parse_name_vsn(NameVsn) when is_binary(NameVsn) -> + parse_name_vsn(binary_to_list(NameVsn)); +parse_name_vsn(NameVsn) when is_list(NameVsn) -> + case lists:splitwith(fun(X) -> X =/= $- end, NameVsn) of + {AppName, [$- | Vsn]} -> {ok, list_to_atom(AppName), Vsn}; + _ -> {error, "bad_name_vsn"} + end. + +%%-------------------------------------------------------------------- +%% Package operations %% @doc Install a .tar.gz package placed in install_dir. -spec ensure_installed(name_vsn()) -> ok | {error, map()}. ensure_installed(NameVsn) -> - case read_plugin(NameVsn, #{}) of + case read_plugin_info(NameVsn, #{}) of {ok, _} -> ok; {error, _} -> @@ -93,33 +135,183 @@ ensure_installed(NameVsn) -> do_ensure_installed(NameVsn) end. -do_ensure_installed(NameVsn) -> - TarGz = pkg_file(NameVsn), - case erl_tar:extract(TarGz, [compressed, memory]) of - {ok, TarContent} -> - ok = write_tar_file_content(install_dir(), TarContent), - case read_plugin(NameVsn, #{}) of - {ok, _} -> - ok; - {error, Reason} -> - ?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}), - ok = delete_tar_file_content(install_dir(), TarContent), - {error, Reason} - end; - {error, {_, enoent}} -> +%% @doc Ensure files and directories for the given plugin are being deleted. +%% If a plugin is running, or enabled, an error is returned. +-spec ensure_uninstalled(name_vsn()) -> ok | {error, any()}. +ensure_uninstalled(NameVsn) -> + case read_plugin_info(NameVsn, #{}) of + {ok, #{running_status := RunningSt}} when RunningSt =/= stopped -> {error, #{ - reason => "failed_to_extract_plugin_package", - path => TarGz, - return => not_found + error_msg => "bad_plugin_running_status", + hint => "stop_the_plugin_first" }}; - {error, Reason} -> + {ok, #{config_status := enabled}} -> {error, #{ - reason => "bad_plugin_package", - path => TarGz, - return => Reason - }} + error_msg => "bad_plugin_config_status", + hint => "disable_the_plugin_first" + }}; + _ -> + purge(NameVsn), + ensure_delete(NameVsn) end. +%% @doc Ensure a plugin is enabled to the end of the plugins list. +-spec ensure_enabled(name_vsn()) -> ok | {error, any()}. +ensure_enabled(NameVsn) -> + ensure_enabled(NameVsn, no_move). + +%% @doc Ensure a plugin is enabled at the given position of the plugin list. +-spec ensure_enabled(name_vsn(), position()) -> ok | {error, any()}. +ensure_enabled(NameVsn, Position) -> + ensure_state(NameVsn, Position, _Enabled = true, _ConfLocation = local). + +-spec ensure_enabled(name_vsn(), position(), local | global) -> ok | {error, any()}. +ensure_enabled(NameVsn, Position, ConfLocation) when + ConfLocation =:= local; ConfLocation =:= global +-> + ensure_state(NameVsn, Position, _Enabled = true, ConfLocation). + +%% @doc Ensure a plugin is disabled. +-spec ensure_disabled(name_vsn()) -> ok | {error, any()}. +ensure_disabled(NameVsn) -> + ensure_state(NameVsn, no_move, false, _ConfLocation = local). + +%% @doc Delete extracted dir +%% In case one lib is shared by multiple plugins. +%% it might be the case that purging one plugin's install dir +%% will cause deletion of loaded beams. +%% It should not be a problem, because shared lib should +%% reside in all the plugin install dirs. +-spec purge(name_vsn()) -> ok. +purge(NameVsn) -> + _ = maybe_purge_plugin_config(NameVsn), + purge_plugin(NameVsn). + +%% @doc Delete the package file. +-spec delete_package(name_vsn()) -> ok. +delete_package(NameVsn) -> + File = pkg_file(NameVsn), + _ = emqx_plugins_serde:delete_schema(NameVsn), + case file:delete(File) of + ok -> + ?SLOG(info, #{msg => "purged_plugin_dir", path => File}), + ok; + {error, enoent} -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_delete_package_file", + path => File, + reason => Reason + }), + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% Plugin runtime management + +%% @doc Start all configured plugins are started. +-spec ensure_started() -> ok. +ensure_started() -> + ok = for_plugins(fun ?MODULE:do_ensure_started/1). + +%% @doc Start a plugin from Management API or CLI. +%% the input is a - string. +-spec ensure_started(name_vsn()) -> ok | {error, term()}. +ensure_started(NameVsn) -> + case do_ensure_started(NameVsn) of + ok -> + ok; + {error, Reason} -> + ?SLOG(alert, Reason#{msg => "failed_to_start_plugin"}), + {error, Reason} + end. + +%% @doc Stop all plugins before broker stops. +-spec ensure_stopped() -> ok. +ensure_stopped() -> + for_plugins(fun ?MODULE:ensure_stopped/1). + +%% @doc Stop a plugin from Management API or CLI. +-spec ensure_stopped(name_vsn()) -> ok | {error, term()}. +ensure_stopped(NameVsn) -> + tryit( + "stop_plugin", + fun() -> + Plugin = do_read_plugin(NameVsn), + ensure_apps_stopped(Plugin) + end + ). + +-spec get_plugin_config(name_vsn()) -> + {ok, plugin_config()} | {error, term()}. +get_plugin_config(NameVsn) -> + get_plugin_config(NameVsn, #{format => ?CONFIG_FORMAT_MAP}). + +-spec get_plugin_config(name_vsn(), Options :: map()) -> + {ok, avro_binary() | plugin_config()} + | {error, term()}. +get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> + case read_plugin_avro(NameVsn) of + {ok, _AvroBin} = Res -> Res; + {error, _Reason} = Err -> Err + end; +get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}) -> + persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), #{}). + +%% @doc Update plugin's config. +%% RPC call from Management API or CLI. +%% the avro binary and plugin config ALWAYS be valid before calling this function. +put_plugin_config(NameVsn, RawAvro, PluginConfig) -> + ok = write_avro_bin(NameVsn, RawAvro), + ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), PluginConfig), + ok. + +%% @doc Stop and then start the plugin. +restart(NameVsn) -> + case ensure_stopped(NameVsn) of + ok -> ensure_started(NameVsn); + {error, Reason} -> {error, Reason} + end. + +%% @doc List all installed plugins. +%% Including the ones that are installed, but not enabled in config. +-spec list() -> [plugin()]. +list() -> + Pattern = filename:join([install_dir(), "*", "release.json"]), + All = lists:filtermap( + fun(JsonFilePath) -> + [_, NameVsn | _] = lists:reverse(filename:split(JsonFilePath)), + case read_plugin_info(NameVsn, #{}) of + {ok, Info} -> + {true, Info}; + {error, Reason} -> + ?SLOG(warning, Reason), + false + end + end, + filelib:wildcard(Pattern) + ), + do_list(configured(), All). + +%%-------------------------------------------------------------------- +%% Package utils + +-spec decode_plugin_avro_config(name_vsn(), binary()) -> {ok, map()} | {error, any()}. +decode_plugin_avro_config(NameVsn, RawAvro) -> + case emqx_plugins_serde:decode(NameVsn, RawAvro) of + {ok, Config} -> {ok, Config}; + {error, ReasonMap} -> {error, ReasonMap} + end. + +get_config(Key, Default) when is_atom(Key) -> + get_config([Key], Default); +get_config(Path, Default) -> + emqx_conf:get([?CONF_ROOT | Path], Default). + +put_config(Key, Value) -> + do_put_config(Key, Value, _ConfLocation = local). + -spec get_tar(name_vsn()) -> {ok, binary()} | {error, any}. get_tar(NameVsn) -> TarGz = pkg_file(NameVsn), @@ -135,10 +327,14 @@ get_tar(NameVsn) -> end end. +%%-------------------------------------------------------------------- +%% Internal +%%-------------------------------------------------------------------- + maybe_create_tar(NameVsn, TarGzName, InstallDir) when is_binary(InstallDir) -> maybe_create_tar(NameVsn, TarGzName, binary_to_list(InstallDir)); maybe_create_tar(NameVsn, TarGzName, InstallDir) -> - case filelib:wildcard(filename:join(dir(NameVsn), "**")) of + case filelib:wildcard(filename:join(plugin_dir(NameVsn), "**")) of [_ | _] = PluginFiles -> InstallDir1 = string:trim(InstallDir, trailing, "/") ++ "/", PluginFiles1 = [{string:prefix(F, InstallDir1), F} || F <- PluginFiles], @@ -207,24 +403,32 @@ top_dir_test_() -> ]. -endif. -%% @doc Ensure files and directories for the given plugin are being deleted. -%% If a plugin is running, or enabled, an error is returned. --spec ensure_uninstalled(name_vsn()) -> ok | {error, any()}. -ensure_uninstalled(NameVsn) -> - case read_plugin(NameVsn, #{}) of - {ok, #{running_status := RunningSt}} when RunningSt =/= stopped -> +do_ensure_installed(NameVsn) -> + TarGz = pkg_file(NameVsn), + case erl_tar:extract(TarGz, [compressed, memory]) of + {ok, TarContent} -> + ok = write_tar_file_content(install_dir(), TarContent), + case read_plugin_info(NameVsn, #{}) of + {ok, _} -> + ok = maybe_post_op_after_install(NameVsn), + ok; + {error, Reason} -> + ?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}), + ok = delete_tar_file_content(install_dir(), TarContent), + {error, Reason} + end; + {error, {_, enoent}} -> {error, #{ - reason => "bad_plugin_running_status", - hint => "stop_the_plugin_first" + error_msg => "failed_to_extract_plugin_package", + path => TarGz, + reason => not_found }}; - {ok, #{config_status := enabled}} -> + {error, Reason} -> {error, #{ - reason => "bad_plugin_config_status", - hint => "disable_the_plugin_first" - }}; - _ -> - purge(NameVsn), - ensure_delete(NameVsn) + error_msg => "bad_plugin_package", + path => TarGz, + reason => Reason + }} end. ensure_delete(NameVsn0) -> @@ -233,37 +437,19 @@ ensure_delete(NameVsn0) -> put_configured(lists:filter(fun(#{name_vsn := N1}) -> bin(N1) =/= NameVsn end, List)), ok. -%% @doc Ensure a plugin is enabled to the end of the plugins list. --spec ensure_enabled(name_vsn()) -> ok | {error, any()}. -ensure_enabled(NameVsn) -> - ensure_enabled(NameVsn, no_move). - -%% @doc Ensure a plugin is enabled at the given position of the plugin list. --spec ensure_enabled(name_vsn(), position()) -> ok | {error, any()}. -ensure_enabled(NameVsn, Position) -> - ensure_state(NameVsn, Position, _Enabled = true, _ConfLocation = local). - --spec ensure_enabled(name_vsn(), position(), local | global) -> ok | {error, any()}. -ensure_enabled(NameVsn, Position, ConfLocation) when - ConfLocation =:= local; ConfLocation =:= global --> - ensure_state(NameVsn, Position, _Enabled = true, ConfLocation). - -%% @doc Ensure a plugin is disabled. --spec ensure_disabled(name_vsn()) -> ok | {error, any()}. -ensure_disabled(NameVsn) -> - ensure_state(NameVsn, no_move, false, _ConfLocation = local). - ensure_state(NameVsn, Position, State, ConfLocation) when is_binary(NameVsn) -> ensure_state(binary_to_list(NameVsn), Position, State, ConfLocation); ensure_state(NameVsn, Position, State, ConfLocation) -> - case read_plugin(NameVsn, #{}) of + case read_plugin_info(NameVsn, #{}) of {ok, _} -> Item = #{ name_vsn => NameVsn, enable => State }, - tryit("ensure_state", fun() -> ensure_configured(Item, Position, ConfLocation) end); + tryit( + "ensure_state", + fun() -> ensure_configured(Item, Position, ConfLocation) end + ); {error, Reason} -> {error, Reason} end. @@ -295,7 +481,7 @@ add_new_configured(Configured, {Action, NameVsn}, Item) -> {Front, Rear} = lists:splitwith(SplitFun, Configured), Rear =:= [] andalso throw(#{ - error => "position_anchor_plugin_not_configured", + error_msg => "position_anchor_plugin_not_configured", hint => "maybe_install_and_configure", name_vsn => NameVsn }), @@ -307,37 +493,21 @@ add_new_configured(Configured, {Action, NameVsn}, Item) -> Front ++ [Anchor, Item | Rear0] end. -%% @doc Delete the package file. --spec delete_package(name_vsn()) -> ok. -delete_package(NameVsn) -> - File = pkg_file(NameVsn), - case file:delete(File) of - ok -> - ?SLOG(info, #{msg => "purged_plugin_dir", path => File}), - ok; - {error, enoent} -> - ok; - {error, Reason} -> - ?SLOG(error, #{ - msg => "failed_to_delete_package_file", - path => File, - reason => Reason - }), - {error, Reason} - end. +maybe_purge_plugin_config(NameVsn) -> + _ = persistent_term:erase(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn)), + ok. -%% @doc Delete extracted dir -%% In case one lib is shared by multiple plugins. -%% it might be the case that purging one plugin's install dir -%% will cause deletion of loaded beams. -%% It should not be a problem, because shared lib should -%% reside in all the plugin install dirs. --spec purge(name_vsn()) -> ok. -purge(NameVsn) -> - Dir = dir(NameVsn), +purge_plugin(NameVsn) -> + Dir = plugin_dir(NameVsn), + purge_plugin_dir(Dir). + +purge_plugin_dir(Dir) -> case file:del_dir_r(Dir) of ok -> - ?SLOG(info, #{msg => "purged_plugin_dir", dir => Dir}); + ?SLOG(info, #{ + msg => "purged_plugin_dir", + dir => Dir + }); {error, enoent} -> ok; {error, Reason} -> @@ -349,72 +519,10 @@ purge(NameVsn) -> {error, Reason} end. -%% @doc Start all configured plugins are started. --spec ensure_started() -> ok. -ensure_started() -> - ok = for_plugins(fun ?MODULE:do_ensure_started/1). - -%% @doc Start a plugin from Management API or CLI. -%% the input is a - string. --spec ensure_started(name_vsn()) -> ok | {error, term()}. -ensure_started(NameVsn) -> - case do_ensure_started(NameVsn) of - ok -> - ok; - {error, Reason} -> - ?SLOG(alert, #{ - msg => "failed_to_start_plugin", - reason => Reason - }), - {error, Reason} - end. - -%% @doc Stop all plugins before broker stops. --spec ensure_stopped() -> ok. -ensure_stopped() -> - for_plugins(fun ?MODULE:ensure_stopped/1). - -%% @doc Stop a plugin from Management API or CLI. --spec ensure_stopped(name_vsn()) -> ok | {error, term()}. -ensure_stopped(NameVsn) -> - tryit( - "stop_plugin", - fun() -> - Plugin = do_read_plugin(NameVsn), - ensure_apps_stopped(Plugin) - end - ). - -%% @doc Stop and then start the plugin. -restart(NameVsn) -> - case ensure_stopped(NameVsn) of - ok -> ensure_started(NameVsn); - {error, Reason} -> {error, Reason} - end. - -%% @doc List all installed plugins. -%% Including the ones that are installed, but not enabled in config. --spec list() -> [plugin()]. -list() -> - Pattern = filename:join([install_dir(), "*", "release.json"]), - All = lists:filtermap( - fun(JsonFile) -> - case read_plugin({file, JsonFile}, #{}) of - {ok, Info} -> - {true, Info}; - {error, Reason} -> - ?SLOG(warning, Reason), - false - end - end, - filelib:wildcard(Pattern) - ), - list(configured(), All). - %% Make sure configured ones are ordered in front. -list([], All) -> +do_list([], All) -> All; -list([#{name_vsn := NameVsn} | Rest], All) -> +do_list([#{name_vsn := NameVsn} | Rest], All) -> SplitF = fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) -> bin([Name, "-", Vsn]) =/= bin(NameVsn) end, @@ -424,9 +532,9 @@ list([#{name_vsn := NameVsn} | Rest], All) -> msg => "configured_plugin_not_installed", name_vsn => NameVsn }), - list(Rest, All); + do_list(Rest, All); {Front, [I | Rear]} -> - [I | list(Rest, Front ++ Rear)] + [I | do_list(Rest, Front ++ Rear)] end. do_ensure_started(NameVsn) -> @@ -439,23 +547,26 @@ do_ensure_started(NameVsn) -> ok = load_code_start_apps(NameVsn, Plugin); {error, plugin_not_found} -> ?SLOG(error, #{ - msg => "plugin_not_found", + error_msg => "plugin_not_found", name_vsn => NameVsn - }) + }), + ok end end ). +%%-------------------------------------------------------------------- + %% try the function, catch 'throw' exceptions as normal 'error' return %% other exceptions with stacktrace logged. tryit(WhichOp, F) -> try F() catch - throw:Reason -> + throw:ReasonMap -> %% thrown exceptions are known errors %% translate to a return value without stacktrace - {error, Reason}; + {error, ReasonMap}; error:Reason:Stacktrace -> %% unexpected errors, log stacktrace ?SLOG(warning, #{ @@ -469,33 +580,44 @@ tryit(WhichOp, F) -> %% read plugin info from the JSON file %% returns {ok, Info} or {error, Reason} -read_plugin(NameVsn, Options) -> +read_plugin_info(NameVsn, Options) -> tryit( - "read_plugin_info", - fun() -> {ok, do_read_plugin(NameVsn, Options)} end + atom_to_list(?FUNCTION_NAME), + fun() -> {ok, do_read_plugin2(NameVsn, Options)} end ). -do_read_plugin(Plugin) -> do_read_plugin(Plugin, #{}). +do_read_plugin(NameVsn) -> + do_read_plugin2(NameVsn, #{}). -do_read_plugin({file, InfoFile}, Options) -> - [_, NameVsn | _] = lists:reverse(filename:split(InfoFile)), - case hocon:load(InfoFile, #{format => richmap}) of - {ok, RichMap} -> - Info0 = check_plugin(hocon_maps:ensure_plain(RichMap), NameVsn, InfoFile), - Info1 = plugins_readme(NameVsn, Options, Info0), - plugin_status(NameVsn, Info1); - {error, Reason} -> - throw(#{ - error => "bad_info_file", - path => InfoFile, - return => Reason - }) - end; -do_read_plugin(NameVsn, Options) -> - do_read_plugin({file, info_file(NameVsn)}, Options). +do_read_plugin2(NameVsn, Option) -> + do_read_plugin3(NameVsn, info_file(NameVsn), Option). + +do_read_plugin3(NameVsn, InfoFilePath, Options) -> + {ok, PlainMap} = (read_file_fun(InfoFilePath, "bad_info_file"))(), + Info0 = check_plugin(PlainMap, NameVsn, InfoFilePath), + Info1 = plugins_readme(NameVsn, Options, Info0), + plugin_status(NameVsn, Info1). + +read_plugin_avsc(NameVsn) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(schema_file(NameVsn), "bad_avsc_file") + ). + +read_plugin_i18n(NameVsn) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(i18n_file(NameVsn), "bad_i18n_file") + ). + +read_plugin_avro(NameVsn) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(schema_file(NameVsn), "bad_avro_file") + ). ensure_exists_and_installed(NameVsn) -> - case filelib:is_dir(dir(NameVsn)) of + case filelib:is_dir(plugin_dir(NameVsn)) of true -> ok; false -> @@ -581,10 +703,6 @@ plugin_status(NameVsn, Info) -> config_status => ConfSt }. -bin(A) when is_atom(A) -> atom_to_binary(A, utf8); -bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8); -bin(B) when is_binary(B) -> B. - check_plugin( #{ <<"name">> := Name, @@ -593,7 +711,7 @@ check_plugin( <<"description">> := _ } = Info, NameVsn, - File + FilePath ) -> case bin(NameVsn) =:= bin([Name, "-", Vsn]) of true -> @@ -605,7 +723,7 @@ check_plugin( catch _:_ -> throw(#{ - error => "bad_rel_apps", + error_msg => "bad_rel_apps", rel_apps => Apps, hint => "A non-empty string list of app_name-app_vsn format" }) @@ -613,16 +731,16 @@ check_plugin( Info; false -> throw(#{ - error => "name_vsn_mismatch", + error_msg => "name_vsn_mismatch", name_vsn => NameVsn, - path => File, + path => FilePath, name => Name, rel_vsn => Vsn }) end; check_plugin(_What, NameVsn, File) -> throw(#{ - error => "bad_info_file_content", + error_msg => "bad_info_file_content", mandatory_fields => [rel_vsn, name, rel_apps, description], name_vsn => NameVsn, path => File @@ -678,7 +796,7 @@ do_load_plugin_app(AppName, Ebin) -> ok; {error, Reason} -> throw(#{ - error => "failed_to_load_plugin_beam", + error_msg => "failed_to_load_plugin_beam", path => BeamFile, reason => Reason }) @@ -693,7 +811,7 @@ do_load_plugin_app(AppName, Ebin) -> ok; {error, Reason} -> throw(#{ - error => "failed_to_load_plugin_app", + error_msg => "failed_to_load_plugin_app", name => AppName, reason => Reason }) @@ -710,7 +828,7 @@ start_app(App) -> ok; {error, {ErrApp, Reason}} -> throw(#{ - error => "failed_to_start_plugin_app", + error_msg => "failed_to_start_plugin_app", app => App, err_app => ErrApp, reason => Reason @@ -775,7 +893,7 @@ stop_app(App) -> ?SLOG(debug, #{msg => "plugin_not_started", app => App}), ok = unload_moudle_and_app(App); {error, Reason} -> - throw(#{error => "failed_to_stop_app", app => App, reason => Reason}) + throw(#{error_msg => "failed_to_stop_app", app => App, reason => Reason}) end. unload_moudle_and_app(App) -> @@ -802,94 +920,22 @@ is_needed_by(AppToStop, RunningApp) -> undefined -> false end. -put_config(Key, Value) -> - put_config(Key, Value, _ConfLocation = local). - -put_config(Key, Value, ConfLocation) when is_atom(Key) -> - put_config([Key], Value, ConfLocation); -put_config(Path, Values, _ConfLocation = local) when is_list(Path) -> +do_put_config(Key, Value, ConfLocation) when is_atom(Key) -> + do_put_config([Key], Value, ConfLocation); +do_put_config(Path, Values, _ConfLocation = local) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, %% Already in cluster_rpc, don't use emqx_conf:update, dead calls case emqx:update_config([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error end; -put_config(Path, Values, _ConfLocation = global) when is_list(Path) -> +do_put_config(Path, Values, _ConfLocation = global) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, case emqx_conf:update([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error end. -bin_key(Map) when is_map(Map) -> - maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map); -bin_key(List = [#{} | _]) -> - lists:map(fun(M) -> bin_key(M) end, List); -bin_key(Term) -> - Term. - -get_config(Key, Default) when is_atom(Key) -> - get_config([Key], Default); -get_config(Path, Default) -> - emqx_conf:get([?CONF_ROOT | Path], Default). - -install_dir() -> get_config(install_dir, ""). - -put_configured(Configured) -> - put_configured(Configured, _ConfLocation = local). - -put_configured(Configured, ConfLocation) -> - ok = put_config(states, bin_key(Configured), ConfLocation). - -configured() -> - get_config(states, []). - -for_plugins(ActionFun) -> - case lists:flatmap(fun(I) -> for_plugin(I, ActionFun) end, configured()) of - [] -> ok; - Errors -> erlang:error(#{function => ActionFun, errors => Errors}) - end. - -for_plugin(#{name_vsn := NameVsn, enable := true}, Fun) -> - case Fun(NameVsn) of - ok -> []; - {error, Reason} -> [{NameVsn, Reason}] - end; -for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) -> - ?SLOG(debug, #{ - msg => "plugin_disabled", - name_vsn => NameVsn - }), - []. - -parse_name_vsn(NameVsn) when is_binary(NameVsn) -> - parse_name_vsn(binary_to_list(NameVsn)); -parse_name_vsn(NameVsn) when is_list(NameVsn) -> - case lists:splitwith(fun(X) -> X =/= $- end, NameVsn) of - {AppName, [$- | Vsn]} -> {ok, list_to_atom(AppName), Vsn}; - _ -> {error, "bad_name_vsn"} - end. - -pkg_file(NameVsn) -> - filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). - -dir(NameVsn) -> - filename:join([install_dir(), NameVsn]). - -info_file(NameVsn) -> - filename:join([dir(NameVsn), "release.json"]). - -readme_file(NameVsn) -> - filename:join([dir(NameVsn), "README.md"]). - -running_apps() -> - lists:map( - fun({N, _, V}) -> - {N, V} - end, - application:which_applications(infinity) - ). - %%-------------------------------------------------------------------- %% `emqx_config_handler' API %%-------------------------------------------------------------------- @@ -913,3 +959,120 @@ enable_disable_plugin(NameVsn, {#{enable := false}, #{enable := true}}) -> ok; enable_disable_plugin(_NameVsn, _Diff) -> ok. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +install_dir() -> + get_config(install_dir, ""). + +put_configured(Configured) -> + put_configured(Configured, _ConfLocation = local). + +put_configured(Configured, ConfLocation) -> + ok = do_put_config(states, bin_key(Configured), ConfLocation). + +configured() -> + get_config(states, []). + +for_plugins(ActionFun) -> + case lists:flatmap(fun(I) -> for_plugin(I, ActionFun) end, configured()) of + [] -> ok; + Errors -> erlang:error(#{function => ActionFun, errors => Errors}) + end. + +for_plugin(#{name_vsn := NameVsn, enable := true}, Fun) -> + case Fun(NameVsn) of + ok -> []; + {error, Reason} -> [{NameVsn, Reason}] + end; +for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) -> + ?SLOG(debug, #{ + msg => "plugin_disabled", + name_vsn => NameVsn + }), + []. + +maybe_post_op_after_install(NameVsn) -> + _ = maybe_load_config_schema(NameVsn), + _ = maybe_create_config_dir(NameVsn), + ok. + +maybe_load_config_schema(NameVsn) -> + case read_plugin_avsc(NameVsn) of + {ok, Avsc} -> + case emqx_plugins_serde:add_schema(NameVsn, Avsc) of + ok -> ok; + {error, already_exists} -> ok; + {error, Reason} -> {error, Reason} + end; + {error, Reason} -> + ?SLOG(warning, Reason) + end. + +maybe_create_config_dir(NameVsn) -> + case filelib:ensure_path(plugin_config_dir(NameVsn)) of + ok -> ok; + {error, Reason} -> ?SLOG(warning, Reason) + end. + +write_avro_bin(NameVsn, AvroBin) -> + ok = file:write_file(avro_config_file(NameVsn), AvroBin). + +read_file_fun(Path, ErrMsg) -> + fun() -> + case hocon:load(Path, #{format => richmap}) of + {ok, RichMap} -> + {ok, hocon_maps:ensure_plain(RichMap)}; + {error, Reason} -> + ErrMeta = #{error_msg => ErrMsg, reason => Reason}, + ?SLOG(warning, ErrMeta), + throw(ErrMeta) + end + end. + +%% Directorys +plugin_dir(NameVsn) -> + filename:join([install_dir(), NameVsn]). + +plugin_config_dir(NameVsn) -> + filename:join([plugin_dir(NameVsn), "data", "configs"]). + +%% Files +pkg_file(NameVsn) -> + filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). + +info_file(NameVsn) -> + filename:join([plugin_dir(NameVsn), "release.json"]). + +schema_file(NameVsn) -> + filename:join([plugin_dir(NameVsn), "config_schema.avsc"]). + +avro_config_file(NameVsn) -> + filename:join([plugin_config_dir(NameVsn), "config.avro"]). + +i18n_file(NameVsn) -> + filename:join([plugin_dir(NameVsn), "i18n.json"]). + +readme_file(NameVsn) -> + filename:join([plugin_dir(NameVsn), "README.md"]). + +running_apps() -> + lists:map( + fun({N, _, V}) -> + {N, V} + end, + application:which_applications(infinity) + ). + +bin_key(Map) when is_map(Map) -> + maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map); +bin_key(List = [#{} | _]) -> + lists:map(fun(M) -> bin_key(M) end, List); +bin_key(Term) -> + Term. + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8); +bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl new file mode 100644 index 000000000..a89f16e70 --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -0,0 +1,274 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugins_serde). + +-include("emqx_plugins.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ + start_link/0, + get_serde/1, + add_schema/2, + get_schema/1, + delete_schema/1 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_continue/2, + terminate/2 +]). + +-export([ + decode/2, + encode/2 +]). + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec get_serde(schema_name()) -> {ok, plugin_schema_serde()} | {error, not_found}. +get_serde(SchemaName) -> + case ets:lookup(?PLUGIN_SERDE_TAB, to_bin(SchemaName)) of + [] -> + {error, not_found}; + [Serde] -> + {ok, Serde} + end. + +-spec add_schema(schema_name(), avsc()) -> ok | {error, term()}. +add_schema(Name, Avsc) -> + case get_serde(Name) of + {ok, _Serde} -> + ?SLOG(warning, #{msg => "plugin_avsc_schema_already_exists", name_vsn => Name}), + {error, already_exists}; + {error, not_found} -> + case gen_server:call(?MODULE, {build_serdes, to_bin(Name), Avsc}) of + ok -> + ?SLOG(debug, #{msg => "plugin_avsc_schema_added", name_vsn => Name}), + ok; + {error, Reason} = E -> + ?SLOG(error, #{ + msg => "plugin_avsc_schema_added_failed", + reason => emqx_utils:readable_error_msg(Reason) + }), + E + end + end. + +get_schema(NameVsn) -> + Path = emqx_plugins:schema_file(NameVsn), + case read_avsc_file(Path) of + {ok, Avsc} -> + {ok, Avsc}; + {error, Reason} -> + ?SLOG(warning, Reason), + {error, Reason} + end. + +-spec delete_schema(schema_name()) -> ok | {error, term()}. +delete_schema(NameVsn) -> + case get_serde(NameVsn) of + {ok, _Serde} -> + async_delete_serdes([NameVsn]), + ok; + {error, not_found} -> + {error, not_found} + end. + +-spec decode(schema_name(), encoded_data()) -> {ok, decoded_data()} | {error, any()}. +decode(SerdeName, RawData) -> + with_serde( + "decode_avro_binary", + eval_serde_fun(?FUNCTION_NAME, "bad_avro_binary", SerdeName, [RawData]) + ). + +-spec encode(schema_name(), decoded_data()) -> {ok, encoded_data()} | {error, any()}. +encode(SerdeName, Data) -> + with_serde( + "encode_avro_data", + eval_serde_fun(?FUNCTION_NAME, "bad_avro_data", SerdeName, [Data]) + ). + +%%------------------------------------------------------------------------------------------------- +%% `gen_server' API +%%------------------------------------------------------------------------------------------------- + +init(_) -> + process_flag(trap_exit, true), + ok = emqx_utils_ets:new(?PLUGIN_SERDE_TAB, [ + public, ordered_set, {keypos, #plugin_schema_serde.name} + ]), + State = #{}, + SchemasMap = read_plugin_avsc(), + {ok, State, {continue, {build_serdes, SchemasMap}}}. + +handle_continue({build_serdes, SchemasMap}, State) -> + _ = build_serdes(SchemasMap), + {noreply, State}. + +handle_call({build_serdes, {NameVsn, Avsc}}, _From, State) -> + BuildRes = do_build_serde(NameVsn, Avsc), + {reply, BuildRes, State}; +handle_call(_Call, _From, State) -> + {reply, {error, unknown_call}, State}. + +handle_cast({delete_serdes, Names}, State) -> + lists:foreach(fun ensure_serde_absent/1, Names), + {noreply, State}; +handle_cast(_Cast, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +read_plugin_avsc() -> + Pattern = filename:join([emqx_plugins:install_dir(), "*", "config_schema.avsc"]), + lists:foldl( + fun(AvscPath, AccIn) -> + case read_avsc_file(AvscPath) of + {ok, Avsc} -> + [_, NameVsn | _] = lists:reverse(filename:split(AvscPath)), + AccIn#{to_bin(NameVsn) => Avsc}; + {error, Reason} -> + ?SLOG(warning, Reason), + AccIn + end + end, + _Acc0 = #{}, + filelib:wildcard(Pattern) + ). + +build_serdes(Schemas) -> + maps:foreach(fun do_build_serde/2, Schemas). + +do_build_serde(NameVsn, Avsc) -> + try + Serde = make_serde(NameVsn, Avsc), + true = ets:insert(?PLUGIN_SERDE_TAB, Serde), + ok + catch + Kind:Error:Stacktrace -> + ?SLOG( + error, + #{ + msg => "error_building_plugin_schema_serde", + name => NameVsn, + kind => Kind, + error => Error, + stacktrace => Stacktrace + } + ), + {error, Error} + end. + +make_serde(NameVsn, Avsc) -> + Store0 = avro_schema_store:new([map]), + %% import the schema into the map store with an assigned name + %% if it's a named schema (e.g. struct), then Name is added as alias + Store = avro_schema_store:import_schema_json(NameVsn, Avsc, Store0), + #plugin_schema_serde{ + name = NameVsn, + eval_context = Store + }. + +ensure_serde_absent(Name) when not is_binary(Name) -> + ensure_serde_absent(to_bin(Name)); +ensure_serde_absent(Name) -> + case get_serde(Name) of + {ok, _Serde} -> + _ = ets:delete(?PLUGIN_SERDE_TAB, Name), + ok; + {error, not_found} -> + ok + end. + +async_delete_serdes(Names) -> + gen_server:cast(?MODULE, {delete_serdes, Names}). + +with_serde(WhichOp, Fun) -> + try + Fun() + catch + throw:Reason -> + ?SLOG(error, Reason#{ + which_op => WhichOp, + reason => emqx_utils:readable_error_msg(Reason) + }), + {error, Reason}; + error:Reason:Stacktrace -> + %% unexpected errors, log stacktrace + ?SLOG(warning, #{ + msg => "plugin_schema_op_failed", + which_op => WhichOp, + exception => Reason, + stacktrace => Stacktrace + }), + {error, #{ + which_op => WhichOp, + reason => Reason + }} + end. + +eval_serde_fun(Op, ErrMsg, SerdeName, Args) -> + fun() -> + case get_serde(SerdeName) of + {ok, Serde} -> + eval_serde(Op, Serde, Args); + {error, not_found} -> + throw(#{ + error_msg => ErrMsg, + reason => plugin_serde_not_found, + serde_name => SerdeName + }) + end + end. + +eval_serde(decode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> + Opts = avro:make_decoder_options([{map_type, map}, {record_type, map}]), + {ok, avro_binary_decoder:decode(Data, Name, Store, Opts)}; +eval_serde(encode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> + {ok, avro_binary_encoder:encode(Store, Name, Data)}; +eval_serde(_, _, _) -> + throw(#{error_msg => "unexpected_plugin_avro_op"}). + +read_avsc_file(Path) -> + case file:read_file(Path) of + {ok, Bin} -> + {ok, Bin}; + {error, _} -> + {error, #{ + error => "failed_to_read_plugin_schema", + path => Path + }} + end. + +to_bin(A) when is_atom(A) -> atom_to_binary(A); +to_bin(L) when is_list(L) -> iolist_to_binary(L); +to_bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_plugins/src/emqx_plugins_sup.erl b/apps/emqx_plugins/src/emqx_plugins_sup.erl index f2092fb28..9f396c14d 100644 --- a/apps/emqx_plugins/src/emqx_plugins_sup.erl +++ b/apps/emqx_plugins/src/emqx_plugins_sup.erl @@ -32,4 +32,14 @@ init([]) -> intensity => 100, period => 10 }, - {ok, {SupFlags, []}}. + ChildSpecs = [child_spec(emqx_plugins_serde)], + {ok, {SupFlags, ChildSpecs}}. + +child_spec(Mod) -> + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5_000, + type => worker + }. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 0f66e20dc..d7bdfad13 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -346,7 +346,7 @@ t_enable_disable(Config) -> ?assertEqual([#{name_vsn => NameVsn, enable => true}], emqx_plugins:configured()), ?assertMatch( {error, #{ - reason := "bad_plugin_config_status", + error_msg := "bad_plugin_config_status", hint := "disable_the_plugin_first" }}, emqx_plugins:ensure_uninstalled(NameVsn) @@ -374,15 +374,15 @@ t_bad_tar_gz(Config) -> ok = file:write_file(FakeTarTz, "a\n"), ?assertMatch( {error, #{ - reason := "bad_plugin_package", - return := eof + error_msg := "bad_plugin_package", + reason := eof }}, emqx_plugins:ensure_installed("fake-vsn") ), ?assertMatch( {error, #{ - reason := "failed_to_extract_plugin_package", - return := not_found + error_msg := "failed_to_extract_plugin_package", + reason := not_found }}, emqx_plugins:ensure_installed("nonexisting") ), @@ -412,7 +412,7 @@ t_bad_tar_gz2(Config) -> ?assert(filelib:is_regular(TarGz)), %% failed to install, it also cleans up the bad content of .tar.gz file ?assertMatch({error, _}, emqx_plugins:ensure_installed(NameVsn)), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir(NameVsn))), %% but the tar.gz file is still around ?assert(filelib:is_regular(TarGz)), ok. @@ -440,8 +440,8 @@ t_tar_vsn_content_mismatch(Config) -> %% failed to install, it also cleans up content of the bad .tar.gz file even %% if in other directory ?assertMatch({error, _}, emqx_plugins:ensure_installed(NameVsn)), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir("foo-0.2"))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir(NameVsn))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir("foo-0.2"))), %% the tar.gz file is still around ?assert(filelib:is_regular(TarGz)), ok. @@ -455,15 +455,15 @@ t_bad_info_json(Config) -> ok = write_info_file(Config, NameVsn, "bad-syntax"), ?assertMatch( {error, #{ - error := "bad_info_file", - return := {parse_error, _} + error_msg := "bad_info_file", + reason := {parse_error, _} }}, emqx_plugins:describe(NameVsn) ), ok = write_info_file(Config, NameVsn, "{\"bad\": \"obj\"}"), ?assertMatch( {error, #{ - error := "bad_info_file_content", + error_msg := "bad_info_file_content", mandatory_fields := _ }}, emqx_plugins:describe(NameVsn) @@ -499,7 +499,7 @@ t_elixir_plugin(Config) -> ok = emqx_plugins:ensure_installed(NameVsn), %% idempotent ok = emqx_plugins:ensure_installed(NameVsn), - {ok, Info} = emqx_plugins:read_plugin(NameVsn, #{}), + {ok, Info} = emqx_plugins:read_plugin_info(NameVsn, #{}), ?assertEqual([Info], emqx_plugins:list()), %% start ok = emqx_plugins:ensure_started(NameVsn), diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index d064cba9d..010d96de9 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -57,7 +57,7 @@ read_plugin_test() -> ok = write_file(InfoFile, FakeInfo), ?assertMatch( {error, #{error := "bad_rel_apps"}}, - emqx_plugins:read_plugin(NameVsn, #{}) + emqx_plugins:read_plugin_info(NameVsn, #{}) ) after emqx_plugins:purge(NameVsn) @@ -109,7 +109,7 @@ purge_test() -> with_rand_install_dir( fun(_Dir) -> File = emqx_plugins:info_file("a-1"), - Dir = emqx_plugins:dir("a-1"), + Dir = emqx_plugins:plugin_dir("a-1"), ok = filelib:ensure_dir(File), ?assertMatch({ok, _}, file:read_file_info(Dir)), ?assertEqual(ok, emqx_plugins:purge("a-1")), From d06f410fd52dbc792207541a4eeb6bc9fef81a13 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 15:24:38 +0800 Subject: [PATCH 37/86] fix(plugins): read avsc file --- apps/emqx_plugins/src/emqx_plugins.erl | 56 +++++++++++++------ apps/emqx_plugins/src/emqx_plugins_serde.erl | 4 +- apps/emqx_plugins/test/emqx_plugins_tests.erl | 6 +- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index ce66fc94d..6d5a4bcbc 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -81,7 +81,9 @@ %% Defines -define(PLUGIN_PERSIS_CONFIG_KEY(NameVsn), {?MODULE, NameVsn}). -%% Types +-define(RAW_BIN, binary). +-define(JSON_MAP, json_map). + %% "my_plugin-0.1.0" -type name_vsn() :: binary() | string(). %% the parse result of the JSON info file @@ -590,30 +592,36 @@ do_read_plugin(NameVsn) -> do_read_plugin2(NameVsn, #{}). do_read_plugin2(NameVsn, Option) -> - do_read_plugin3(NameVsn, info_file(NameVsn), Option). + do_read_plugin3(NameVsn, info_file_path(NameVsn), Option). do_read_plugin3(NameVsn, InfoFilePath, Options) -> - {ok, PlainMap} = (read_file_fun(InfoFilePath, "bad_info_file"))(), + {ok, PlainMap} = (read_file_fun(InfoFilePath, "bad_info_file", #{read_mode => ?JSON_MAP}))(), Info0 = check_plugin(PlainMap, NameVsn, InfoFilePath), Info1 = plugins_readme(NameVsn, Options, Info0), plugin_status(NameVsn, Info1). read_plugin_avsc(NameVsn) -> + read_plugin_avsc(NameVsn, #{read_mode => ?JSON_MAP}). +read_plugin_avsc(NameVsn, Options) -> tryit( atom_to_list(?FUNCTION_NAME), - read_file_fun(schema_file(NameVsn), "bad_avsc_file") + read_file_fun(avsc_file_path(NameVsn), "bad_avsc_file", Options) ). read_plugin_i18n(NameVsn) -> + read_plugin_i18n(NameVsn, #{read_mode => ?JSON_MAP}). +read_plugin_i18n(NameVsn, Options) -> tryit( atom_to_list(?FUNCTION_NAME), - read_file_fun(i18n_file(NameVsn), "bad_i18n_file") + read_file_fun(i18n_file_path(NameVsn), "bad_i18n_file", Options) ). read_plugin_avro(NameVsn) -> + read_plugin_avro(NameVsn, #{read_mode => ?RAW_BIN}). +read_plugin_avro(NameVsn, Options) -> tryit( atom_to_list(?FUNCTION_NAME), - read_file_fun(schema_file(NameVsn), "bad_avro_file") + read_file_fun(avro_config_file(NameVsn), "bad_avro_file", Options) ). ensure_exists_and_installed(NameVsn) -> @@ -1000,15 +1008,22 @@ maybe_post_op_after_install(NameVsn) -> ok. maybe_load_config_schema(NameVsn) -> - case read_plugin_avsc(NameVsn) of - {ok, Avsc} -> - case emqx_plugins_serde:add_schema(NameVsn, Avsc) of + filelib:is_regular(avsc_file_path(NameVsn)) andalso + do_load_config_schema(NameVsn). + +do_load_config_schema(NameVsn) -> + case read_plugin_avsc(NameVsn, #{read_mode => ?RAW_BIN}) of + {ok, AvscBin} -> + case emqx_plugins_serde:add_schema(NameVsn, AvscBin) of ok -> ok; {error, already_exists} -> ok; - {error, Reason} -> {error, Reason} + {error, _Reason} -> ok end; {error, Reason} -> - ?SLOG(warning, Reason) + ?SLOG(warning, #{ + msg => "failed_to_read_plugin_avsc", reason => Reason, name_vsn => NameVsn + }), + ok end. maybe_create_config_dir(NameVsn) -> @@ -1020,14 +1035,23 @@ maybe_create_config_dir(NameVsn) -> write_avro_bin(NameVsn, AvroBin) -> ok = file:write_file(avro_config_file(NameVsn), AvroBin). -read_file_fun(Path, ErrMsg) -> +read_file_fun(Path, ErrMsg, #{read_mode := ?RAW_BIN}) -> + fun() -> + case file:read_file(Path) of + {ok, Bin} -> + {ok, Bin}; + {error, Reason} -> + ErrMeta = #{error_msg => ErrMsg, reason => Reason}, + throw(ErrMeta) + end + end; +read_file_fun(Path, ErrMsg, #{read_mode := ?JSON_MAP}) -> fun() -> case hocon:load(Path, #{format => richmap}) of {ok, RichMap} -> {ok, hocon_maps:ensure_plain(RichMap)}; {error, Reason} -> ErrMeta = #{error_msg => ErrMsg, reason => Reason}, - ?SLOG(warning, ErrMeta), throw(ErrMeta) end end. @@ -1043,16 +1067,16 @@ plugin_config_dir(NameVsn) -> pkg_file(NameVsn) -> filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). -info_file(NameVsn) -> +info_file_path(NameVsn) -> filename:join([plugin_dir(NameVsn), "release.json"]). -schema_file(NameVsn) -> +avsc_file_path(NameVsn) -> filename:join([plugin_dir(NameVsn), "config_schema.avsc"]). avro_config_file(NameVsn) -> filename:join([plugin_config_dir(NameVsn), "config.avro"]). -i18n_file(NameVsn) -> +i18n_file_path(NameVsn) -> filename:join([plugin_dir(NameVsn), "i18n.json"]). readme_file(NameVsn) -> diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index a89f16e70..083f9740d 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -79,7 +79,7 @@ add_schema(Name, Avsc) -> end. get_schema(NameVsn) -> - Path = emqx_plugins:schema_file(NameVsn), + Path = emqx_plugins:avsc_file_path(NameVsn), case read_avsc_file(Path) of {ok, Avsc} -> {ok, Avsc}; @@ -129,7 +129,7 @@ handle_continue({build_serdes, SchemasMap}, State) -> _ = build_serdes(SchemasMap), {noreply, State}. -handle_call({build_serdes, {NameVsn, Avsc}}, _From, State) -> +handle_call({build_serdes, NameVsn, Avsc}, _From, State) -> BuildRes = do_build_serde(NameVsn, Avsc), {reply, BuildRes, State}; handle_call(_Call, _From, State) -> diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index 010d96de9..910c30564 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -49,7 +49,7 @@ read_plugin_test() -> with_rand_install_dir( fun(_Dir) -> NameVsn = "bar-5", - InfoFile = emqx_plugins:info_file(NameVsn), + InfoFile = emqx_plugins:info_file_path(NameVsn), FakeInfo = "name=bar, rel_vsn=\"5\", rel_apps=[justname_no_vsn]," "description=\"desc bar\"", @@ -90,7 +90,7 @@ delete_package_test() -> meck_emqx(), with_rand_install_dir( fun(_Dir) -> - File = emqx_plugins:pkg_file("a-1"), + File = emqx_plugins:pkg_file_path("a-1"), ok = write_file(File, "a"), ok = emqx_plugins:delete_package("a-1"), %% delete again should be ok @@ -108,7 +108,7 @@ purge_test() -> meck_emqx(), with_rand_install_dir( fun(_Dir) -> - File = emqx_plugins:info_file("a-1"), + File = emqx_plugins:info_file_path("a-1"), Dir = emqx_plugins:plugin_dir("a-1"), ok = filelib:ensure_dir(File), ?assertMatch({ok, _}, file:read_file_info(Dir)), From d2e0c09f2e0d1117350e8d1cad41907106c08770 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 16:44:14 +0800 Subject: [PATCH 38/86] fix: make static check happy --- .../src/emqx_mgmt_api_plugins.erl | 6 +++--- .../proto/emqx_mgmt_api_plugins_proto_v2.erl | 1 - apps/emqx_plugins/rebar.config | 5 ++++- apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins.erl | 17 +++++++++++++---- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 63a8ba517..ac79c6d84 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -519,7 +519,7 @@ plugin_config(put, #{bindings := #{name := Name}, body := #{<<"config">> := RawA plugin_schema(get, #{bindings := #{name := NameVsn}}) -> case emqx_plugins:describe(NameVsn) of {ok, _Plugin} -> - {200, format_plugin_schema_with_i18n(NameVsn)}; + {200, format_plugin_avsc_and_i18n(NameVsn)}; _ -> {404, #{ code => 'NOT_FOUND', @@ -685,7 +685,7 @@ aggregate_status([{Node, Plugins} | List], Acc) -> ), aggregate_status(List, NewAcc). -format_plugin_schema_with_i18n(NameVsn) -> +format_plugin_avsc_and_i18n(NameVsn) -> #{ avsc => try_read_file(fun() -> emqx_plugins:plugin_avsc(NameVsn) end), i18n => try_read_file(fun() -> emqx_plugins:plugin_i18n(NameVsn) end) @@ -693,7 +693,7 @@ format_plugin_schema_with_i18n(NameVsn) -> try_read_file(Fun) -> case Fun() of - {ok, Bin} -> Bin; + {ok, Json} -> Json; _ -> null end. diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl index 91d4674f9..781096af0 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v2.erl @@ -24,7 +24,6 @@ describe_package/2, delete_package/1, ensure_action/2 - %% plugin_config/2 ]). -include_lib("emqx/include/bpapi.hrl"). diff --git a/apps/emqx_plugins/rebar.config b/apps/emqx_plugins/rebar.config index 9f17b7657..63d56ac72 100644 --- a/apps/emqx_plugins/rebar.config +++ b/apps/emqx_plugins/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {erlavro, {git, "https://github.com/emqx/erlavro.git", {tag, "2.10.0"}}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index 0e0945c7f..6501d5654 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -4,6 +4,6 @@ {vsn, "0.2.0"}, {modules, []}, {mod, {emqx_plugins_app, []}}, - {applications, [kernel, stdlib, emqx]}, + {applications, [kernel, stdlib, emqx, erlavro]}, {env, []} ]}. diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 6d5a4bcbc..7465b5ff1 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -62,7 +62,8 @@ get_config/2, put_config/2, get_tar/1, - install_dir/0 + install_dir/0, + avsc_file_path/1 ]). %% `emqx_config_handler' API @@ -1027,9 +1028,17 @@ do_load_config_schema(NameVsn) -> end. maybe_create_config_dir(NameVsn) -> - case filelib:ensure_path(plugin_config_dir(NameVsn)) of - ok -> ok; - {error, Reason} -> ?SLOG(warning, Reason) + ConfigDir = plugin_config_dir(NameVsn), + case filelib:ensure_path(ConfigDir) of + ok -> + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "failed_to_create_plugin_config_dir", + dir => ConfigDir, + reason => Reason + }), + {error, {mkdir_failed, ConfigDir, Reason}} end. write_avro_bin(NameVsn, AvroBin) -> From 2686a66b1455297a4f6797ba73b8b844d5b0c676 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 17:33:08 +0800 Subject: [PATCH 39/86] fix: make eunit happy --- apps/emqx_plugins/src/emqx_plugins.erl | 12 ++++++------ apps/emqx_plugins/test/emqx_plugins_tests.erl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 7465b5ff1..722a2152c 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -193,7 +193,7 @@ purge(NameVsn) -> %% @doc Delete the package file. -spec delete_package(name_vsn()) -> ok. delete_package(NameVsn) -> - File = pkg_file(NameVsn), + File = pkg_file_path(NameVsn), _ = emqx_plugins_serde:delete_schema(NameVsn), case file:delete(File) of ok -> @@ -317,7 +317,7 @@ put_config(Key, Value) -> -spec get_tar(name_vsn()) -> {ok, binary()} | {error, any}. get_tar(NameVsn) -> - TarGz = pkg_file(NameVsn), + TarGz = pkg_file_path(NameVsn), case file:read_file(TarGz) of {ok, Content} -> {ok, Content}; @@ -407,7 +407,7 @@ top_dir_test_() -> -endif. do_ensure_installed(NameVsn) -> - TarGz = pkg_file(NameVsn), + TarGz = pkg_file_path(NameVsn), case erl_tar:extract(TarGz, [compressed, memory]) of {ok, TarContent} -> ok = write_tar_file_content(install_dir(), TarContent), @@ -633,7 +633,7 @@ ensure_exists_and_installed(NameVsn) -> %% Do we have the package, but it's not extracted yet? case get_tar(NameVsn) of {ok, TarContent} -> - ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = file:write_file(pkg_file_path(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); _ -> %% If not, try to get it from the cluster. @@ -645,7 +645,7 @@ do_get_from_cluster(NameVsn) -> Nodes = [N || N <- mria:running_nodes(), N /= node()], case get_from_any_node(Nodes, NameVsn, []) of {ok, TarContent} -> - ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = file:write_file(pkg_file_path(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); {error, NodeErrors} when Nodes =/= [] -> ?SLOG(error, #{ @@ -1073,7 +1073,7 @@ plugin_config_dir(NameVsn) -> filename:join([plugin_dir(NameVsn), "data", "configs"]). %% Files -pkg_file(NameVsn) -> +pkg_file_path(NameVsn) -> filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). info_file_path(NameVsn) -> diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index 910c30564..fb4277c4f 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -56,7 +56,7 @@ read_plugin_test() -> try ok = write_file(InfoFile, FakeInfo), ?assertMatch( - {error, #{error := "bad_rel_apps"}}, + {error, #{error_msg := "bad_rel_apps"}}, emqx_plugins:read_plugin_info(NameVsn, #{}) ) after From 27d1f91cac762aa0482158bbc027d08228f02c58 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 17:36:24 +0800 Subject: [PATCH 40/86] refactor: refine function name --- apps/emqx_plugins/src/emqx_plugins_serde.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index 083f9740d..da8515390 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -22,7 +22,7 @@ %% API -export([ start_link/0, - get_serde/1, + lookup_serde/1, add_schema/2, get_schema/1, delete_schema/1 @@ -49,8 +49,8 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec get_serde(schema_name()) -> {ok, plugin_schema_serde()} | {error, not_found}. -get_serde(SchemaName) -> +-spec lookup_serde(schema_name()) -> {ok, plugin_schema_serde()} | {error, not_found}. +lookup_serde(SchemaName) -> case ets:lookup(?PLUGIN_SERDE_TAB, to_bin(SchemaName)) of [] -> {error, not_found}; @@ -60,7 +60,7 @@ get_serde(SchemaName) -> -spec add_schema(schema_name(), avsc()) -> ok | {error, term()}. add_schema(Name, Avsc) -> - case get_serde(Name) of + case lookup_serde(Name) of {ok, _Serde} -> ?SLOG(warning, #{msg => "plugin_avsc_schema_already_exists", name_vsn => Name}), {error, already_exists}; @@ -90,7 +90,7 @@ get_schema(NameVsn) -> -spec delete_schema(schema_name()) -> ok | {error, term()}. delete_schema(NameVsn) -> - case get_serde(NameVsn) of + case lookup_serde(NameVsn) of {ok, _Serde} -> async_delete_serdes([NameVsn]), ok; @@ -201,7 +201,7 @@ make_serde(NameVsn, Avsc) -> ensure_serde_absent(Name) when not is_binary(Name) -> ensure_serde_absent(to_bin(Name)); ensure_serde_absent(Name) -> - case get_serde(Name) of + case lookup_serde(Name) of {ok, _Serde} -> _ = ets:delete(?PLUGIN_SERDE_TAB, Name), ok; @@ -238,7 +238,7 @@ with_serde(WhichOp, Fun) -> eval_serde_fun(Op, ErrMsg, SerdeName, Args) -> fun() -> - case get_serde(SerdeName) of + case lookup_serde(SerdeName) of {ok, Serde} -> eval_serde(Op, Serde, Args); {error, not_found} -> From c0429ca333ddd7626dd3d00aa0774bad37ca7f09 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 17:38:35 +0800 Subject: [PATCH 41/86] fix(plugin): refine schema serde log --- apps/emqx_plugins/src/emqx_plugins_serde.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index da8515390..5bc5aaa0c 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -62,16 +62,17 @@ lookup_serde(SchemaName) -> add_schema(Name, Avsc) -> case lookup_serde(Name) of {ok, _Serde} -> - ?SLOG(warning, #{msg => "plugin_avsc_schema_already_exists", name_vsn => Name}), + ?SLOG(warning, #{msg => "plugin_schema_already_exists", plugin => Name}), {error, already_exists}; {error, not_found} -> case gen_server:call(?MODULE, {build_serdes, to_bin(Name), Avsc}) of ok -> - ?SLOG(debug, #{msg => "plugin_avsc_schema_added", name_vsn => Name}), + ?SLOG(debug, #{msg => "plugin_schema_added", plugin => Name}), ok; {error, Reason} = E -> ?SLOG(error, #{ - msg => "plugin_avsc_schema_added_failed", + msg => "plugin_schema_add_failed", + plugin => Name, reason => emqx_utils:readable_error_msg(Reason) }), E From 1f00ce789fbb4947069d3471a5be77ac74fdd862 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 17:45:36 +0800 Subject: [PATCH 42/86] fix(plugin): gen_server call timeout infinity --- apps/emqx_plugins/src/emqx_plugins_serde.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index 5bc5aaa0c..726d3c04b 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -65,7 +65,7 @@ add_schema(Name, Avsc) -> ?SLOG(warning, #{msg => "plugin_schema_already_exists", plugin => Name}), {error, already_exists}; {error, not_found} -> - case gen_server:call(?MODULE, {build_serdes, to_bin(Name), Avsc}) of + case gen_server:call(?MODULE, {build_serdes, to_bin(Name), Avsc}, infinity) of ok -> ?SLOG(debug, #{msg => "plugin_schema_added", plugin => Name}), ok; From c180b6a417ce8397c154dfcd181ce8d01a6cf02a Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Apr 2024 17:48:26 +0800 Subject: [PATCH 43/86] fix(api): plugin api docs --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index ac79c6d84..e065e0301 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -170,10 +170,9 @@ schema("/plugins/:name/config") -> #{ 'operationId' => plugin_config, get => #{ - summary => - <<"Get plugin config">>, + summary => <<"Get plugin config">>, description => - "Get plugin config by avro encoded binary config. Schema defined by user's schema.avsc file.
", + "Get plugin config. Config schema is defined by user's schema.avsc file.
", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -186,7 +185,7 @@ schema("/plugins/:name/config") -> summary => <<"Update plugin config">>, description => - "Update plugin config by avro encoded binary config. Schema defined by user's schema.avsc file.
", + "Update plugin config. Config schema defined by user's schema.avsc file.
", tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => #{ @@ -215,9 +214,8 @@ schema("/plugins/:name/schema") -> #{ 'operationId' => plugin_schema, get => #{ - summary => <<"Get installed plugin's avro schema">>, - description => - "Get plugin's config avro schema.", + summary => <<"Get installed plugin's AVRO schema">>, + description => "Get plugin's config AVRO schema.", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ From d2ecccc2ff06011befacfcfa9cac865b54e863b9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 13:40:55 +0800 Subject: [PATCH 44/86] fix: call json encoder/decoder for plugin config --- .../src/emqx_mgmt_api_plugins.erl | 27 +++++++++---------- .../proto/emqx_mgmt_api_plugins_proto_v3.erl | 8 ++++-- apps/emqx_plugins/src/emqx_plugins.erl | 19 +++++++------ apps/emqx_plugins/src/emqx_plugins_serde.erl | 10 +++---- 4 files changed, 35 insertions(+), 29 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index e065e0301..c98f1053e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -176,7 +176,7 @@ schema("/plugins/:name/config") -> tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ - %% binary avro encoded config + %% avro data, json encoded 200 => hoconsc:mk(binary()), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } @@ -190,14 +190,10 @@ schema("/plugins/:name/config") -> parameters => [hoconsc:ref(name)], 'requestBody' => #{ content => #{ - 'multipart/form-data' => #{ + 'application/json' => #{ schema => #{ - type => object, - properties => #{ - ?CONTENT_CONFIG => #{type => string, format => binary} - } - }, - encoding => #{?CONTENT_CONFIG => #{'contentType' => 'application/gzip'}} + type => object + } } } }, @@ -499,12 +495,14 @@ plugin_config(get, #{bindings := #{name := Name}}) -> message => <<"Failed to get plugin config">> }} end; -plugin_config(put, #{bindings := #{name := Name}, body := #{<<"config">> := RawAvro}}) -> - case emqx_plugins:decode_plugin_avro_config(Name, RawAvro) of - {ok, Config} -> +plugin_config(put, #{bindings := #{name := Name}, body := AvroJsonMap}) -> + AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), + case emqx_plugins:decode_plugin_avro_config(Name, AvroJsonBin) of + {ok, AvroValueConfig} -> Nodes = emqx:running_nodes(), + %% cluster call with config in map (binary key-value) _Res = emqx_mgmt_api_plugins_proto_v3:update_plugin_config( - Nodes, Name, RawAvro, Config + Nodes, Name, AvroJsonMap, AvroValueConfig ), {204}; {error, Reason} -> @@ -595,8 +593,9 @@ ensure_action(Name, restart) -> ok. %% for RPC plugin avro encoded config update -do_update_plugin_config(Name, Avro, PluginConfig) -> - emqx_plugins:put_plugin_config(Name, Avro, PluginConfig). +do_update_plugin_config(Name, AvroJsonMap, PluginConfigMap) -> + %% TOOD: maybe use `PluginConfigMap` to validate config + emqx_plugins:put_plugin_config(Name, AvroJsonMap, PluginConfigMap). %%-------------------------------------------------------------------- %% Helper functions diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl index 13c13bae7..b428a38cc 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl @@ -59,7 +59,11 @@ ensure_action(Name, Action) -> map() ) -> emqx_rpc:multicall_result(). -update_plugin_config(Nodes, Name, RawAvro, PluginConfig) -> +update_plugin_config(Nodes, Name, AvroJsonMap, PluginConfig) -> rpc:multicall( - Nodes, emqx_mgmt_api_plugins, do_update_plugin_config, [Name, RawAvro, PluginConfig], 10000 + Nodes, + emqx_mgmt_api_plugins, + do_update_plugin_config, + [Name, AvroJsonMap, PluginConfig], + 10000 ). diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 722a2152c..0d0dc1fae 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -51,7 +51,7 @@ ensure_stopped/1, get_plugin_config/1, get_plugin_config/2, - put_plugin_config/3, + put_plugin_config/4, restart/1, list/0 ]). @@ -256,7 +256,7 @@ get_plugin_config(NameVsn) -> | {error, term()}. get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> case read_plugin_avro(NameVsn) of - {ok, _AvroBin} = Res -> Res; + {ok, _AvroJson} = Res -> Res; {error, _Reason} = Err -> Err end; get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}) -> @@ -265,9 +265,10 @@ get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}) -> %% @doc Update plugin's config. %% RPC call from Management API or CLI. %% the avro binary and plugin config ALWAYS be valid before calling this function. -put_plugin_config(NameVsn, RawAvro, PluginConfig) -> - ok = write_avro_bin(NameVsn, RawAvro), - ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), PluginConfig), +put_plugin_config(NameVsn, AvroJsonMap, DecodedPluginConfig) -> + AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), + ok = write_avro_bin(NameVsn, AvroJsonBin), + ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), AvroJsonMap), ok. %% @doc Stop and then start the plugin. @@ -300,9 +301,11 @@ list() -> %%-------------------------------------------------------------------- %% Package utils --spec decode_plugin_avro_config(name_vsn(), binary()) -> {ok, map()} | {error, any()}. -decode_plugin_avro_config(NameVsn, RawAvro) -> - case emqx_plugins_serde:decode(NameVsn, RawAvro) of +-spec decode_plugin_avro_config(name_vsn(), map() | binary()) -> {ok, map()} | {error, any()}. +decode_plugin_avro_config(NameVsn, AvroJsonMap) when is_map(AvroJsonMap) -> + decode_plugin_avro_config(NameVsn, emqx_utils_json:encode(AvroJsonMap)); +decode_plugin_avro_config(NameVsn, AvroJsonBin) -> + case emqx_plugins_serde:decode(NameVsn, AvroJsonBin) of {ok, Config} -> {ok, Config}; {error, ReasonMap} -> {error, ReasonMap} end. diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index 726d3c04b..b936020a6 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -102,14 +102,14 @@ delete_schema(NameVsn) -> -spec decode(schema_name(), encoded_data()) -> {ok, decoded_data()} | {error, any()}. decode(SerdeName, RawData) -> with_serde( - "decode_avro_binary", + "decode_avro_json", eval_serde_fun(?FUNCTION_NAME, "bad_avro_binary", SerdeName, [RawData]) ). -spec encode(schema_name(), decoded_data()) -> {ok, encoded_data()} | {error, any()}. encode(SerdeName, Data) -> with_serde( - "encode_avro_data", + "encode_avro_json", eval_serde_fun(?FUNCTION_NAME, "bad_avro_data", SerdeName, [Data]) ). @@ -252,10 +252,10 @@ eval_serde_fun(Op, ErrMsg, SerdeName, Args) -> end. eval_serde(decode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> - Opts = avro:make_decoder_options([{map_type, map}, {record_type, map}]), - {ok, avro_binary_decoder:decode(Data, Name, Store, Opts)}; + Opts = avro:make_decoder_options([{map_type, map}, {record_type, map}, {encoding, avro_json}]), + {ok, avro_json_decoder:decode_value(Data, Name, Store, Opts)}; eval_serde(encode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> - {ok, avro_binary_encoder:encode(Store, Name, Data)}; + {ok, avro_json_encoder:encode(Store, Name, Data)}; eval_serde(_, _, _) -> throw(#{error_msg => "unexpected_plugin_avro_op"}). From f343cd2021bd8e39b3f07f1dda06daea88e6f288 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 13:43:00 +0800 Subject: [PATCH 45/86] fix: get plugin config in json --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index c98f1053e..0a575befd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -486,9 +486,9 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> return(204, Res). plugin_config(get, #{bindings := #{name := Name}}) -> - case emqx_plugins:get_plugin_config(Name, #{format => ?CONFIG_FORMAT_AVRO}) of - {ok, AvroBin} -> - {200, #{<<"content-type">> => <<"application/octet-stream">>}, AvroBin}; + case emqx_plugins:get_plugin_config(Name, #{format => ?CONFIG_FORMAT_MAP}) of + {ok, AvroJson} -> + {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; {error, _} -> {400, #{ code => 'BAD_CONFIG', From b0aa3bb70fe99f9bdd10f13a7a43f39f17aa2ab5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 13:49:54 +0800 Subject: [PATCH 46/86] fix(plugin): get plugin config api --- apps/emqx_plugins/src/emqx_plugins.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 0d0dc1fae..7a73cc72b 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -51,7 +51,7 @@ ensure_stopped/1, get_plugin_config/1, get_plugin_config/2, - put_plugin_config/4, + put_plugin_config/3, restart/1, list/0 ]). @@ -249,7 +249,7 @@ ensure_stopped(NameVsn) -> -spec get_plugin_config(name_vsn()) -> {ok, plugin_config()} | {error, term()}. get_plugin_config(NameVsn) -> - get_plugin_config(NameVsn, #{format => ?CONFIG_FORMAT_MAP}). + get_plugin_config(bin(NameVsn), #{format => ?CONFIG_FORMAT_MAP}). -spec get_plugin_config(name_vsn(), Options :: map()) -> {ok, avro_binary() | plugin_config()} @@ -260,12 +260,12 @@ get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> {error, _Reason} = Err -> Err end; get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}) -> - persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), #{}). + {ok, persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), #{})}. %% @doc Update plugin's config. %% RPC call from Management API or CLI. %% the avro binary and plugin config ALWAYS be valid before calling this function. -put_plugin_config(NameVsn, AvroJsonMap, DecodedPluginConfig) -> +put_plugin_config(NameVsn, AvroJsonMap, _DecodedPluginConfig) -> AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), ok = write_avro_bin(NameVsn, AvroJsonBin), ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), AvroJsonMap), From 1869f6fd0a7213ec4a30634da587705f3dd84373 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 14:32:32 +0800 Subject: [PATCH 47/86] fix: enusre plugin installed when do config operation --- .../src/emqx_mgmt_api_plugins.erl | 71 +++++++++++-------- .../proto/emqx_mgmt_api_plugins_proto_v3.erl | 4 +- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 0a575befd..d308cb42c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -200,7 +200,7 @@ schema("/plugins/:name/config") -> responses => #{ 204 => <<"Config updated successfully">>, 400 => emqx_dashboard_swagger:error_codes( - ['UNEXPECTED_ERROR'], <<"Update plugin config failed">> + ['BAD_CONFIG', 'UNEXPECTED_ERROR'], <<"Update plugin config failed">> ), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } @@ -485,31 +485,41 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> Res = emqx_mgmt_api_plugins_proto_v2:ensure_action(Name, Action), return(204, Res). -plugin_config(get, #{bindings := #{name := Name}}) -> - case emqx_plugins:get_plugin_config(Name, #{format => ?CONFIG_FORMAT_MAP}) of - {ok, AvroJson} -> - {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; - {error, _} -> - {400, #{ - code => 'BAD_CONFIG', - message => <<"Failed to get plugin config">> - }} +plugin_config(get, #{bindings := #{name := NameVsn}}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _} -> + case emqx_plugins:get_plugin_config(NameVsn, #{format => ?CONFIG_FORMAT_MAP}) of + {ok, AvroJson} -> + {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; + {error, _} -> + {400, #{ + code => 'BAD_CONFIG', + message => <<"Failed to get plugin config">> + }} + end; + _ -> + {404, plugin_not_found_msg()} end; -plugin_config(put, #{bindings := #{name := Name}, body := AvroJsonMap}) -> - AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), - case emqx_plugins:decode_plugin_avro_config(Name, AvroJsonBin) of - {ok, AvroValueConfig} -> - Nodes = emqx:running_nodes(), - %% cluster call with config in map (binary key-value) - _Res = emqx_mgmt_api_plugins_proto_v3:update_plugin_config( - Nodes, Name, AvroJsonMap, AvroValueConfig - ), - {204}; - {error, Reason} -> - {400, #{ - code => 'BAD_CONFIG', - message => readable_error_msg(Reason) - }} +plugin_config(put, #{bindings := #{name := NameVsn}, body := AvroJsonMap}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _} -> + AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), + case emqx_plugins:decode_plugin_avro_config(NameVsn, AvroJsonBin) of + {ok, AvroValueConfig} -> + Nodes = emqx:running_nodes(), + %% cluster call with config in map (binary key-value) + _Res = emqx_mgmt_api_plugins_proto_v3:update_plugin_config( + Nodes, NameVsn, AvroJsonMap, AvroValueConfig + ), + {204}; + {error, Reason} -> + {400, #{ + code => 'BAD_CONFIG', + message => readable_error_msg(Reason) + }} + end; + _ -> + {404, plugin_not_found_msg()} end. plugin_schema(get, #{bindings := #{name := NameVsn}}) -> @@ -517,10 +527,7 @@ plugin_schema(get, #{bindings := #{name := NameVsn}}) -> {ok, _Plugin} -> {200, format_plugin_avsc_and_i18n(NameVsn)}; _ -> - {404, #{ - code => 'NOT_FOUND', - message => <<"Plugin Not Found">> - }} + {404, plugin_not_found_msg()} end. update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> @@ -610,6 +617,12 @@ return(_, {error, #{error_msg := "bad_avro_config_file", reason := {enoent, _} = return(_, {error, Reason}) -> {400, #{code => 'PARAM_ERROR', message => readable_error_msg(Reason)}}. +plugin_not_found_msg() -> + #{ + code => 'NOT_FOUND', + message => <<"Plugin Not Found">> + }. + readable_error_msg(Msg) -> emqx_utils:readable_error_msg(Msg). diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl index b428a38cc..641d35f70 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl @@ -59,11 +59,11 @@ ensure_action(Name, Action) -> map() ) -> emqx_rpc:multicall_result(). -update_plugin_config(Nodes, Name, AvroJsonMap, PluginConfig) -> +update_plugin_config(Nodes, NameVsn, AvroJsonMap, PluginConfig) -> rpc:multicall( Nodes, emqx_mgmt_api_plugins, do_update_plugin_config, - [Name, AvroJsonMap, PluginConfig], + [NameVsn, AvroJsonMap, PluginConfig], 10000 ). From c884dfb4515d3915e177b5b9e0f958159c002c35 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 16:42:38 +0800 Subject: [PATCH 48/86] test: make eunit happy --- apps/emqx_plugins/test/emqx_plugins_tests.erl | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index fb4277c4f..4911c174e 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -16,6 +16,7 @@ -module(emqx_plugins_tests). +-include("emqx_plugins.hrl"). -include_lib("eunit/include/eunit.hrl"). -compile(nowarn_export_all). @@ -28,20 +29,20 @@ ensure_configured_test_todo() -> after emqx_plugins:put_configured([]) end, - meck:unload(emqx). + unmeck_emqx(). test_ensure_configured() -> ok = emqx_plugins:put_configured([]), P1 = #{name_vsn => "p-1", enable => true}, P2 = #{name_vsn => "p-2", enable => true}, P3 = #{name_vsn => "p-3", enable => false}, - emqx_plugins:ensure_configured(P1, front), - emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}), - emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}), + emqx_plugins:ensure_configured(P1, front, local), + emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}, local), + emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}, local), ?assertEqual([P2, P3, P1], emqx_plugins:configured()), ?assertThrow( #{error := "position_anchor_plugin_not_configured"}, - emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>}) + emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>}, local) ). read_plugin_test() -> @@ -64,7 +65,7 @@ read_plugin_test() -> end end ), - meck:unload(emqx). + unmeck_emqx(). with_rand_install_dir(F) -> N = rand:uniform(10000000), @@ -100,7 +101,7 @@ delete_package_test() -> ?assertMatch({error, _}, emqx_plugins:delete_package("a-1")) end ), - meck:unload(emqx). + unmeck_emqx(). %% purge plugin's install dir should mostly work and return ok %% but it may fail in case the dir is read-only @@ -120,10 +121,11 @@ purge_test() -> ?assertEqual(ok, emqx_plugins:purge("a-1")) end ), - meck:unload(emqx). + unmeck_emqx(). meck_emqx() -> meck:new(emqx, [unstick, passthrough]), + meck:new(emqx_plugins_serde), meck:expect( emqx, update_config, @@ -131,4 +133,14 @@ meck_emqx() -> emqx_config:put(Path, Values) end ), + meck:expect( + emqx_plugins_serde, + delete_schema, + fun(_NameVsn) -> ok end + ), + ok. + +unmeck_emqx() -> + meck:unload(emqx), + meck:unload(emqx_plugins_serde), ok. From 670ddae57c9fed72b12c92be4eb7327146d2bdcb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 16:46:14 +0800 Subject: [PATCH 49/86] chore: fix typo --- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl index 33c4059df..eaa550d64 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl @@ -72,7 +72,7 @@ dashboard_addr(desc) -> ?DESC(dashboard_addr); dashboard_addr(default) -> <<"https://127.0.0.1:18083">>; dashboard_addr(_) -> undefined. -%% TOOD: support raw xml metadata in hocon (maybe?🤔) +%% TODO: support raw xml metadata in hocon (maybe?🤔) idp_metadata_url(type) -> binary(); idp_metadata_url(desc) -> ?DESC(idp_metadata_url); idp_metadata_url(default) -> <<"https://idp.example.com">>; diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index d308cb42c..9cc518dd0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -601,7 +601,7 @@ ensure_action(Name, restart) -> %% for RPC plugin avro encoded config update do_update_plugin_config(Name, AvroJsonMap, PluginConfigMap) -> - %% TOOD: maybe use `PluginConfigMap` to validate config + %% TODO: maybe use `PluginConfigMap` to validate config emqx_plugins:put_plugin_config(Name, AvroJsonMap, PluginConfigMap). %%-------------------------------------------------------------------- From e5bd747b32e4d8f08c0504e3147dd1e6d4d11c93 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Apr 2024 17:01:16 +0800 Subject: [PATCH 50/86] refactor: read avsc file when make serde --- apps/emqx_plugins/src/emqx_plugins.erl | 23 +++------ apps/emqx_plugins/src/emqx_plugins_serde.erl | 52 +++++++++----------- 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 7a73cc72b..c4ef79e17 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -1012,22 +1012,15 @@ maybe_post_op_after_install(NameVsn) -> ok. maybe_load_config_schema(NameVsn) -> - filelib:is_regular(avsc_file_path(NameVsn)) andalso - do_load_config_schema(NameVsn). + AvscPath = avsc_file_path(NameVsn), + filelib:is_regular(AvscPath) andalso + do_load_config_schema(NameVsn, AvscPath). -do_load_config_schema(NameVsn) -> - case read_plugin_avsc(NameVsn, #{read_mode => ?RAW_BIN}) of - {ok, AvscBin} -> - case emqx_plugins_serde:add_schema(NameVsn, AvscBin) of - ok -> ok; - {error, already_exists} -> ok; - {error, _Reason} -> ok - end; - {error, Reason} -> - ?SLOG(warning, #{ - msg => "failed_to_read_plugin_avsc", reason => Reason, name_vsn => NameVsn - }), - ok +do_load_config_schema(NameVsn, AvscPath) -> + case emqx_plugins_serde:add_schema(NameVsn, AvscPath) of + ok -> ok; + {error, already_exists} -> ok; + {error, _Reason} -> ok end. maybe_create_config_dir(NameVsn) -> diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index b936020a6..00fd04b63 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -59,20 +59,20 @@ lookup_serde(SchemaName) -> end. -spec add_schema(schema_name(), avsc()) -> ok | {error, term()}. -add_schema(Name, Avsc) -> - case lookup_serde(Name) of +add_schema(NameVsn, Path) -> + case lookup_serde(NameVsn) of {ok, _Serde} -> - ?SLOG(warning, #{msg => "plugin_schema_already_exists", plugin => Name}), + ?SLOG(warning, #{msg => "plugin_schema_already_exists", plugin => NameVsn}), {error, already_exists}; {error, not_found} -> - case gen_server:call(?MODULE, {build_serdes, to_bin(Name), Avsc}, infinity) of + case gen_server:call(?MODULE, {build_serdes, to_bin(NameVsn), Path}, infinity) of ok -> - ?SLOG(debug, #{msg => "plugin_schema_added", plugin => Name}), + ?SLOG(debug, #{msg => "plugin_schema_added", plugin => NameVsn}), ok; {error, Reason} = E -> ?SLOG(error, #{ msg => "plugin_schema_add_failed", - plugin => Name, + plugin => NameVsn, reason => emqx_utils:readable_error_msg(Reason) }), E @@ -123,15 +123,15 @@ init(_) -> public, ordered_set, {keypos, #plugin_schema_serde.name} ]), State = #{}, - SchemasMap = read_plugin_avsc(), - {ok, State, {continue, {build_serdes, SchemasMap}}}. + AvscPaths = get_plugin_avscs(), + {ok, State, {continue, {build_serdes, AvscPaths}}}. -handle_continue({build_serdes, SchemasMap}, State) -> - _ = build_serdes(SchemasMap), +handle_continue({build_serdes, AvscPaths}, State) -> + _ = build_serdes(AvscPaths), {noreply, State}. -handle_call({build_serdes, NameVsn, Avsc}, _From, State) -> - BuildRes = do_build_serde(NameVsn, Avsc), +handle_call({build_serdes, NameVsn, AvscPath}, _From, State) -> + BuildRes = do_build_serde({NameVsn, AvscPath}), {reply, BuildRes, State}; handle_call(_Call, _From, State) -> {reply, {error, unknown_call}, State}. @@ -149,29 +149,24 @@ terminate(_Reason, _State) -> %% Internal fns %%------------------------------------------------------------------------------------------------- -read_plugin_avsc() -> +-spec get_plugin_avscs() -> [{string(), string()}]. +get_plugin_avscs() -> Pattern = filename:join([emqx_plugins:install_dir(), "*", "config_schema.avsc"]), lists:foldl( fun(AvscPath, AccIn) -> - case read_avsc_file(AvscPath) of - {ok, Avsc} -> - [_, NameVsn | _] = lists:reverse(filename:split(AvscPath)), - AccIn#{to_bin(NameVsn) => Avsc}; - {error, Reason} -> - ?SLOG(warning, Reason), - AccIn - end + [_, NameVsn | _] = lists:reverse(filename:split(AvscPath)), + [{NameVsn, AvscPath} | AccIn] end, - _Acc0 = #{}, + _Acc0 = [], filelib:wildcard(Pattern) ). -build_serdes(Schemas) -> - maps:foreach(fun do_build_serde/2, Schemas). +build_serdes(AvscPaths) -> + ok = lists:foreach(fun do_build_serde/1, AvscPaths). -do_build_serde(NameVsn, Avsc) -> +do_build_serde({NameVsn, AvscPath}) -> try - Serde = make_serde(NameVsn, Avsc), + Serde = make_serde(NameVsn, AvscPath), true = ets:insert(?PLUGIN_SERDE_TAB, Serde), ok catch @@ -189,11 +184,12 @@ do_build_serde(NameVsn, Avsc) -> {error, Error} end. -make_serde(NameVsn, Avsc) -> +make_serde(NameVsn, AvscPath) -> + {ok, AvscBin} = read_avsc_file(AvscPath), Store0 = avro_schema_store:new([map]), %% import the schema into the map store with an assigned name %% if it's a named schema (e.g. struct), then Name is added as alias - Store = avro_schema_store:import_schema_json(NameVsn, Avsc, Store0), + Store = avro_schema_store:import_schema_json(NameVsn, AvscBin, Store0), #plugin_schema_serde{ name = NameVsn, eval_context = Store From 28e8131984b32becc0eeabec70597afb3bb31b51 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 10:37:15 +0800 Subject: [PATCH 51/86] refactor: avoid make when do serde --- apps/emqx_plugins/src/emqx_plugins_serde.erl | 44 ++++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl index 00fd04b63..fc1321ff1 100644 --- a/apps/emqx_plugins/src/emqx_plugins_serde.erl +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -102,15 +102,17 @@ delete_schema(NameVsn) -> -spec decode(schema_name(), encoded_data()) -> {ok, decoded_data()} | {error, any()}. decode(SerdeName, RawData) -> with_serde( - "decode_avro_json", - eval_serde_fun(?FUNCTION_NAME, "bad_avro_binary", SerdeName, [RawData]) + ?FUNCTION_NAME, + SerdeName, + [RawData] ). -spec encode(schema_name(), decoded_data()) -> {ok, encoded_data()} | {error, any()}. encode(SerdeName, Data) -> with_serde( - "encode_avro_json", - eval_serde_fun(?FUNCTION_NAME, "bad_avro_data", SerdeName, [Data]) + ?FUNCTION_NAME, + SerdeName, + [Data] ). %%------------------------------------------------------------------------------------------------- @@ -209,9 +211,11 @@ ensure_serde_absent(Name) -> async_delete_serdes(Names) -> gen_server:cast(?MODULE, {delete_serdes, Names}). -with_serde(WhichOp, Fun) -> +with_serde(Op, SerdeName, Args) -> + WhichOp = which_op(Op), + ErrMsg = error_msg(Op), try - Fun() + eval_serde(Op, ErrMsg, SerdeName, Args) catch throw:Reason -> ?SLOG(error, Reason#{ @@ -233,18 +237,16 @@ with_serde(WhichOp, Fun) -> }} end. -eval_serde_fun(Op, ErrMsg, SerdeName, Args) -> - fun() -> - case lookup_serde(SerdeName) of - {ok, Serde} -> - eval_serde(Op, Serde, Args); - {error, not_found} -> - throw(#{ - error_msg => ErrMsg, - reason => plugin_serde_not_found, - serde_name => SerdeName - }) - end +eval_serde(Op, ErrMsg, SerdeName, Args) -> + case lookup_serde(SerdeName) of + {ok, Serde} -> + eval_serde(Op, Serde, Args); + {error, not_found} -> + throw(#{ + error_msg => ErrMsg, + reason => plugin_serde_not_found, + serde_name => SerdeName + }) end. eval_serde(decode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> @@ -255,6 +257,12 @@ eval_serde(encode, #plugin_schema_serde{name = Name, eval_context = Store}, [Dat eval_serde(_, _, _) -> throw(#{error_msg => "unexpected_plugin_avro_op"}). +which_op(Op) -> + atom_to_list(Op) ++ "_avro_json". + +error_msg(Op) -> + atom_to_list(Op) ++ "_avro_data". + read_avsc_file(Path) -> case file:read_file(Path) of {ok, Bin} -> From 11389bc086086359a2897234398c6235c0dd05a5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 11:12:24 +0800 Subject: [PATCH 52/86] fix: i18n file renamed --- apps/emqx_plugins/src/emqx_plugins.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index c4ef79e17..87932b2b7 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -1082,7 +1082,7 @@ avro_config_file(NameVsn) -> filename:join([plugin_config_dir(NameVsn), "config.avro"]). i18n_file_path(NameVsn) -> - filename:join([plugin_dir(NameVsn), "i18n.json"]). + filename:join([plugin_dir(NameVsn), "config_i18n.json"]). readme_file(NameVsn) -> filename:join([plugin_dir(NameVsn), "README.md"]). From 00cab33fde233e3fc305bccade96c273060b7ea8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 11:49:12 +0800 Subject: [PATCH 53/86] fix: plugin's internal config api --- .../src/emqx_mgmt_api_plugins.erl | 5 +-- apps/emqx_plugins/src/emqx_plugins.erl | 38 ++++++++++++++----- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 9cc518dd0..375d3d2dd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -488,7 +488,7 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> plugin_config(get, #{bindings := #{name := NameVsn}}) -> case emqx_plugins:describe(NameVsn) of {ok, _} -> - case emqx_plugins:get_plugin_config(NameVsn, #{format => ?CONFIG_FORMAT_MAP}) of + case emqx_plugins:get_plugin_config(NameVsn) of {ok, AvroJson} -> {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; {error, _} -> @@ -503,8 +503,7 @@ plugin_config(get, #{bindings := #{name := NameVsn}}) -> plugin_config(put, #{bindings := #{name := NameVsn}, body := AvroJsonMap}) -> case emqx_plugins:describe(NameVsn) of {ok, _} -> - AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), - case emqx_plugins:decode_plugin_avro_config(NameVsn, AvroJsonBin) of + case emqx_plugins:decode_plugin_avro_config(NameVsn, AvroJsonMap) of {ok, AvroValueConfig} -> Nodes = emqx:running_nodes(), %% cluster call with config in map (binary key-value) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 87932b2b7..065ab701b 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -28,7 +28,8 @@ plugin_avsc/1, plugin_i18n/1, plugin_avro/1, - parse_name_vsn/1 + parse_name_vsn/1, + make_name_vsn_string/2 ]). %% Package operations @@ -49,19 +50,22 @@ ensure_started/1, ensure_stopped/0, ensure_stopped/1, - get_plugin_config/1, - get_plugin_config/2, - put_plugin_config/3, restart/1, list/0 ]). +%% Plugin config APIs +-export([ + get_plugin_config/1, + get_plugin_config/2, + get_plugin_config/3, + get_plugin_config/4, + put_plugin_config/3 +]). + %% Package utils -export([ decode_plugin_avro_config/2, - get_config/2, - put_config/2, - get_tar/1, install_dir/0, avsc_file_path/1 ]). @@ -73,6 +77,8 @@ %% Internal export -export([do_ensure_started/1]). +%% for test cases +-export([put_config/2]). -ifdef(TEST). -compile(export_all). @@ -124,6 +130,9 @@ parse_name_vsn(NameVsn) when is_list(NameVsn) -> _ -> {error, "bad_name_vsn"} end. +make_name_vsn_string(Name, Vsn) -> + binary_to_list(iolist_to_binary([Name, "-", Vsn])). + %%-------------------------------------------------------------------- %% Package operations @@ -246,21 +255,30 @@ ensure_stopped(NameVsn) -> end ). +get_plugin_config(Name, Vsn, Options, Default) -> + get_plugin_config(make_name_vsn_string(Name, Vsn), Options, Default). + -spec get_plugin_config(name_vsn()) -> - {ok, plugin_config()} | {error, term()}. + {ok, plugin_config()} + | {error, term()}. get_plugin_config(NameVsn) -> get_plugin_config(bin(NameVsn), #{format => ?CONFIG_FORMAT_MAP}). -spec get_plugin_config(name_vsn(), Options :: map()) -> {ok, avro_binary() | plugin_config()} | {error, term()}. + get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> + %% no default value when get raw binary config case read_plugin_avro(NameVsn) of {ok, _AvroJson} = Res -> Res; {error, _Reason} = Err -> Err end; -get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}) -> - {ok, persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), #{})}. +get_plugin_config(NameVsn, Options = #{format := ?CONFIG_FORMAT_MAP}) -> + get_plugin_config(NameVsn, Options, #{}). + +get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}, Default) -> + {ok, persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), Default)}. %% @doc Update plugin's config. %% RPC call from Management API or CLI. From 5ff4e7690494dac7c5d445065fd9bcd906a011a6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 18:06:28 +0800 Subject: [PATCH 54/86] refactor: rename plugins config api functions --- .../src/emqx_mgmt_api_plugins.erl | 4 +- .../test/emqx_mgmt_api_plugins_SUITE.erl | 6 +- apps/emqx_plugins/src/emqx_plugins.erl | 61 +++++++++---------- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 10 +-- apps/emqx_plugins/test/emqx_plugins_tests.erl | 4 +- 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 375d3d2dd..94b16320a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -488,7 +488,7 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> plugin_config(get, #{bindings := #{name := NameVsn}}) -> case emqx_plugins:describe(NameVsn) of {ok, _} -> - case emqx_plugins:get_plugin_config(NameVsn) of + case emqx_plugins:get_config(NameVsn) of {ok, AvroJson} -> {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; {error, _} -> @@ -601,7 +601,7 @@ ensure_action(Name, restart) -> %% for RPC plugin avro encoded config update do_update_plugin_config(Name, AvroJsonMap, PluginConfigMap) -> %% TODO: maybe use `PluginConfigMap` to validate config - emqx_plugins:put_plugin_config(Name, AvroJsonMap, PluginConfigMap). + emqx_plugins:put_config(Name, AvroJsonMap, PluginConfigMap). %%-------------------------------------------------------------------- %% Helper functions diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 4e5dacc7a..e563ba262 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -37,10 +37,10 @@ init_per_suite(Config) -> ok = filelib:ensure_dir(WorkDir), DemoShDir1 = string:replace(WorkDir, "emqx_mgmt_api_plugins", "emqx_plugins"), DemoShDir = lists:flatten(string:replace(DemoShDir1, "emqx_management", "emqx_plugins")), - OrigInstallDir = emqx_plugins:get_config(install_dir, undefined), + OrigInstallDir = emqx_plugins:get_config_interal(install_dir, undefined), ok = filelib:ensure_dir(DemoShDir), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_plugins]), - emqx_plugins:put_config(install_dir, DemoShDir), + emqx_plugins:put_config_internal(install_dir, DemoShDir), [{demo_sh_dir, DemoShDir}, {orig_install_dir, OrigInstallDir} | Config]. end_per_suite(Config) -> @@ -48,7 +48,7 @@ end_per_suite(Config) -> %% restore config case proplists:get_value(orig_install_dir, Config) of undefined -> ok; - OrigInstallDir -> emqx_plugins:put_config(install_dir, OrigInstallDir) + OrigInstallDir -> emqx_plugins:put_config_internal(install_dir, OrigInstallDir) end, emqx_mgmt_api_test_util:end_suite([emqx_plugins, emqx_conf]), ok. diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 065ab701b..11edac123 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -56,11 +56,11 @@ %% Plugin config APIs -export([ - get_plugin_config/1, - get_plugin_config/2, - get_plugin_config/3, - get_plugin_config/4, - put_plugin_config/3 + get_config/1, + get_config/2, + get_config/3, + get_config/4, + put_config/3 ]). %% Package utils @@ -78,7 +78,7 @@ %% Internal export -export([do_ensure_started/1]). %% for test cases --export([put_config/2]). +-export([put_config_internal/2]). -ifdef(TEST). -compile(export_all). @@ -255,35 +255,34 @@ ensure_stopped(NameVsn) -> end ). -get_plugin_config(Name, Vsn, Options, Default) -> - get_plugin_config(make_name_vsn_string(Name, Vsn), Options, Default). +get_config(Name, Vsn, Options, Default) -> + get_config(make_name_vsn_string(Name, Vsn), Options, Default). --spec get_plugin_config(name_vsn()) -> +-spec get_config(name_vsn()) -> {ok, plugin_config()} | {error, term()}. -get_plugin_config(NameVsn) -> - get_plugin_config(bin(NameVsn), #{format => ?CONFIG_FORMAT_MAP}). +get_config(NameVsn) -> + get_config(bin(NameVsn), #{format => ?CONFIG_FORMAT_MAP}). --spec get_plugin_config(name_vsn(), Options :: map()) -> +-spec get_config(name_vsn(), Options :: map()) -> {ok, avro_binary() | plugin_config()} | {error, term()}. - -get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> +get_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> %% no default value when get raw binary config case read_plugin_avro(NameVsn) of {ok, _AvroJson} = Res -> Res; {error, _Reason} = Err -> Err end; -get_plugin_config(NameVsn, Options = #{format := ?CONFIG_FORMAT_MAP}) -> - get_plugin_config(NameVsn, Options, #{}). +get_config(NameVsn, Options = #{format := ?CONFIG_FORMAT_MAP}) -> + get_config(NameVsn, Options, #{}). -get_plugin_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}, Default) -> +get_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}, Default) -> {ok, persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), Default)}. %% @doc Update plugin's config. %% RPC call from Management API or CLI. -%% the avro binary and plugin config ALWAYS be valid before calling this function. -put_plugin_config(NameVsn, AvroJsonMap, _DecodedPluginConfig) -> +%% the avro Json Map and plugin config ALWAYS be valid before calling this function. +put_config(NameVsn, AvroJsonMap, _DecodedPluginConfig) -> AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), ok = write_avro_bin(NameVsn, AvroJsonBin), ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), AvroJsonMap), @@ -328,13 +327,13 @@ decode_plugin_avro_config(NameVsn, AvroJsonBin) -> {error, ReasonMap} -> {error, ReasonMap} end. -get_config(Key, Default) when is_atom(Key) -> - get_config([Key], Default); -get_config(Path, Default) -> +get_config_interal(Key, Default) when is_atom(Key) -> + get_config_interal([Key], Default); +get_config_interal(Path, Default) -> emqx_conf:get([?CONF_ROOT | Path], Default). -put_config(Key, Value) -> - do_put_config(Key, Value, _ConfLocation = local). +put_config_internal(Key, Value) -> + do_put_config_internal(Key, Value, _ConfLocation = local). -spec get_tar(name_vsn()) -> {ok, binary()} | {error, any}. get_tar(NameVsn) -> @@ -950,16 +949,16 @@ is_needed_by(AppToStop, RunningApp) -> undefined -> false end. -do_put_config(Key, Value, ConfLocation) when is_atom(Key) -> - do_put_config([Key], Value, ConfLocation); -do_put_config(Path, Values, _ConfLocation = local) when is_list(Path) -> +do_put_config_internal(Key, Value, ConfLocation) when is_atom(Key) -> + do_put_config_internal([Key], Value, ConfLocation); +do_put_config_internal(Path, Values, _ConfLocation = local) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, %% Already in cluster_rpc, don't use emqx_conf:update, dead calls case emqx:update_config([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error end; -do_put_config(Path, Values, _ConfLocation = global) when is_list(Path) -> +do_put_config_internal(Path, Values, _ConfLocation = global) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, case emqx_conf:update([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; @@ -995,16 +994,16 @@ enable_disable_plugin(_NameVsn, _Diff) -> %%-------------------------------------------------------------------- install_dir() -> - get_config(install_dir, ""). + get_config_interal(install_dir, ""). put_configured(Configured) -> put_configured(Configured, _ConfLocation = local). put_configured(Configured, ConfLocation) -> - ok = do_put_config(states, bin_key(Configured), ConfLocation). + ok = do_put_config_internal(states, bin_key(Configured), ConfLocation). configured() -> - get_config(states, []). + get_config_interal(states, []). for_plugins(ActionFun) -> case lists:flatmap(fun(I) -> for_plugin(I, ActionFun) end, configured()) of diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index d7bdfad13..80f3d7a48 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -626,9 +626,9 @@ group_t_copy_plugin_to_a_new_node({init, Config}) -> } ), [CopyFromNode] = emqx_cth_cluster:start([SpecCopyFrom#{join_to => undefined}]), - ok = rpc:call(CopyFromNode, emqx_plugins, put_config, [install_dir, FromInstallDir]), + ok = rpc:call(CopyFromNode, emqx_plugins, put_config_internal, [install_dir, FromInstallDir]), [CopyToNode] = emqx_cth_cluster:start([SpecCopyTo#{join_to => undefined}]), - ok = rpc:call(CopyToNode, emqx_plugins, put_config, [install_dir, ToInstallDir]), + ok = rpc:call(CopyToNode, emqx_plugins, put_config_internal, [install_dir, ToInstallDir]), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), ok = rpc:call(CopyFromNode, emqx_plugins, ensure_installed, [NameVsn]), ok = rpc:call(CopyFromNode, emqx_plugins, ensure_started, [NameVsn]), @@ -658,7 +658,7 @@ group_t_copy_plugin_to_a_new_node(Config) -> CopyFromNode = proplists:get_value(copy_from_node, Config), CopyToNode = proplists:get_value(copy_to_node, Config), CopyToDir = proplists:get_value(to_install_dir, Config), - CopyFromPluginsState = rpc:call(CopyFromNode, emqx_plugins, get_config, [[states], []]), + CopyFromPluginsState = rpc:call(CopyFromNode, emqx_plugins, get_config_interal, [[states], []]), NameVsn = proplists:get_value(name_vsn, Config), PluginName = proplists:get_value(plugin_name, Config), PluginApp = list_to_atom(PluginName), @@ -681,7 +681,7 @@ group_t_copy_plugin_to_a_new_node(Config) -> ), ok = rpc:call(CopyToNode, ekka, join, [CopyFromNode]), %% Mimic cluster-override conf copying - ok = rpc:call(CopyToNode, emqx_plugins, put_config, [[states], CopyFromPluginsState]), + ok = rpc:call(CopyToNode, emqx_plugins, put_config_internal, [[states], CopyFromPluginsState]), %% Plugin copying is triggered upon app restart on a new node. %% This is similar to emqx_conf, which copies cluster-override conf upon start, %% see: emqx_conf_app:init_conf/0 @@ -734,7 +734,7 @@ group_t_copy_plugin_to_a_new_node_single_node(Config) -> %% successfully even if it's not extracted yet. Simply starting %% the node would crash if not working properly. ct:pal("~p config:\n ~p", [ - CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config, [[], #{}]) + CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config_interal, [[], #{}]) ]), ct:pal("~p install_dir:\n ~p", [ CopyToNode, erpc:call(CopyToNode, file, list_dir, [ToInstallDir]) diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index 4911c174e..1ae0bcef3 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -72,12 +72,12 @@ with_rand_install_dir(F) -> TmpDir = integer_to_list(N), OriginalInstallDir = emqx_plugins:install_dir(), ok = filelib:ensure_dir(filename:join([TmpDir, "foo"])), - ok = emqx_plugins:put_config(install_dir, TmpDir), + ok = emqx_plugins:put_config_internal(install_dir, TmpDir), try F(TmpDir) after file:del_dir_r(TmpDir), - ok = emqx_plugins:put_config(install_dir, OriginalInstallDir) + ok = emqx_plugins:put_config_internal(install_dir, OriginalInstallDir) end. write_file(Path, Content) -> From 43ac4f5dfeb091ce63a323ad686eba26439127d3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 21:17:01 +0800 Subject: [PATCH 55/86] fix: make bpapi check happy --- apps/emqx_plugins/src/emqx_plugins.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 11edac123..67d25bf7a 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -75,6 +75,9 @@ post_config_update/5 ]). +%% RPC call +-export([get_tar/1]). + %% Internal export -export([do_ensure_started/1]). %% for test cases From b98f3d27b88dd64336f01d65e8c1737c5bdd7d40 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Apr 2024 22:30:08 +0800 Subject: [PATCH 56/86] docs: add change log entry for #12910 --- changes/feat-12910.en.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 changes/feat-12910.en.md diff --git a/changes/feat-12910.en.md b/changes/feat-12910.en.md new file mode 100644 index 000000000..d31fa7ef7 --- /dev/null +++ b/changes/feat-12910.en.md @@ -0,0 +1,15 @@ +Provided a configuration API endpoint for plugin functionality. +This allows users to describe the configuration struct of their plugins using AVRO schema. +During plugin runtime, the plugin's configuration can be accessed via the API. + +Added new API endpoints: +- `/plugins/:name/schema` + To get plugins avro schema and i18n config in one json object. +- `/plugins/:name/config` + To get or update plugin's own config + +Changed API endpoints: +- `/plugins/install` + Status code when succeeded change to `204`. It was `200` previously. +- `/plugins/:name/move` + Status code when succeeded change to `204`. It was `200` previously. From b96c6c243aa056d3fb263e1e5528d68c00e3fd3d Mon Sep 17 00:00:00 2001 From: zmstone Date: Fri, 26 Apr 2024 10:45:10 +0200 Subject: [PATCH 57/86] docs: improve mqtt_path doc --- rel/i18n/emqx_schema.hocon | 12 +++++++++--- scripts/spellcheck/dicts/emqx.txt | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index cb504694c..b7c212dfd 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -268,9 +268,15 @@ common_ssl_opts_schema_cacerts.desc: common_ssl_opts_schema_cacerts.label: """Use System CA Certificates""" -fields_ws_opts_mqtt_path.desc: -"""WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: -ws://{ip}:{port}/mqtt""" +fields_ws_opts_mqtt_path.desc: """~ + WebSocket's MQTT protocol path. By default, the full URL for the WebSocket client to connect is: + `ws://{ip}:{port}/mqtt`. + Append `/[...]` to the end of the path to make EMQX accept any subpath. + For example, specifying `mqtt/[...]` would allow clients to connect at paths like + `mqtt/org1` or `mqtt/group2`, etc. + + NOTE: An unmatched path will cause the client to be rejected immediately at the HTTP layer, + meaning it will not be traceable at the MQTT layer.""" fields_ws_opts_mqtt_path.label: """WS MQTT Path""" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index d68c85716..af5a34531 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -234,6 +234,7 @@ src ssl statsd structs +subpath subprotocol subprotocols superset From 5f4215b3333f8007bf6c1004417ab12da43161ef Mon Sep 17 00:00:00 2001 From: zmstone Date: Sat, 27 Apr 2024 21:02:09 +0200 Subject: [PATCH 58/86] fix(mgmt): avoid 500 error when hocon syntax error --- apps/emqx/src/emqx_logger_jsonfmt.erl | 9 +++-- .../src/emqx_mgmt_api_configs.erl | 7 +--- .../test/emqx_mgmt_api_configs_SUITE.erl | 38 ++++++++++++++----- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 2efb0e032..92c0bb561 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -32,7 +32,7 @@ -export([format/2]). %% For CLI HTTP API outputs --export([best_effort_json/1, best_effort_json/2]). +-export([best_effort_json/1, best_effort_json/2, best_effort_json_obj/1]). -ifdef(TEST). -include_lib("proper/include/proper.hrl"). @@ -65,10 +65,13 @@ best_effort_json(Input) -> best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> - Config = #{depth => unlimited, single_line => true, chars_limit => unlimited}, - JsonReady = best_effort_json_obj(Input, Config), + JsonReady = best_effort_json_obj(Input), emqx_utils_json:encode(JsonReady, Opts). +best_effort_json_obj(Input) -> + Config = #{depth => unlimited, single_line => true, chars_limit => unlimited}, + best_effort_json_obj(Input, Config). + -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) -> Config = add_default_config(Config0), diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index f013dfcd1..decabb64b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -366,11 +366,8 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> ok -> {200}; %% bad hocon format - {error, MsgList = [{_, _} | _]} -> - JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end, - JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun), - {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap}; - {error, Msg} -> + {error, Errors} -> + Msg = emqx_logger_jsonfmt:best_effort_json_obj(#{errors => Errors}), {400, #{<<"content-type">> => <<"text/plain">>}, Msg} end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2c90c9dac..80b9aa9df 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -336,13 +336,15 @@ t_configs_key(_Config) -> BadLog = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"erro1r">>), {error, Error} = update_configs_with_binary(iolist_to_binary(hocon_pp:do(BadLog, #{}))), ExpectError = #{ - <<"log">> => - #{ - <<"kind">> => <<"validation_error">>, - <<"path">> => <<"log.console.level">>, - <<"reason">> => <<"unable_to_convert_to_enum_symbol">>, - <<"value">> => <<"erro1r">> - } + <<"errors">> => #{ + <<"log">> => + #{ + <<"kind">> => <<"validation_error">>, + <<"path">> => <<"log.console.level">>, + <<"reason">> => <<"unable_to_convert_to_enum_symbol">>, + <<"value">> => <<"erro1r">> + } + } }, ?assertEqual(ExpectError, emqx_utils_json:decode(Error, [return_maps])), ReadOnlyConf = #{ @@ -355,7 +357,7 @@ t_configs_key(_Config) -> }, ReadOnlyBin = iolist_to_binary(hocon_pp:do(ReadOnlyConf, #{})), {error, ReadOnlyError} = update_configs_with_binary(ReadOnlyBin), - ?assertEqual(<<"Cannot update read-only key 'cluster'.">>, ReadOnlyError), + ?assertEqual(<<"{\"errors\":\"Cannot update read-only key 'cluster'.\"}">>, ReadOnlyError), ok. t_get_configs_in_different_accept(_Config) -> @@ -487,6 +489,22 @@ t_create_webhook_v1_bridges_api(Config) -> ?assertEqual(#{<<"webhook">> => #{}}, emqx_conf:get_raw([<<"bridges">>])), ok. +t_config_update_parse_error(_Config) -> + ?assertMatch( + {error, <<"{\"errors\":\"{parse_error,", _/binary>>}, + update_configs_with_binary(<<"not an object">>) + ), + ?assertMatch( + {error, <<"{\"errors\":\"{parse_error,", _/binary>>}, + update_configs_with_binary(<<"a = \"tlsv1\"\"\"3e-01">>) + ). + +t_config_update_unknown_root(_Config) -> + ?assertMatch( + {error, <<"{\"errors\":{\"a\":\"{root_key_not_found,", _/binary>>}, + update_configs_with_binary(<<"a = \"tlsv1.3\"">>) + ). + %% Helpers get_config(Name) -> @@ -547,10 +565,10 @@ update_configs_with_binary(Bin) -> Code >= 200 andalso Code =< 299 -> Body; - {ok, {{"HTTP/1.1", _Code, _}, _Headers, Body}} -> + {ok, {{"HTTP/1.1", 400, _}, _Headers, Body}} -> {error, Body}; Error -> - Error + error({unexpected, Error}) end. update_config(Name, Change) -> From e76c350d302c7a625f82688862beda89feccfa71 Mon Sep 17 00:00:00 2001 From: zmstone Date: Sat, 27 Apr 2024 21:19:40 +0200 Subject: [PATCH 59/86] docs: add changelog for PR 12940 --- changes/ce/fix-12940.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ce/fix-12940.en.md diff --git a/changes/ce/fix-12940.en.md b/changes/ce/fix-12940.en.md new file mode 100644 index 000000000..8304effd5 --- /dev/null +++ b/changes/ce/fix-12940.en.md @@ -0,0 +1,3 @@ +Fix config update API `/configs` 500 error when config has bad HOCON syntax. + +Now bad syntax will cause the API to return 400 (BAD_REQUEST). From d6c203b4fd200c55ddfedec0f74ae1bb276cb8c0 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 28 Apr 2024 16:54:32 +0800 Subject: [PATCH 60/86] fix(dynamo): fixed the keys checking for Dynamo --- .../src/emqx_bridge_dynamo_connector.erl | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 598b3342d..f89786929 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -134,20 +134,10 @@ on_add_channel( create_channel_state( #{parameters := Conf} = _ChannelConfig ) -> - Keys = maps:with([hash_key, range_key], Conf), - Keys1 = maps:fold( - fun(K, V, Acc) -> - Acc#{K := erlang:binary_to_existing_atom(V)} - end, - Keys, - Keys - ), - - Base = maps:without([template, hash_key, range_key], Conf), - Base1 = maps:merge(Base, Keys1), + Base = maps:without([template], Conf), Templates = parse_template_from_conf(Conf), - State = Base1#{ + State = Base#{ templates => Templates }, {ok, State}. @@ -318,12 +308,12 @@ get_query_tuple([InsertQuery | _]) -> ensuare_dynamo_keys({_, Data} = Query, State) when is_map(Data) -> ensuare_dynamo_keys([Query], State); ensuare_dynamo_keys([{_, Data} | _] = Queries, State) when is_map(Data) -> - Keys = maps:to_list(maps:with([hash_key, range_key], State)), + Keys = maps:values(maps:with([hash_key, range_key], State)), lists:all( fun({_, Query}) -> lists:all( - fun({_, Key}) -> - maps:is_key(Key, Query) + fun(Key) -> + is_dynamo_key_existing(Key, Query) end, Keys ) @@ -371,3 +361,17 @@ get_host_info(Server) -> redact(Data) -> emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). + +is_dynamo_key_existing(Bin, Query) when is_binary(Bin) -> + case maps:is_key(Bin, Query) of + true -> + true; + _ -> + try + Key = erlang:binary_to_existing_atom(Bin), + maps:is_key(Key, Query) + catch + _:_ -> + false + end + end. From 10625eacacc8cbced485b22a06d3643f57c4609f Mon Sep 17 00:00:00 2001 From: zmstone Date: Sun, 28 Apr 2024 14:05:30 +0200 Subject: [PATCH 61/86] chore: upgrade to hocon-0.42.2 hocon pretty-print quotes more string values if a string has '.' or '-', or if it starts with a digit 0-9, then it's quoted. see details here: https://github.com/emqx/hocon/pull/293 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b58734cf8..b58cd0cb7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.2"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index bbb11cc52..2cc44099a 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.42.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.1"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index e0f88893c..cd70493e5 100644 --- a/rebar.config +++ b/rebar.config @@ -97,7 +97,7 @@ {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, {getopt, "1.0.2"}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.1"}}}, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, From 07cbdc6e90116bd8a678e93b50d0e3e077ebdbcd Mon Sep 17 00:00:00 2001 From: zmstone Date: Sun, 28 Apr 2024 14:45:53 +0200 Subject: [PATCH 62/86] feat(mgmt): add ignore_readonly qeury-string to PUT /configs API --- apps/emqx_conf/src/emqx_conf_cli.erl | 22 +++++++++++-------- .../src/emqx_mgmt_api_configs.erl | 13 ++++++++--- .../test/emqx_mgmt_api_configs_SUITE.erl | 20 +++++++++++++---- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index d6462a0b6..7085eb897 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -242,7 +242,7 @@ load_config(Bin, Opts) when is_binary(Bin) -> load_config_from_raw(RawConf0, Opts) -> SchemaMod = emqx_conf:schema_module(), RawConf1 = emqx_config:upgrade_raw_conf(SchemaMod, RawConf0), - case check_config(RawConf1) of + case check_config(RawConf1, Opts) of {ok, RawConf} -> %% It has been ensured that the connector is always the first configuration to be updated. %% However, when deleting the connector, we need to clean up the dependent actions/sources first; @@ -395,24 +395,28 @@ suggest_msg(#{kind := validation_error, reason := unknown_fields}, Mode) -> suggest_msg(_, _) -> <<"">>. -check_config(Conf) -> - case check_keys_is_not_readonly(Conf) of - ok -> - Conf1 = emqx_config:fill_defaults(Conf), - case check_config_schema(Conf1) of - ok -> {ok, Conf1}; +check_config(Conf0, Opts) -> + case check_keys_is_not_readonly(Conf0, Opts) of + {ok, Conf1} -> + Conf = emqx_config:fill_defaults(Conf1), + case check_config_schema(Conf) of + ok -> {ok, Conf}; {error, Reason} -> {error, Reason} end; Error -> Error end. -check_keys_is_not_readonly(Conf) -> +check_keys_is_not_readonly(Conf, Opts) -> + IgnoreReadonly = maps:get(ignore_readonly, Opts, false), Keys = maps:keys(Conf), ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case lists:filter(fun(K) -> lists:member(K, Keys) end, ReadOnlyKeys) of [] -> - ok; + {ok, Conf}; + BadKeys when IgnoreReadonly -> + ?SLOG(warning, #{msg => "readonly_root_keys_ignored", keys => BadKeys}), + {ok, maps:without(BadKeys, Conf)}; BadKeys -> BadKeysStr = lists:join(<<",">>, BadKeys), {error, ?UPDATE_READONLY_KEYS_PROHIBITED, BadKeysStr} diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index decabb64b..75341facc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -147,7 +147,9 @@ schema("/configs") -> hoconsc:mk( hoconsc:enum([replace, merge]), #{in => query, default => merge, required => false} - )} + )}, + {ignore_readonly, + hoconsc:mk(boolean(), #{in => query, default => false, required => false})} ], 'requestBody' => #{ content => @@ -361,8 +363,13 @@ configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> {ok, <<"text/plain">>} -> get_configs_v2(QueryStr); {error, _} = Error -> {400, #{code => 'INVALID_ACCEPT', message => ?ERR_MSG(Error)}} end; -configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> - case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of +configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode} = QS}, _Req) -> + IngnoreReadonly = maps:get(<<"ignore_readonly">>, QS, false), + case + emqx_conf_cli:load_config(Conf, #{ + mode => Mode, log => none, ignore_readonly => IngnoreReadonly + }) + of ok -> {200}; %% bad hocon format diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 80b9aa9df..6d4d94013 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -331,7 +331,7 @@ t_configs_key(_Config) -> Log ), Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>), - ?assertEqual(<<>>, update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), + ?assertEqual({ok, <<>>}, update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), BadLog = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"erro1r">>), {error, Error} = update_configs_with_binary(iolist_to_binary(hocon_pp:do(BadLog, #{}))), @@ -358,6 +358,7 @@ t_configs_key(_Config) -> ReadOnlyBin = iolist_to_binary(hocon_pp:do(ReadOnlyConf, #{})), {error, ReadOnlyError} = update_configs_with_binary(ReadOnlyBin), ?assertEqual(<<"{\"errors\":\"Cannot update read-only key 'cluster'.\"}">>, ReadOnlyError), + ?assertMatch({ok, <<>>}, update_configs_with_binary(ReadOnlyBin, _InogreReadonly = true)), ok. t_get_configs_in_different_accept(_Config) -> @@ -407,7 +408,7 @@ t_create_webhook_v1_bridges_api(Config) -> WebHookFile = filename:join(?config(data_dir, Config), "webhook_v1.conf"), ?assertMatch({ok, _}, hocon:files([WebHookFile])), {ok, WebHookBin} = file:read_file(WebHookFile), - ?assertEqual(<<>>, update_configs_with_binary(WebHookBin)), + ?assertEqual({ok, <<>>}, update_configs_with_binary(WebHookBin)), Actions = #{ <<"http">> => @@ -557,14 +558,25 @@ get_configs_with_binary(Key, Node) -> end. update_configs_with_binary(Bin) -> - Path = emqx_mgmt_api_test_util:api_path(["configs"]), + update_configs_with_binary(Bin, _InogreReadonly = undefined). + +update_configs_with_binary(Bin, IgnoreReadonly) -> + Path = + case IgnoreReadonly of + undefined -> + emqx_mgmt_api_test_util:api_path(["configs"]); + Boolean -> + emqx_mgmt_api_test_util:api_path([ + "configs?ignore_readonly=" ++ atom_to_list(Boolean) + ]) + end, Auth = emqx_mgmt_api_test_util:auth_header_(), Headers = [{"accept", "text/plain"}, Auth], case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], [{body_format, binary}]) of {ok, {{"HTTP/1.1", Code, _}, _Headers, Body}} when Code >= 200 andalso Code =< 299 -> - Body; + {ok, Body}; {ok, {{"HTTP/1.1", 400, _}, _Headers, Body}} -> {error, Body}; Error -> From c3d27347b03947e349e54806a25d914cc91c5d53 Mon Sep 17 00:00:00 2001 From: zmstone Date: Sun, 28 Apr 2024 14:54:01 +0200 Subject: [PATCH 63/86] docs: update changlog for pr 12940 --- changes/ce/feat-12940.en.md | 12 ++++++++++++ changes/ce/fix-12940.en.md | 3 --- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-12940.en.md delete mode 100644 changes/ce/fix-12940.en.md diff --git a/changes/ce/feat-12940.en.md b/changes/ce/feat-12940.en.md new file mode 100644 index 000000000..24bdcc4e2 --- /dev/null +++ b/changes/ce/feat-12940.en.md @@ -0,0 +1,12 @@ +Add `ignore_readonly` argument to `PUT /configs` API. + +Prior to this change, EMQX would retrun 400 (BAD_REQUEST) if the raw config +included readonly root keys (`cluster`, `rpc`, and `node`). + +After this enhancement it can be called as `PUT /configs?ignore_readonly=true`, +EMQX will in this case ignore readonly root config keys, and apply the rest. + +A warning message is logged to warn that the readonly keys are dropped. + +Also fixed an exception when config has bad HOCON syntax (returns 500). +Now bad syntax will cause the API to return 400 (BAD_REQUEST). diff --git a/changes/ce/fix-12940.en.md b/changes/ce/fix-12940.en.md deleted file mode 100644 index 8304effd5..000000000 --- a/changes/ce/fix-12940.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix config update API `/configs` 500 error when config has bad HOCON syntax. - -Now bad syntax will cause the API to return 400 (BAD_REQUEST). From 1db932df213c8980cf317299ae93c9aadeab7571 Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 29 Apr 2024 09:33:43 +0200 Subject: [PATCH 64/86] chore(mgmt): PUT /configs?ignore_readonly=true, lower log to info level --- apps/emqx_conf/src/emqx_conf_cli.erl | 2 +- changes/ce/feat-12940.en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 7085eb897..f1909e59b 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -415,7 +415,7 @@ check_keys_is_not_readonly(Conf, Opts) -> [] -> {ok, Conf}; BadKeys when IgnoreReadonly -> - ?SLOG(warning, #{msg => "readonly_root_keys_ignored", keys => BadKeys}), + ?SLOG(info, #{msg => "readonly_root_keys_ignored", keys => BadKeys}), {ok, maps:without(BadKeys, Conf)}; BadKeys -> BadKeysStr = lists:join(<<",">>, BadKeys), diff --git a/changes/ce/feat-12940.en.md b/changes/ce/feat-12940.en.md index 24bdcc4e2..77e626194 100644 --- a/changes/ce/feat-12940.en.md +++ b/changes/ce/feat-12940.en.md @@ -6,7 +6,7 @@ included readonly root keys (`cluster`, `rpc`, and `node`). After this enhancement it can be called as `PUT /configs?ignore_readonly=true`, EMQX will in this case ignore readonly root config keys, and apply the rest. -A warning message is logged to warn that the readonly keys are dropped. +For observability purposes, an info level message is logged if any readonly keys are dropped. Also fixed an exception when config has bad HOCON syntax (returns 500). Now bad syntax will cause the API to return 400 (BAD_REQUEST). From 67ef42220eb3af16c3a6cf5b3c11b932ae000b6e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 29 Apr 2024 11:49:07 +0200 Subject: [PATCH 65/86] chore: bump builder version to 5.3-4 --- .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/_pr_entrypoint.yaml | 10 +++++----- .github/workflows/_push-entrypoint.yaml | 10 +++++----- .github/workflows/build_and_push_docker_images.yaml | 2 +- .github/workflows/build_packages.yaml | 2 +- .github/workflows/build_packages_cron.yaml | 4 ++-- .github/workflows/build_slim_packages.yaml | 4 ++-- .github/workflows/codeql.yaml | 2 +- .github/workflows/performance_test.yaml | 2 +- Makefile | 2 +- build | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- scripts/pr-sanity-checks.sh | 2 +- scripts/relup-test/start-relup-test-cluster.sh | 2 +- 16 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index bfae12182..158db93f1 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -18,7 +18,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 container_name: kdc.emqx.net expose: - 88 # kdc diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index c21c0dc82..d354a9281 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04} env_file: - credentials.env - conf.env diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index fcec256b0..779b73d16 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -20,15 +20,15 @@ permissions: jobs: sanity-checks: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" outputs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} - builder: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04" - builder_vsn: "5.3-2" + builder: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" + builder_vsn: "5.3-4" otp_vsn: "26.2.1-2" elixir_vsn: "1.15.7" @@ -95,12 +95,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index e9dbb41a0..b141989d5 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -23,7 +23,7 @@ env: jobs: prepare: runs-on: ubuntu-22.04 - container: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' + container: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' outputs: profile: ${{ steps.parse-git-ref.outputs.profile }} release: ${{ steps.parse-git-ref.outputs.release }} @@ -31,8 +31,8 @@ jobs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} - builder: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' - builder_vsn: '5.3-2' + builder: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' + builder_vsn: '5.3-4' otp_vsn: '26.2.1-2' elixir_vsn: '1.15.7' @@ -62,12 +62,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index f1b4a3d90..583b0b42a 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -61,7 +61,7 @@ on: builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' permissions: contents: read diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 3b19e7094..fecb55f3f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -63,7 +63,7 @@ on: builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' permissions: contents: read diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 16c50ceb8..c3e76d1b2 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: profile: - - ['emqx', 'master', '5.3-2:1.15.7-26.2.1-2'] - - ['emqx-enterprise', 'release-56', '5.3-2:1.15.7-26.2.1-2'] + - ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2'] + - ['emqx-enterprise', 'release-56', '5.3-4:1.15.7-26.2.1-2'] os: - debian10 - ubuntu22.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 44e94c35a..c36a6ebd3 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,11 +27,11 @@ on: builder: required: false type: string - default: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' + default: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' otp_vsn: required: false type: string diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 7b9c14d5f..5c2a9a4d3 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -17,7 +17,7 @@ jobs: actions: read security-events: write container: - image: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 strategy: fail-fast: false diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 16874163c..fc524a5f9 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -26,7 +26,7 @@ jobs: prepare: runs-on: ubuntu-latest if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu20.04 outputs: BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} diff --git a/Makefile b/Makefile index b899017b4..83d041260 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 diff --git a/build b/build index 73b83a5b6..18880ca04 100755 --- a/build +++ b/build @@ -395,7 +395,7 @@ function is_ecr_and_enterprise() { ## Build the default docker image based on debian 12. make_docker() { - local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-2}" + local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-4}" local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}" local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}" local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}" diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index ea7bb27cc..1bcc9f30c 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim ARG SOURCE_TYPE=src # tgz diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 90d33e9be..dbe20d501 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +## --builder ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 set -euo pipefail @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12" + echo " E.g. ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12" } die() { diff --git a/scripts/pr-sanity-checks.sh b/scripts/pr-sanity-checks.sh index f9bf53bf2..61265c7e5 100755 --- a/scripts/pr-sanity-checks.sh +++ b/scripts/pr-sanity-checks.sh @@ -12,7 +12,7 @@ if ! type "yq" > /dev/null; then exit 1 fi -EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-2} +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-4} EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.2.1-2} EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index c471aef8b..862c5377b 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE" From 7a05a4754f8c670de2dce1af60259865ae14eabd Mon Sep 17 00:00:00 2001 From: zmstone Date: Sat, 27 Apr 2024 00:36:42 +0200 Subject: [PATCH 66/86] docs: expose zone config in schema doc --- apps/emqx/src/emqx_schema.erl | 27 ++++++++++++++++++--------- rel/i18n/emqx_schema.hocon | 15 +++++++++++---- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 02e31387e..b7ec41011 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -217,14 +217,7 @@ roots(high) -> importance => ?IMPORTANCE_MEDIUM } )}, - {zones, - sc( - map(name, ref("zone")), - #{ - desc => ?DESC(zones), - importance => ?IMPORTANCE_HIDDEN - } - )} + {zones, zones_field_schema()} ] ++ emqx_schema_hooks:injection_point( 'roots.high', @@ -1859,7 +1852,7 @@ base_listener(Bind) -> #{ desc => ?DESC(base_listener_zone), default => 'default', - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_LOW } )}, {"limiter", @@ -1883,6 +1876,22 @@ base_listener(Bind) -> )} ] ++ emqx_limiter_schema:short_paths_fields(). +%% @hidden Starting from 5.7, listenrs.{TYPE}.{NAME}.zone is no longer hidden +%% However, the root key 'zones' is still hidden because the fields' schema +%% just repeat other root field's schema, which makes the dumped schema doc +%% unnecessarily bloated. +%% +%% zone schema is documented here since 5.7: +%% https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html +zones_field_schema() -> + sc( + map(name, ref("zone")), + #{ + desc => ?DESC(zones), + importance => ?IMPORTANCE_HIDDEN + } + ). + desc("persistent_session_store") -> "Settings for message persistence."; desc("persistent_session_builtin") -> diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index b7c212dfd..33e659622 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1198,11 +1198,18 @@ fields_mqtt_quic_listener_max_ack_delay_ms.desc: fields_mqtt_quic_listener_max_ack_delay_ms.label: """Max ack delay ms""" -base_listener_zone.desc: -"""The configuration zone to which the listener belongs.""" +base_listener_zone.desc: """~ + The configuration zone to which the listener belongs. + Clients connected to this listener will inherit zone-settings created under this zone name. -base_listener_zone.label: -"""Zone""" + A zone can override the configs under below root names: + - `mqtt` + - `force_shutdown` + - `force_gc` + - `flapping_detect` + - `session_persistence`""" + +base_listener_zone.label: "Zone" fields_mqtt_quic_listener_handshake_idle_timeout.desc: """How long a handshake can idle before it is discarded.""" From 154eb18657d1a5718a69506368be641c5c3d031c Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 29 Apr 2024 15:45:32 +0200 Subject: [PATCH 67/86] chore: pin mimerl 1.2.0 for some reason, mix pulls 1.3.0 (the lastest from hex.pm) --- mix.exs | 3 ++- rebar.config | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 2cc44099a..4af251882 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,8 @@ defmodule EMQXUmbrella.MixProject do {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, - {:ra, "2.7.3", override: true} + {:ra, "2.7.3", override: true}, + {:mimerl, "1.2.0", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index cd70493e5..0bdefd998 100644 --- a/rebar.config +++ b/rebar.config @@ -111,7 +111,8 @@ {ssl_verify_fun, "1.1.7"}, {rfc3339, {git, "https://github.com/emqx/rfc3339.git", {tag, "0.2.3"}}}, {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.2"}}}, - {ra, "2.7.3"} + {ra, "2.7.3"}, + {mimerl, "1.2.0"} ]}. {xref_ignores, From 79526d539a7ed31f5b66f3bc9caca2a6ec3a42b5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Apr 2024 09:57:35 -0300 Subject: [PATCH 68/86] fix(resource manager): clean up any running health checks when terminating Fixes https://github.com/emqx/emqx/pull/12812#discussion_r1555564254 --- .../src/emqx_resource_manager.erl | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6d9ad50e4..d2f527883 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -446,6 +446,7 @@ init({DataIn, Opts}) -> terminate({shutdown, removed}, _State, _Data) -> ok; terminate(_Reason, _State, Data) -> + ok = terminate_health_check_workers(Data), _ = maybe_stop_resource(Data), _ = erase_cache(Data), ok. @@ -634,6 +635,7 @@ health_check_actions(Data) -> handle_remove_event(From, ClearMetrics, Data) -> %% stop the buffer workers first, brutal_kill, so it should be fast ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), + ok = terminate_health_check_workers(Data), %% now stop the resource, this can be slow _ = stop_resource(Data), case ClearMetrics of @@ -793,6 +795,35 @@ safe_call_remove_channel(_ResId, _Mod, undefined = State, _ChannelID) -> safe_call_remove_channel(ResId, Mod, State, ChannelID) -> emqx_resource:call_remove_channel(ResId, Mod, State, ChannelID). +%% For cases where we need to terminate and there are running health checks. +terminate_health_check_workers(Data) -> + #data{ + hc_workers = #{resource := RHCWorkers, channel := CHCWorkers}, + hc_pending_callers = #{resource := RPending, channel := CPending} + } = Data, + maps:foreach( + fun({Pid, _Ref}, _) -> + exit(Pid, kill) + end, + RHCWorkers + ), + maps:foreach( + fun + ({Pid, _Ref}, _) when is_pid(Pid) -> + exit(Pid, kill); + (_, _) -> + ok + end, + CHCWorkers + ), + Pending = lists:flatten([RPending, maps:values(CPending)]), + lists:foreach( + fun(From) -> + gen_statem:reply(From, {error, resource_shutting_down}) + end, + Pending + ). + make_test_id() -> RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. From 475077c7981edcbb2e8986515f9bf62ba1572496 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Apr 2024 15:11:10 -0300 Subject: [PATCH 69/86] fix(resource): account for ongoing channel health checks, update data and reply immediately when receiving an update --- .../src/emqx_resource_manager.erl | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index d2f527883..ff91d99fe 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -81,15 +81,15 @@ hc_workers = #{ resource => #{}, channel => #{ - pending => [], - previous_status => #{} + ongoing => #{}, + pending => [] } } :: #{ resource := #{{pid(), reference()} => true}, channel := #{ {pid(), reference()} => channel_id(), - pending := [channel_id()], - previous_status := #{channel_id() => channel_status_map()} + ongoing := #{channel_id() => channel_status_map()}, + pending := [channel_id()] } }, %% Callers waiting on health check @@ -1039,12 +1039,12 @@ handle_manual_channel_health_check( #data{ added_channels = Channels, hc_pending_callers = #{channel := CPending0} = Pending0, - hc_workers = #{channel := #{previous_status := PreviousStatus}} + hc_workers = #{channel := #{ongoing := Ongoing}} } = Data0, ChannelId ) when is_map_key(ChannelId, Channels), - is_map_key(ChannelId, PreviousStatus) + is_map_key(ChannelId, Ongoing) -> %% Ongoing health check. CPending = maps:update_with( @@ -1189,53 +1189,54 @@ resource_not_connected_channel_error_msg(ResourceStatus, ChannelId, Data1) -> %% `?status_connected'. -spec trigger_health_check_for_added_channels(data()) -> data(). trigger_health_check_for_added_channels(Data0 = #data{hc_workers = HCWorkers0}) -> - #{channel := CHCWorkers0} = HCWorkers0, - PreviousStatus = maps:from_list([ - {ChannelId, OldStatus} - || {ChannelId, OldStatus} <- maps:to_list(Data0#data.added_channels), - channel_status_is_channel_added(OldStatus) - ]), - ChannelsToCheck = maps:keys(PreviousStatus), + #{ + channel := CHCWorkers0 = + #{ + pending := CPending0, + ongoing := Ongoing0 + } + } = HCWorkers0, + NewOngoing = maps:filter( + fun(ChannelId, OldStatus) -> + not is_map_key(ChannelId, Ongoing0) and + channel_status_is_channel_added(OldStatus) + end, + Data0#data.added_channels + ), + ChannelsToCheck = maps:keys(NewOngoing), case ChannelsToCheck of [] -> %% Nothing to do. Data0; [ChannelId | Rest] -> %% Shooting one check at a time. We could increase concurrency in the future. - CHCWorkers = CHCWorkers0#{pending := Rest, previous_status := PreviousStatus}, + CHCWorkers = CHCWorkers0#{ + pending := CPending0 ++ Rest, + ongoing := maps:merge(Ongoing0, NewOngoing) + }, Data1 = Data0#data{hc_workers = HCWorkers0#{channel := CHCWorkers}}, start_channel_health_check(Data1, ChannelId) end. --spec continue_channel_health_check_connected(data()) -> data(). -continue_channel_health_check_connected(Data0) -> +-spec continue_channel_health_check_connected(channel_id(), channel_status_map(), data()) -> data(). +continue_channel_health_check_connected(ChannelId, OldStatus, Data0) -> #data{hc_workers = HCWorkers0} = Data0, - #{channel := #{previous_status := PreviousStatus} = CHCWorkers0} = HCWorkers0, - CHCWorkers = CHCWorkers0#{previous_status := #{}}, + #{channel := CHCWorkers0} = HCWorkers0, + CHCWorkers = emqx_utils_maps:deep_remove([ongoing, ChannelId], CHCWorkers0), Data1 = Data0#data{hc_workers = HCWorkers0#{channel := CHCWorkers}}, %% Remove the added channels with a a status different from connected or connecting - CheckedChannels = [ - {ChannelId, NewStatus} - || {ChannelId, NewStatus} <- maps:to_list(Data0#data.added_channels), - is_map_key(ChannelId, PreviousStatus) - ], - ChannelsToRemove = [ - ChannelId - || {ChannelId, NewStatus} <- CheckedChannels, - not channel_status_is_channel_added(NewStatus) - ], + NewStatus = maps:get(ChannelId, Data0#data.added_channels), + ChannelsToRemove = [ChannelId || not channel_status_is_channel_added(NewStatus)], Data = remove_channels_in_list(ChannelsToRemove, Data1, true), %% Raise/clear alarms - lists:foreach( - fun - ({ID, #{status := ?status_connected}}) -> - _ = maybe_clear_alarm(ID); - ({ID, NewStatus}) -> - OldStatus = maps:get(ID, PreviousStatus), - _ = maybe_alarm(NewStatus, ID, NewStatus, OldStatus) - end, - CheckedChannels - ), + case NewStatus of + #{status := ?status_connected} -> + _ = maybe_clear_alarm(ChannelId), + ok; + _ -> + _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus), + ok + end, Data. -spec start_channel_health_check(data(), channel_id()) -> data(). @@ -1271,19 +1272,24 @@ handle_channel_health_check_worker_down(Data0, WorkerRef, ExitResult) -> %% `emqx_resource:call_channel_health_check' catches all exceptions. AddedChannels = maps:put(ChannelId, NewStatus, AddedChannels0) end, + #{ongoing := Ongoing0} = CHCWorkers1, + {PreviousChanStatus, Ongoing1} = maps:take(ChannelId, Ongoing0), + CHCWorkers2 = CHCWorkers1#{ongoing := Ongoing1}, + CHCWorkers3 = emqx_utils_maps:deep_remove([ongoing, ChannelId], CHCWorkers2), Data1 = Data0#data{added_channels = AddedChannels}, {Replies, Data2} = reply_pending_channel_health_check_callers(ChannelId, NewStatus, Data1), case CHCWorkers1 of #{pending := [NextChannelId | Rest]} -> - CHCWorkers = CHCWorkers1#{pending := Rest}, + CHCWorkers = CHCWorkers3#{pending := Rest}, HCWorkers = HCWorkers0#{channel := CHCWorkers}, Data3 = Data2#data{hc_workers = HCWorkers}, - Data = start_channel_health_check(Data3, NextChannelId), + Data4 = continue_channel_health_check_connected(ChannelId, PreviousChanStatus, Data3), + Data = start_channel_health_check(Data4, NextChannelId), {keep_state, update_state(Data, Data0), Replies}; #{pending := []} -> - HCWorkers = HCWorkers0#{channel := CHCWorkers1}, + HCWorkers = HCWorkers0#{channel := CHCWorkers3}, Data3 = Data2#data{hc_workers = HCWorkers}, - Data = continue_channel_health_check_connected(Data3), + Data = continue_channel_health_check_connected(ChannelId, PreviousChanStatus, Data3), {keep_state, update_state(Data, Data0), Replies} end. @@ -1339,7 +1345,7 @@ remove_runtime_data(#data{} = Data0) -> Data0#data{ hc_workers = #{ resource => #{}, - channel => #{pending => [], previous_status => #{}} + channel => #{pending => [], ongoing => #{}} }, hc_pending_callers = #{ resource => [], From 5cf92dcb73108b5ce1a967eec410b432692763ea Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Apr 2024 13:26:49 -0300 Subject: [PATCH 70/86] refactor: use `spawn_link` instead of `spawn_monitor` This should cover the case when the resource manager is brutally killed. --- .../src/emqx_resource_manager.erl | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index ff91d99fe..1c0c3f4ca 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -556,22 +556,22 @@ handle_event( {keep_state_and_data, {reply, From, {ok, Channels}}}; handle_event( info, - {'DOWN', Ref, process, Pid, Res}, + {'EXIT', Pid, Res}, State0, Data0 = #data{hc_workers = #{resource := RHCWorkers}} ) when - is_map_key({Pid, Ref}, RHCWorkers) + is_map_key(Pid, RHCWorkers) -> - handle_resource_health_check_worker_down(State0, Data0, {Pid, Ref}, Res); + handle_resource_health_check_worker_down(State0, Data0, Pid, Res); handle_event( info, - {'DOWN', Ref, process, Pid, Res}, + {'EXIT', Pid, Res}, _State, Data0 = #data{hc_workers = #{channel := CHCWorkers}} ) when - is_map_key({Pid, Ref}, CHCWorkers) + is_map_key(Pid, CHCWorkers) -> - handle_channel_health_check_worker_down(Data0, {Pid, Ref}, Res); + handle_channel_health_check_worker_down(Data0, Pid, Res); % Ignore all other events handle_event(EventType, EventData, State, Data) -> ?SLOG( @@ -802,14 +802,14 @@ terminate_health_check_workers(Data) -> hc_pending_callers = #{resource := RPending, channel := CPending} } = Data, maps:foreach( - fun({Pid, _Ref}, _) -> + fun(Pid, _) -> exit(Pid, kill) end, RHCWorkers ), maps:foreach( fun - ({Pid, _Ref}, _) when is_pid(Pid) -> + (Pid, _) when is_pid(Pid) -> exit(Pid, kill); (_, _) -> ok @@ -947,14 +947,14 @@ start_resource_health_check(#data{hc_workers = #{resource := HCWorkers}}) when keep_state_and_data; start_resource_health_check(#data{} = Data0) -> #data{hc_workers = HCWorkers0 = #{resource := RHCWorkers0}} = Data0, - WorkerRef = {_Pid, _Ref} = spawn_resource_health_check_worker(Data0), - HCWorkers = HCWorkers0#{resource := RHCWorkers0#{WorkerRef => true}}, + WorkerPid = spawn_resource_health_check_worker(Data0), + HCWorkers = HCWorkers0#{resource := RHCWorkers0#{WorkerPid => true}}, Data = Data0#data{hc_workers = HCWorkers}, {keep_state, Data}. --spec spawn_resource_health_check_worker(data()) -> {pid(), reference()}. +-spec spawn_resource_health_check_worker(data()) -> pid(). spawn_resource_health_check_worker(#data{} = Data) -> - spawn_monitor(?MODULE, worker_resource_health_check, [Data]). + spawn_link(?MODULE, worker_resource_health_check, [Data]). %% separated so it can be spec'ed and placate dialyzer tantrums... -spec worker_resource_health_check(data()) -> no_return(). @@ -1242,13 +1242,13 @@ continue_channel_health_check_connected(ChannelId, OldStatus, Data0) -> -spec start_channel_health_check(data(), channel_id()) -> data(). start_channel_health_check(#data{} = Data0, ChannelId) -> #data{hc_workers = HCWorkers0 = #{channel := CHCWorkers0}} = Data0, - WorkerRef = {_Pid, _Ref} = spawn_channel_health_check_worker(Data0, ChannelId), - HCWorkers = HCWorkers0#{channel := CHCWorkers0#{WorkerRef => ChannelId}}, + WorkerPid = spawn_channel_health_check_worker(Data0, ChannelId), + HCWorkers = HCWorkers0#{channel := CHCWorkers0#{WorkerPid => ChannelId}}, Data0#data{hc_workers = HCWorkers}. --spec spawn_channel_health_check_worker(data(), channel_id()) -> {pid(), reference()}. +-spec spawn_channel_health_check_worker(data(), channel_id()) -> pid(). spawn_channel_health_check_worker(#data{} = Data, ChannelId) -> - spawn_monitor(?MODULE, worker_channel_health_check, [Data, ChannelId]). + spawn_link(?MODULE, worker_channel_health_check, [Data, ChannelId]). %% separated so it can be spec'ed and placate dialyzer tantrums... -spec worker_channel_health_check(data(), channel_id()) -> no_return(). From a84738915947f1c7185265821c81427f9254bd37 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Apr 2024 13:19:23 -0300 Subject: [PATCH 71/86] fix(http connector): deobfuscate sensitive headers Fixes https://emqx.atlassian.net/browse/EMQX-12213 --- .../test/emqx_bridge_v2_testlib.erl | 13 +++ .../test/emqx_bridge_http_v2_SUITE.erl | 96 ++++++++++++++++++- apps/emqx_utils/src/emqx_utils_redact.erl | 16 +++- changes/ce/fix-12948.en.md | 1 + 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-12948.en.md diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index a3316e39d..b057a648e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -173,6 +173,11 @@ source_hookpoint(Config) -> BridgeId = emqx_bridge_resource:bridge_id(Type, Name), emqx_bridge_v2:source_hookpoint(BridgeId). +action_hookpoint(Config) -> + #{kind := action, type := Type, name := Name} = get_common_values(Config), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + emqx_bridge_resource:bridge_hookpoint(BridgeId). + add_source_hookpoint(Config) -> Hookpoint = source_hookpoint(Config), ok = emqx_hooks:add(Hookpoint, {?MODULE, source_hookpoint_callback, [self()]}, 1000), @@ -378,6 +383,14 @@ start_connector_api(ConnectorName, ConnectorType) -> ct:pal("connector update (http) result:\n ~p", [Res]), Res. +get_connector_api(ConnectorType, ConnectorName) -> + ConnectorId = emqx_connector_resource:connector_id(ConnectorType, ConnectorName), + Path = emqx_mgmt_api_test_util:api_path(["connectors", ConnectorId]), + ct:pal("get connector ~s (http)", [ConnectorId]), + Res = request(get, Path, _Params = []), + ct:pal("get connector (http) result:\n ~p", [Res]), + Res. + create_action_api(Config) -> create_action_api(Config, _Overrides = #{}). diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl index f9ae0c7e0..3b73abfd9 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl @@ -69,10 +69,21 @@ end_per_suite(Config) -> suite() -> [{timetrap, {seconds, 60}}]. +init_per_testcase(t_update_with_sensitive_data, Config) -> + HTTPPath = <<"/foo/bar">>, + ServerSSLOpts = false, + {ok, {HTTPPort, _Pid}} = emqx_bridge_http_connector_test_server:start_link( + _Port = random, HTTPPath, ServerSSLOpts + ), + ok = emqx_bridge_http_connector_test_server:set_handler(success_handler()), + [{path, HTTPPath}, {http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; init_per_testcase(_TestCase, Config) -> Server = start_http_server(#{response_delay_ms => 0}), [{http_server, Server} | Config]. +end_per_testcase(t_update_with_sensitive_data, Config) -> + ok = emqx_bridge_http_connector_test_server:stop(), + end_per_testcase(common, proplists:delete(http_server, Config)); end_per_testcase(_TestCase, Config) -> case ?config(http_server, Config) of undefined -> ok; @@ -112,6 +123,69 @@ t_compose_connector_url_and_action_path(Config) -> ), ok. +%% Checks that we can successfully update a connector containing sensitive headers and +%% they won't be clobbered by the update. +t_update_with_sensitive_data(Config) -> + ?check_trace( + begin + ConnectorCfg0 = make_connector_config(Config), + AuthHeader = <<"Bearer some_token">>, + ConnectorCfg1 = emqx_utils_maps:deep_merge( + ConnectorCfg0, + #{<<"headers">> => #{<<"authorization">> => AuthHeader}} + ), + ActionCfg = make_action_config(Config), + CreateConfig = [ + {bridge_kind, action}, + {action_type, ?BRIDGE_TYPE}, + {action_name, ?BRIDGE_NAME}, + {action_config, ActionCfg}, + {connector_type, ?BRIDGE_TYPE}, + {connector_name, ?CONNECTOR_NAME}, + {connector_config, ConnectorCfg1} + ], + {ok, {{_, 201, _}, _, #{<<"headers">> := #{<<"authorization">> := Obfuscated}}}} = + emqx_bridge_v2_testlib:create_connector_api(CreateConfig), + {ok, _} = + emqx_bridge_v2_testlib:create_kind_api(CreateConfig), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + {ok, _} = emqx_bridge_v2_testlib:create_rule_api( + #{ + sql => <<"select * from \"t/http\" ">>, + actions => [BridgeId] + } + ), + emqx:publish(emqx_message:make(<<"t/http">>, <<"1">>)), + ?assertReceive({http, #{<<"authorization">> := AuthHeader}, _}), + + %% Now update the connector and see if the header stays deobfuscated. We send the old + %% auth header as an obfuscated value to simulate the behavior of the frontend. + ConnectorCfg2 = emqx_utils_maps:deep_merge( + ConnectorCfg1, + #{ + <<"headers">> => #{ + <<"authorization">> => Obfuscated, + <<"other_header">> => <<"new">> + } + } + ), + {ok, _} = emqx_bridge_v2_testlib:update_connector_api( + ?CONNECTOR_NAME, + ?BRIDGE_TYPE, + ConnectorCfg2 + ), + + emqx:publish(emqx_message:make(<<"t/http">>, <<"2">>)), + %% Should not be obfuscated. + ?assertReceive({http, #{<<"authorization">> := AuthHeader}, _}, 2_000), + + ok + end, + [] + ), + + ok. + %%-------------------------------------------------------------------- %% helpers %%-------------------------------------------------------------------- @@ -123,7 +197,10 @@ make_connector_config(Config) -> <<"url">> => iolist_to_binary(io_lib:format("http://localhost:~p", [Port])), <<"headers">> => #{}, <<"pool_type">> => <<"hash">>, - <<"pool_size">> => 1 + <<"pool_size">> => 1, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"100ms">> + } }. make_action_config(Config) -> @@ -136,5 +213,22 @@ make_action_config(Config) -> <<"method">> => <<"post">>, <<"headers">> => #{}, <<"body">> => <<"${.}">> + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"100ms">> } }. + +success_handler() -> + TestPid = self(), + fun(Req0, State) -> + {ok, Body, Req} = cowboy_req:read_body(Req0), + TestPid ! {http, cowboy_req:headers(Req), Body}, + Rep = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + <<"{}">>, + Req + ), + {ok, Rep, State} + end. diff --git a/apps/emqx_utils/src/emqx_utils_redact.erl b/apps/emqx_utils/src/emqx_utils_redact.erl index c830048a9..cdff70282 100644 --- a/apps/emqx_utils/src/emqx_utils_redact.erl +++ b/apps/emqx_utils/src/emqx_utils_redact.erl @@ -152,19 +152,24 @@ redact_v(_V) -> ?REDACT_VAL. deobfuscate(NewConf, OldConf) -> + deobfuscate(NewConf, OldConf, fun(_) -> false end). + +deobfuscate(NewConf, OldConf, IsSensitiveFun) -> maps:fold( fun(K, V, Acc) -> case maps:find(K, OldConf) of error -> - case is_redacted(K, V) of + case is_redacted(K, V, IsSensitiveFun) of %% don't put redacted value into new config true -> Acc; false -> Acc#{K => V} end; + {ok, OldV} when is_map(V), is_map(OldV), ?IS_KEY_HEADERS(K) -> + Acc#{K => deobfuscate(V, OldV, fun check_is_sensitive_header/1)}; {ok, OldV} when is_map(V), is_map(OldV) -> - Acc#{K => deobfuscate(V, OldV)}; + Acc#{K => deobfuscate(V, OldV, IsSensitiveFun)}; {ok, OldV} -> - case is_redacted(K, V) of + case is_redacted(K, V, IsSensitiveFun) of true -> Acc#{K => OldV}; _ -> @@ -280,6 +285,11 @@ deobfuscate_test() -> %% Don't have password before and should allow put non-redact-val into new config NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), + + HeaderConf1 = #{<<"headers">> => #{<<"Authorization">> => <<"Bearer token">>}}, + HeaderConf1Obs = #{<<"headers">> => #{<<"Authorization">> => ?REDACT_VAL}}, + ?assertEqual(HeaderConf1, deobfuscate(HeaderConf1Obs, HeaderConf1)), + ok. redact_header_test_() -> diff --git a/changes/ce/fix-12948.en.md b/changes/ce/fix-12948.en.md new file mode 100644 index 000000000..71ea3004a --- /dev/null +++ b/changes/ce/fix-12948.en.md @@ -0,0 +1 @@ +Fixed an issue where sensitive HTTP header values like `Authorization` would be substituted by `******` after updating a connector. From ffedce014f92956e552bb240ec40d7c376ff288c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Apr 2024 14:50:18 -0300 Subject: [PATCH 72/86] fix(message validation): validate duplicated topics Fixes https://emqx.atlassian.net/browse/EMQX-12254 --- .../src/emqx_message_validation_schema.erl | 21 +++++++ .../test/emqx_message_validation_tests.erl | 59 +++++++++++++++++++ changes/ee/fix-12950.en.md | 1 + 3 files changed, 81 insertions(+) create mode 100644 changes/ee/fix-12950.en.md diff --git a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl index 9a27915d8..f151cdf62 100644 --- a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl +++ b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl @@ -65,6 +65,7 @@ fields(validation) -> #{ desc => ?DESC("topics"), converter => fun ensure_array/2, + validator => fun validate_unique_topics/1, required => true } )}, @@ -269,3 +270,23 @@ do_validate_unique_schema_checks( end; do_validate_unique_schema_checks([_Check | Rest], Seen, Duplicated) -> do_validate_unique_schema_checks(Rest, Seen, Duplicated). + +validate_unique_topics(Topics) -> + Grouped = maps:groups_from_list( + fun(T) -> T end, + Topics + ), + DuplicatedMap = maps:filter( + fun(_T, Ts) -> length(Ts) > 1 end, + Grouped + ), + case maps:keys(DuplicatedMap) of + [] -> + ok; + Duplicated -> + Msg = iolist_to_binary([ + <<"duplicated topics: ">>, + lists:join(", ", Duplicated) + ]), + {error, Msg} + end. diff --git a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl index c344f6202..7c3dfc9d8 100644 --- a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl +++ b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl @@ -232,6 +232,65 @@ check_test_() -> duplicated_check_test_() -> [ + {"duplicated topics 1", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{<<"topics">> => [<<"t/1">>, <<"t/1">>]} + ) + ]) + )}, + {"duplicated topics 2", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{<<"topics">> => [<<"t/1">>, <<"t/#">>, <<"t/1">>]} + ) + ]) + )}, + {"duplicated topics 3", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1, t/2">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{ + <<"topics">> => [ + <<"t/1">>, + <<"t/#">>, + <<"t/1">>, + <<"t/2">>, + <<"t/2">> + ] + } + ) + ]) + )}, {"duplicated sql checks are not checked", ?_assertMatch( [#{<<"checks">> := [_, _]}], diff --git a/changes/ee/fix-12950.en.md b/changes/ee/fix-12950.en.md new file mode 100644 index 000000000..595833c49 --- /dev/null +++ b/changes/ee/fix-12950.en.md @@ -0,0 +1 @@ +Added a validation to prevent duplicated topics when configuring a message validation. From 2972d2df7ca9e9d67db2e227382a0843b724b773 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Apr 2024 15:25:11 -0300 Subject: [PATCH 73/86] docs: update supported otp versions --- README-CN.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-CN.md b/README-CN.md index c2c5e80f4..c6efab682 100644 --- a/README-CN.md +++ b/README-CN.md @@ -86,7 +86,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl `master` 分支是最新的 5 版本，`main-v4.4` 是 4.4 版本。 -EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 24 和 25 构建。 +EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 25 和 26 构建。 ```bash git clone https://github.com/emqx/emqx.git diff --git a/README.md b/README.md index ad710b5e6..98f78c297 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The `master` branch tracks the latest version 5. For version 4.4 checkout the `m EMQX 4.4 requires OTP 24. EMQX 5.0 ~ 5.3 can be built with OTP 24 or 25. -EMQX 5.4 and newer can be built with OTP 24 or 25. +EMQX 5.4 and newer can be built with OTP 25 or 26. ```bash git clone https://github.com/emqx/emqx.git From 9917293fd022202ae4560f3c98d2c032c539d447 Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 29 Apr 2024 20:50:37 +0200 Subject: [PATCH 74/86] fix(schema): description should be in binary() type --- apps/emqx/src/emqx_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index b7ec41011..b4ba08bbc 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1876,7 +1876,7 @@ base_listener(Bind) -> )} ] ++ emqx_limiter_schema:short_paths_fields(). -%% @hidden Starting from 5.7, listenrs.{TYPE}.{NAME}.zone is no longer hidden +%% @hidden Starting from 5.7, listeners.{TYPE}.{NAME}.zone is no longer hidden %% However, the root key 'zones' is still hidden because the fields' schema %% just repeat other root field's schema, which makes the dumped schema doc %% unnecessarily bloated. @@ -3721,7 +3721,7 @@ default_mem_check_interval() -> description_schema() -> sc( - string(), + binary(), #{ default => <<"">>, desc => ?DESC(description), From ad91ca4401bb134b10a90a06e7acdd987eb4b0e8 Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 29 Apr 2024 22:01:32 +0200 Subject: [PATCH 75/86] chore: upgrade to rulesql 0.2.1 from 0.2.0 a minor enhancement: rule SQL without WHERE or FROM will return "Missing FROM or WHERE" instead of "syntax error before {SELECT" --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 4af251882..d77bfc99c 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,7 @@ defmodule EMQXUmbrella.MixProject do # maybe forbid to fetch quicer {:emqtt, github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()}, - {:rulesql, github: "emqx/rulesql", tag: "0.2.0"}, + {:rulesql, github: "emqx/rulesql", tag: "0.2.1"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, {:telemetry, "1.1.0"}, diff --git a/rebar.config b/rebar.config index 0bdefd998..a0488b509 100644 --- a/rebar.config +++ b/rebar.config @@ -91,7 +91,7 @@ {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.1"}}}, - {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.2.0"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.2.1"}}}, % NOTE: depends on recon 2.5.x {observer_cli, "1.7.1"}, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, From c8d6976b141bc30dac927872fea7c02b988497b0 Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 16 Apr 2024 14:00:06 +0200 Subject: [PATCH 76/86] feat: add conditions to variform expressions - refactored `coalesce` function to allow lazy evaluation - added `iif(Cond, IfExpr, EleseExpr)` to allow simple conditions --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 6 -- apps/emqx_utils/src/emqx_variform.erl | 69 +++++++++++++---- apps/emqx_utils/src/emqx_variform_bif.erl | 77 ++++++++++++++----- apps/emqx_utils/test/emqx_variform_tests.erl | 76 +++++++++++++++++- 4 files changed, 186 insertions(+), 42 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 4e28efb5f..604f43d82 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -132,8 +132,6 @@ %% String Funcs -export([ - coalesce/1, - coalesce/2, lower/1, ltrim/1, reverse/1, @@ -759,10 +757,6 @@ is_array(_) -> false. %% String Funcs %%------------------------------------------------------------------------------ -coalesce(List) -> emqx_variform_bif:coalesce(List). - -coalesce(A, B) -> emqx_variform_bif:coalesce(A, B). - lower(S) -> emqx_variform_bif:lower(S). ltrim(S) -> emqx_variform_bif:ltrim(S). diff --git a/apps/emqx_utils/src/emqx_variform.erl b/apps/emqx_utils/src/emqx_variform.erl index 09a673851..97096559d 100644 --- a/apps/emqx_utils/src/emqx_variform.erl +++ b/apps/emqx_utils/src/emqx_variform.erl @@ -42,14 +42,7 @@ M =:= maps) ). --define(COALESCE_BADARG, - throw(#{ - reason => coalesce_badarg, - explain => - "must be an array, or a call to a function which returns an array, " - "for example: coalesce([a,b,c]) or coalesce(tokens(var,','))" - }) -). +-define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)). %% @doc Render a variform expression with bindings. %% A variform expression is a template string which supports variable substitution @@ -99,6 +92,7 @@ eval_as_string(Expr, Bindings, _Opts) -> return_str(Str) when is_binary(Str) -> Str; return_str(Num) when is_integer(Num) -> integer_to_binary(Num); return_str(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]); +return_str(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); return_str(Other) -> throw(#{ reason => bad_return, @@ -133,6 +127,10 @@ decompile(#{expr := Expression}) -> decompile(Expression) -> Expression. +eval(Atom, _Bindings, _Opts) when is_atom(Atom) -> + %% There is no atom literals in variform, + %% but some bif functions such as regex_match may return an atom. + atom_to_binary(Atom, utf8); eval({str, Str}, _Bindings, _Opts) -> unicode:characters_to_binary(Str); eval({integer, Num}, _Bindings, _Opts) -> @@ -145,6 +143,8 @@ eval({call, FuncNameStr, Args}, Bindings, Opts) -> {Mod, Fun} = resolve_func_name(FuncNameStr), ok = assert_func_exported(Mod, Fun, length(Args)), case {Mod, Fun} of + {?BIF_MOD, iif} -> + eval_iif(Args, Bindings, Opts); {?BIF_MOD, coalesce} -> eval_coalesce(Args, Bindings, Opts); _ -> @@ -158,19 +158,41 @@ eval_loop([H | T], Bindings, Opts) -> [eval(H, Bindings, Opts) | eval_loop(T, Bi %% coalesce treats var_unbound exception as empty string '' eval_coalesce([{array, Args}], Bindings, Opts) -> - NewArgs = [lists:map(fun(Arg) -> try_eval(Arg, Bindings, Opts) end, Args)], - call(?BIF_MOD, coalesce, NewArgs); + %% The input arg is an array + eval_coalesce_loop(Args, Bindings, Opts); eval_coalesce([Arg], Bindings, Opts) -> + %% Arg is an expression (which is expected to return an array) case try_eval(Arg, Bindings, Opts) of List when is_list(List) -> - call(?BIF_MOD, coalesce, List); + case lists:dropwhile(fun(I) -> ?IS_EMPTY(I) end, List) of + [] -> + <<>>; + [H | _] -> + H + end; <<>> -> <<>>; _ -> - ?COALESCE_BADARG + throw(#{ + reason => coalesce_badarg, + explain => "the arg expression did not yield an array" + }) end; -eval_coalesce(_Args, _Bindings, _Opts) -> - ?COALESCE_BADARG. +eval_coalesce(Args, Bindings, Opts) -> + %% It also accepts arbitrary number of args + %% equivalent to [{array, Args}] + eval_coalesce_loop(Args, Bindings, Opts). + +eval_coalesce_loop([], _Bindings, _Opts) -> + <<>>; +eval_coalesce_loop([Arg | Args], Bindings, Opts) -> + Result = try_eval(Arg, Bindings, Opts), + case ?IS_EMPTY(Result) of + true -> + eval_coalesce_loop(Args, Bindings, Opts); + false -> + Result + end. try_eval(Arg, Bindings, Opts) -> try @@ -180,6 +202,21 @@ try_eval(Arg, Bindings, Opts) -> <<>> end. +eval_iif([Cond, If, Else], Bindings, Opts) -> + CondVal = try_eval(Cond, Bindings, Opts), + case is_iif_condition_met(CondVal) of + true -> + eval(If, Bindings, Opts); + false -> + eval(Else, Bindings, Opts) + end. + +%% If iif condition expression yielded boolean, use the boolean value. +%% otherwise it's met as long as it's not an empty string. +is_iif_condition_met(true) -> true; +is_iif_condition_met(false) -> false; +is_iif_condition_met(V) -> not ?IS_EMPTY(V). + %% Some functions accept arbitrary number of arguments but implemented as /1. call(Mod, Fun, Args) -> erlang:apply(Mod, Fun, Args). @@ -237,6 +274,10 @@ resolve_var_value(VarName, Bindings, _Opts) -> }) end. +assert_func_exported(?BIF_MOD, coalesce, _Arity) -> + ok; +assert_func_exported(?BIF_MOD, iif, _Arity) -> + ok; assert_func_exported(Mod, Fun, Arity) -> ok = try_load(Mod), case erlang:function_exported(Mod, Fun, Arity) of diff --git a/apps/emqx_utils/src/emqx_variform_bif.erl b/apps/emqx_utils/src/emqx_variform_bif.erl index 5c598efbd..91bd4f9cf 100644 --- a/apps/emqx_utils/src/emqx_variform_bif.erl +++ b/apps/emqx_utils/src/emqx_variform_bif.erl @@ -58,9 +58,6 @@ %% Array functions -export([nth/2]). -%% Control functions --export([coalesce/1, coalesce/2]). - %% Random functions -export([rand_str/1, rand_int/1]). @@ -76,26 +73,16 @@ %% Hash functions -export([hash/2, hash_to_range/3, map_to_range/3]). --define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)). +%% String compare functions +-export([str_comp/2, str_eq/2, str_lt/2, str_lte/2, str_gt/2, str_gte/2]). + +%% Number compare functions +-export([num_comp/2, num_eq/2, num_lt/2, num_lte/2, num_gt/2, num_gte/2]). %%------------------------------------------------------------------------------ %% String Funcs %%------------------------------------------------------------------------------ -%% @doc Return the first non-empty string -coalesce(A, B) when ?IS_EMPTY(A) andalso ?IS_EMPTY(B) -> - <<>>; -coalesce(A, B) when ?IS_EMPTY(A) -> - B; -coalesce(A, _B) -> - A. - -%% @doc Return the first non-empty string -coalesce([]) -> - <<>>; -coalesce([H | T]) -> - coalesce(H, coalesce(T)). - lower(S) when is_binary(S) -> string:lowercase(S). @@ -523,3 +510,57 @@ map_to_range(Int, Min, Max) when Min + (Int rem Range); map_to_range(_, _, _) -> throw(#{reason => badarg, function => ?FUNCTION_NAME}). + +compare(A, A) -> eq; +compare(A, B) when A < B -> lt; +compare(_A, _B) -> gt. + +%% @doc Compare two strings, returns +%% - 'eq' if they are the same. +%% - 'lt' if arg-1 is ordered before arg-2 +%% - `gt` if arg-1 is ordered after arg-2 +str_comp(A0, B0) -> + A = any_to_str(A0), + B = any_to_str(B0), + compare(A, B). + +%% @doc Return 'true' if two strings are the same, otherwise 'false'. +str_eq(A, B) -> eq =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered before arg-2, otherwise 'false'. +str_lt(A, B) -> lt =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered after arg-2, otherwise 'false'. +str_gt(A, B) -> gt =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is not ordered after arg-2, otherwise 'false'. +str_lte(A, B) -> + R = str_comp(A, B), + R =:= lt orelse R =:= eq. + +%% @doc Return 'true' if arg-1 is not ordered bfore arg-2, otherwise 'false'. +str_gte(A, B) -> + R = str_comp(A, B), + R =:= gt orelse R =:= eq. + +num_comp(A, B) when is_number(A) andalso is_number(B) -> + compare(A, B). + +%% @doc Return 'true' if two numbers are the same, otherwise 'false'. +num_eq(A, B) -> eq =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered before arg-2, otherwise 'false'. +num_lt(A, B) -> lt =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered after arg-2, otherwise 'false'. +num_gt(A, B) -> gt =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is not ordered after arg-2, otherwise 'false'. +num_lte(A, B) -> + R = num_comp(A, B), + R =:= lt orelse R =:= eq. + +%% @doc Return 'true' if arg-1 is not ordered bfore arg-2, otherwise 'false'. +num_gte(A, B) -> + R = num_comp(A, B), + R =:= gt orelse R =:= eq. diff --git a/apps/emqx_utils/test/emqx_variform_tests.erl b/apps/emqx_utils/test/emqx_variform_tests.erl index 5f9a13326..59c39cf1c 100644 --- a/apps/emqx_utils/test/emqx_variform_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_tests.erl @@ -151,6 +151,9 @@ coalesce_test_() -> {"arg from other func", fun() -> ?assertEqual({ok, <<"b">>}, render("coalesce(tokens(a,','))", #{a => <<",,b,c">>})) end}, + {"arg from other func, but no result", fun() -> + ?assertEqual({ok, <<"">>}, render("coalesce(tokens(a,','))", #{a => <<",,,">>})) + end}, {"var unbound", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a)", #{})) end}, {"var unbound in call", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(concat(a))", #{})) @@ -158,18 +161,70 @@ coalesce_test_() -> {"var unbound in calls", fun() -> ?assertEqual({ok, <<"c">>}, render("coalesce([any_to_str(a),any_to_str(b),'c'])", #{})) end}, - {"badarg", fun() -> - ?assertMatch( - {error, #{reason := coalesce_badarg}}, render("coalesce(a,b)", #{a => 1, b => 2}) + {"coalesce n-args", fun() -> + ?assertEqual( + {ok, <<"2">>}, render("coalesce(a,b)", #{a => <<"">>, b => 2}) ) end}, - {"badarg from return", fun() -> + {"coalesce 1-arg", fun() -> ?assertMatch( {error, #{reason := coalesce_badarg}}, render("coalesce(any_to_str(a))", #{a => 1}) ) end} ]. +compare_string_test_() -> + [ + %% Testing str_eq/2 + ?_assertEqual({ok, <<"true">>}, render("str_eq('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_eq('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_eq('', '')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_eq('a', '')", #{})), + + %% Testing str_lt/2 + ?_assertEqual({ok, <<"true">>}, render("str_lt('a', 'b')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('b', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gt('b', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('a', 'b')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_lte('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_lte('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lte('b', 'a')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_lte('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gte('b', 'a')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_gte('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gte('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_gte('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gt(9, 10)", #{})) + ]. + +compare_numbers_test_() -> + [ + ?_assertEqual({ok, <<"true">>}, render("num_eq(1, 1)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_eq(2, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_lt(1, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_lt(2, 2)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_gt(2, 1)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_gt(1, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_lte(1, 1)", #{})), + ?_assertEqual({ok, <<"true">>}, render("num_lte(1, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_lte(2, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_gte(2, -1)", #{})), + ?_assertEqual({ok, <<"true">>}, render("num_gte(2, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_gte(-1, 2)", #{})) + ]. + syntax_error_test_() -> [ {"empty expression", fun() -> ?assertMatch(?SYNTAX_ERROR, render("", #{})) end}, @@ -218,3 +273,16 @@ to_range_badarg_test_() -> ?ASSERT_BADARG(map_to_range, "('a','1',2)"), ?ASSERT_BADARG(map_to_range, "('a',2,1)") ]. + +iif_test_() -> + %% if clientid has to words separated by a -, take the suffix, and append with `/#` + Expr1 = "iif(nth(2,tokens(clientid,'-')),concat([nth(2,tokens(clientid,'-')),'/#']),'')", + [ + ?_assertEqual({ok, <<"yes-A">>}, render("iif(a,'yes-A','no-A')", #{a => <<"x">>})), + ?_assertEqual({ok, <<"no-A">>}, render("iif(a,'yes-A','no-A')", #{})), + ?_assertEqual({ok, <<"2">>}, render("iif(str_eq(a,1),2,3)", #{a => 1})), + ?_assertEqual({ok, <<"3">>}, render("iif(str_eq(a,1),2,3)", #{a => <<"not-1">>})), + ?_assertEqual({ok, <<"3">>}, render("iif(str_eq(a,1),2,3)", #{})), + ?_assertEqual({ok, <<"">>}, render(Expr1, #{clientid => <<"a">>})), + ?_assertEqual({ok, <<"suffix/#">>}, render(Expr1, #{clientid => <<"a-suffix">>})) + ]. From b91ff971702a9dbc99e0672f0b3ad49e9b96552d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 25 Apr 2024 18:21:29 +0200 Subject: [PATCH 77/86] feat(s3): separate streaming upload logic into dedicated module And use it in `emqx_s3_uploader`, while also turning it into a simple gen_server. --- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 2 +- apps/emqx_s3/src/emqx_s3_upload.erl | 217 ++++++++++++++++ apps/emqx_s3/src/emqx_s3_uploader.erl | 300 +++++----------------- 3 files changed, 276 insertions(+), 243 deletions(-) create mode 100644 apps/emqx_s3/src/emqx_s3_upload.erl diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 67f9d33b3..7bb5d87f1 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -53,7 +53,7 @@ emqx_s3_client:bucket(), emqx_s3_client:config(), emqx_s3_client:upload_options(), - emqx_s3_uploader:config() + emqx_s3_upload:config() }. -define(DEFAULT_CALL_TIMEOUT, 5000). diff --git a/apps/emqx_s3/src/emqx_s3_upload.erl b/apps/emqx_s3/src/emqx_s3_upload.erl new file mode 100644 index 000000000..565c6b8bc --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_upload.erl @@ -0,0 +1,217 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_upload). + +-include_lib("emqx/include/types.hrl"). + +-export([ + new/4, + append/2, + write/1, + complete/1, + abort/1 +]). + +-export([format/1]). + +-export_type([t/0, config/0]). + +-type config() :: #{ + min_part_size => pos_integer(), + max_part_size => pos_integer() +}. + +-type t() :: #{ + started := boolean(), + client := emqx_s3_client:client(), + key := emqx_s3_client:key(), + upload_opts := emqx_s3_client:upload_options(), + buffer := iodata(), + buffer_size := non_neg_integer(), + min_part_size := pos_integer(), + max_part_size := pos_integer(), + upload_id := undefined | emqx_s3_client:upload_id(), + etags := [emqx_s3_client:etag()], + part_number := emqx_s3_client:part_number() +}. + +%% 5MB +-define(DEFAULT_MIN_PART_SIZE, 5242880). +%% 5GB +-define(DEFAULT_MAX_PART_SIZE, 5368709120). + +%% + +-spec new( + emqx_s3_client:client(), + emqx_s3_client:key(), + emqx_s3_client:upload_options(), + config() +) -> + t(). +new(Client, Key, UploadOpts, Config) -> + #{ + started => false, + client => Client, + key => Key, + upload_opts => UploadOpts, + buffer => [], + buffer_size => 0, + min_part_size => maps:get(min_part_size, Config, ?DEFAULT_MIN_PART_SIZE), + max_part_size => maps:get(max_part_size, Config, ?DEFAULT_MAX_PART_SIZE), + upload_id => undefined, + etags => [], + part_number => 1 + }. + +-spec append(iodata(), t()) -> {ok, t()} | {error, term()}. +append(WriteData, #{buffer := Buffer, buffer_size := BufferSize} = Upload) -> + case is_valid_part(WriteData, Upload) of + true -> + {ok, Upload#{ + buffer => [Buffer, WriteData], + buffer_size => BufferSize + iolist_size(WriteData) + }}; + false -> + {error, {too_large, iolist_size(WriteData)}} + end. + +-spec write(t()) -> {ok, t()} | {cont, t()} | {error, term()}. +write(U0 = #{started := false}) -> + case maybe_start_upload(U0) of + not_started -> + {ok, U0}; + {started, U1} -> + {cont, U1#{started := true}}; + {error, _} = Error -> + Error + end; +write(U0 = #{started := true}) -> + maybe_upload_part(U0). + +-spec complete(t()) -> {ok, t()} | {error, term()}. +complete( + #{ + started := true, + client := Client, + key := Key, + upload_id := UploadId + } = U0 +) -> + case upload_part(U0) of + {ok, #{etags := ETagsRev} = U1} -> + ETags = lists:reverse(ETagsRev), + case emqx_s3_client:complete_multipart(Client, Key, UploadId, ETags) of + ok -> + {ok, U1}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end; +complete(#{started := false} = Upload) -> + put_object(Upload). + +-spec abort(t()) -> ok_or_error(term()). +abort(#{ + started := true, + client := Client, + key := Key, + upload_id := UploadId +}) -> + case emqx_s3_client:abort_multipart(Client, Key, UploadId) of + ok -> + ok; + {error, _} = Error -> + Error + end; +abort(#{started := false}) -> + ok. + +%%-------------------------------------------------------------------- + +-spec format(t()) -> map(). +format(Upload = #{client := Client}) -> + Upload#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec maybe_start_upload(t()) -> not_started | {started, t()} | {error, term()}. +maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + start_upload(Data); + false -> + not_started + end. + +-spec start_upload(t()) -> {started, t()} | {error, term()}. +start_upload(#{client := Client, key := Key, upload_opts := UploadOpts} = Data) -> + case emqx_s3_client:start_multipart(Client, Key, UploadOpts) of + {ok, UploadId} -> + NewData = Data#{upload_id => UploadId}, + {started, NewData}; + {error, _} = Error -> + Error + end. + +-spec maybe_upload_part(t()) -> ok_or_error(t(), term()). +maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + upload_part(Data); + false -> + {ok, Data} + end. + +-spec upload_part(t()) -> ok_or_error(t(), term()). +upload_part(#{buffer_size := 0} = Upload) -> + {ok, Upload}; +upload_part( + #{ + client := Client, + key := Key, + upload_id := UploadId, + buffer := Buffer, + part_number := PartNumber, + etags := ETags + } = Upload +) -> + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of + {ok, ETag} -> + {ok, Upload#{ + buffer => [], + buffer_size => 0, + part_number => PartNumber + 1, + etags => [{PartNumber, ETag} | ETags] + }}; + {error, _} = Error -> + Error + end. + +-spec put_object(t()) -> ok_or_error(t(), term()). +put_object( + #{ + client := Client, + key := Key, + upload_opts := UploadOpts, + buffer := Buffer + } = Upload +) -> + case emqx_s3_client:put_object(Client, Key, UploadOpts, Buffer) of + ok -> + {ok, Upload}; + {error, _} = Error -> + Error + end. + +is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> + BufferSize + iolist_size(WriteData) =< MaxPartSize. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 160ecbfef..99a89eb92 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -6,7 +6,7 @@ -include_lib("emqx/include/types.hrl"). --behaviour(gen_statem). +-behaviour(gen_server). -export([ start_link/3, @@ -25,46 +25,23 @@ -export([ init/1, - callback_mode/0, - handle_event/4, - terminate/3, - code_change/4, - format_status/1, - format_status/2 + handle_call/3, + handle_cast/2, + terminate/2, + format_status/1 ]). --export_type([config/0]). - --type config() :: #{ - min_part_size => pos_integer(), - max_part_size => pos_integer() -}. - -type data() :: #{ profile_id => emqx_s3:profile_id(), - client := emqx_s3_client:client(), - key := emqx_s3_client:key(), - upload_opts := emqx_s3_client:upload_options(), - buffer := iodata(), - buffer_size := non_neg_integer(), - min_part_size := pos_integer(), - max_part_size := pos_integer(), - upload_id := undefined | emqx_s3_client:upload_id(), - etags := [emqx_s3_client:etag()], - part_number := emqx_s3_client:part_number() + upload := emqx_s3_upload:t() | aborted }. -%% 5MB --define(DEFAULT_MIN_PART_SIZE, 5242880). -%% 5GB --define(DEFAULT_MAX_PART_SIZE, 5368709120). - -define(DEFAULT_TIMEOUT, 30000). -spec start_link(emqx_s3:profile_id(), emqx_s3_client:key(), emqx_s3_client:upload_options()) -> - gen_statem:start_ret(). + gen_server:start_ret(). start_link(ProfileId, Key, UploadOpts) when is_list(Key) -> - gen_statem:start_link(?MODULE, {profile, ProfileId, Key, UploadOpts}, []). + gen_server:start_link(?MODULE, {profile, ProfileId, Key, UploadOpts}, []). -spec write(pid(), iodata()) -> ok_or_error(term()). write(Pid, WriteData) -> @@ -72,7 +49,7 @@ write(Pid, WriteData) -> -spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). write(Pid, WriteData, Timeout) -> - gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). + gen_server:call(Pid, {write, wrap(WriteData)}, Timeout). -spec complete(pid()) -> ok_or_error(term()). complete(Pid) -> @@ -80,7 +57,7 @@ complete(Pid) -> -spec complete(pid(), timeout()) -> ok_or_error(term()). complete(Pid, Timeout) -> - gen_statem:call(Pid, complete, Timeout). + gen_server:call(Pid, complete, Timeout). -spec abort(pid()) -> ok_or_error(term()). abort(Pid) -> @@ -88,7 +65,7 @@ abort(Pid) -> -spec abort(pid(), timeout()) -> ok_or_error(term()). abort(Pid, Timeout) -> - gen_statem:call(Pid, abort, Timeout). + gen_server:call(Pid, abort, Timeout). -spec shutdown(pid()) -> ok. shutdown(Pid) -> @@ -99,231 +76,73 @@ shutdown(Pid) -> %% gen_statem callbacks %%-------------------------------------------------------------------- -callback_mode() -> handle_event_function. - init({profile, ProfileId, Key, UploadOpts}) -> + _ = process_flag(trap_exit, true), {Bucket, ClientConfig, BaseOpts, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Upload = #{ - profile_id => ProfileId, - client => client(Bucket, ClientConfig), - key => Key, - upload_opts => maps:merge(BaseOpts, UploadOpts) - }, - init({upload, UploaderConfig, Upload}); -init({upload, Config, Upload}) -> - process_flag(trap_exit, true), - {ok, upload_not_started, Upload#{ - buffer => [], - buffer_size => 0, - min_part_size => maps:get(min_part_size, Config, ?DEFAULT_MIN_PART_SIZE), - max_part_size => maps:get(max_part_size, Config, ?DEFAULT_MAX_PART_SIZE), - upload_id => undefined, - etags => [], - part_number => 1 - }}. + Upload = emqx_s3_upload:new( + client(Bucket, ClientConfig), + Key, + maps:merge(BaseOpts, UploadOpts), + UploaderConfig + ), + {ok, #{profile_id => ProfileId, upload => Upload}}. -handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> +-spec handle_call(_Call, gen_server:from(), data()) -> + {reply, _Result, data()} | {stop, _Reason, _Result, data()}. +handle_call({write, WriteDataWrapped}, _From, St0 = #{upload := U0}) -> WriteData = unwrap(WriteDataWrapped), - case is_valid_part(WriteData, Data0) of - true -> - handle_write(State, From, WriteData, Data0); - false -> - {keep_state_and_data, {reply, From, {error, {too_large, iolist_size(WriteData)}}}} + case emqx_s3_upload:append(WriteData, U0) of + {ok, U1} -> + handle_write(St0#{upload := U1}); + {error, _} = Error -> + {reply, Error, St0} end; -handle_event({call, From}, complete, upload_not_started, Data0) -> - case put_object(Data0) of +handle_call(complete, _From, St0 = #{upload := U0}) -> + case emqx_s3_upload:complete(U0) of + {ok, U1} -> + {stop, normal, ok, St0#{upload := U1}}; + {error, _} = Error -> + {stop, Error, Error, St0} + end; +handle_call(abort, _From, St = #{upload := Upload}) -> + case emqx_s3_upload:abort(Upload) of ok -> - {stop_and_reply, normal, {reply, From, ok}}; + {stop, normal, ok, St}; {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} - end; -handle_event({call, From}, complete, upload_started, Data0) -> - case complete_upload(Data0) of - {ok, Data1} -> - {stop_and_reply, normal, {reply, From, ok}, Data1}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} - end; -handle_event({call, From}, abort, upload_not_started, _Data) -> - {stop_and_reply, normal, {reply, From, ok}}; -handle_event({call, From}, abort, upload_started, Data0) -> - case abort_upload(Data0) of - ok -> - {stop_and_reply, normal, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} + {stop, Error, Error, St} end. -handle_write(upload_not_started, From, WriteData, Data0) -> - Data1 = append_buffer(Data0, WriteData), - case maybe_start_upload(Data1) of - not_started -> - {keep_state, Data1, {reply, From, ok}}; - {started, Data2} -> - case upload_part(Data2) of - {ok, Data3} -> - {next_state, upload_started, Data3, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data2} - end; +handle_write(St = #{upload := U0}) -> + case emqx_s3_upload:write(U0) of + {ok, U1} -> + {reply, ok, St#{upload := U1}}; + {cont, U1} -> + handle_write(St#{upload := U1}); {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data1} - end; -handle_write(upload_started, From, WriteData, Data0) -> - Data1 = append_buffer(Data0, WriteData), - case maybe_upload_part(Data1) of - {ok, Data2} -> - {keep_state, Data2, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data1} + {stop, Error, Error, St} end. -terminate(Reason, _State, #{client := Client, upload_id := UploadId, key := Key}) when - (UploadId =/= undefined) andalso (Reason =/= normal) --> - emqx_s3_client:abort_multipart(Client, Key, UploadId); -terminate(_Reason, _State, _Data) -> - ok. +-spec handle_cast(_Cast, data()) -> {noreply, data()}. +handle_cast(_Cast, St) -> + {noreply, St}. -code_change(_OldVsn, StateName, State, _Extra) -> - {ok, StateName, State}. +-spec terminate(_Reason, data()) -> ok. +terminate(normal, _St) -> + ok; +terminate({shutdown, _}, _St) -> + ok; +terminate(_Reason, #{upload := Upload}) -> + emqx_s3_upload:abort(Upload). -format_status(#{data := #{client := Client} = Data} = Status) -> - Status#{ - data => Data#{ - client => emqx_s3_client:format(Client), - buffer => [<<"...">>] - } - }. - -format_status(_Opt, [PDict, State, #{client := Client} = Data]) -> - #{ - data => Data#{ - client => emqx_s3_client:format(Client), - buffer => [<<"...">>] - }, - state => State, - pdict => PDict - }. +format_status(#{state := State = #{upload := Upload}} = Status) -> + StateRedacted = State#{upload := emqx_s3_upload:format(Upload)}, + Status#{state := StateRedacted}. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- --spec maybe_start_upload(data()) -> not_started | {started, data()} | {error, term()}. -maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> - case BufferSize >= MinPartSize of - true -> - start_upload(Data); - false -> - not_started - end. - --spec start_upload(data()) -> {started, data()} | {error, term()}. -start_upload(#{client := Client, key := Key, upload_opts := UploadOpts} = Data) -> - case emqx_s3_client:start_multipart(Client, Key, UploadOpts) of - {ok, UploadId} -> - NewData = Data#{upload_id => UploadId}, - {started, NewData}; - {error, _} = Error -> - Error - end. - --spec maybe_upload_part(data()) -> ok_or_error(data(), term()). -maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> - case BufferSize >= MinPartSize of - true -> - upload_part(Data); - false -> - {ok, Data} - end. - --spec upload_part(data()) -> ok_or_error(data(), term()). -upload_part(#{buffer_size := 0} = Data) -> - {ok, Data}; -upload_part( - #{ - client := Client, - key := Key, - upload_id := UploadId, - buffer := Buffer, - part_number := PartNumber, - etags := ETags - } = Data -) -> - case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of - {ok, ETag} -> - NewData = Data#{ - buffer => [], - buffer_size => 0, - part_number => PartNumber + 1, - etags => [{PartNumber, ETag} | ETags] - }, - {ok, NewData}; - {error, _} = Error -> - Error - end. - --spec complete_upload(data()) -> ok_or_error(data(), term()). -complete_upload( - #{ - client := Client, - key := Key, - upload_id := UploadId - } = Data0 -) -> - case upload_part(Data0) of - {ok, #{etags := ETagsRev} = Data1} -> - ETags = lists:reverse(ETagsRev), - case emqx_s3_client:complete_multipart(Client, Key, UploadId, ETags) of - ok -> - {ok, Data1}; - {error, _} = Error -> - Error - end; - {error, _} = Error -> - Error - end. - --spec abort_upload(data()) -> ok_or_error(term()). -abort_upload( - #{ - client := Client, - key := Key, - upload_id := UploadId - } -) -> - case emqx_s3_client:abort_multipart(Client, Key, UploadId) of - ok -> - ok; - {error, _} = Error -> - Error - end. - --spec put_object(data()) -> ok_or_error(term()). -put_object( - #{ - client := Client, - key := Key, - upload_opts := UploadOpts, - buffer := Buffer - } -) -> - case emqx_s3_client:put_object(Client, Key, UploadOpts, Buffer) of - ok -> - ok; - {error, _} = Error -> - Error - end. - --spec append_buffer(data(), iodata()) -> data(). -append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> - Data#{ - buffer => [Buffer, WriteData], - buffer_size => BufferSize + iolist_size(WriteData) - }. - -compile({inline, [wrap/1, unwrap/1]}). wrap(Data) -> fun() -> Data end. @@ -331,8 +150,5 @@ wrap(Data) -> unwrap(WrappedData) -> WrappedData(). -is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> - BufferSize + iolist_size(WriteData) =< MaxPartSize. - client(Bucket, Config) -> emqx_s3_client:create(Bucket, Config). From ccbcc0c4e3782c6e1af823f9ec7631118d587c81 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 25 Apr 2024 18:43:02 +0200 Subject: [PATCH 78/86] feat(s3-bridge): implement aggregated upload action --- apps/emqx_bridge/src/emqx_action_info.erl | 3 +- .../emqx_bridge_s3/src/emqx_bridge_s3.app.src | 10 +- apps/emqx_bridge_s3/src/emqx_bridge_s3.erl | 104 ---- apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl | 7 +- .../src/emqx_bridge_s3_aggreg_buffer.erl | 138 +++++ .../src/emqx_bridge_s3_aggreg_csv.erl | 98 ++++ .../src/emqx_bridge_s3_aggreg_delivery.erl | 162 ++++++ .../src/emqx_bridge_s3_aggreg_upload.erl | 240 +++++++++ ...qx_bridge_s3_aggreg_upload_action_info.erl | 21 + .../src/emqx_bridge_s3_aggreg_upload_sup.erl | 72 +++ .../src/emqx_bridge_s3_aggregator.erl | 485 ++++++++++++++++++ .../src/emqx_bridge_s3_aggregator.hrl | 15 + .../emqx_bridge_s3/src/emqx_bridge_s3_app.erl | 16 + .../src/emqx_bridge_s3_connector.erl | 183 ++++++- .../src/emqx_bridge_s3_connector_info.erl | 2 +- .../emqx_bridge_s3/src/emqx_bridge_s3_sup.erl | 42 ++ .../src/emqx_bridge_s3_upload.erl | 142 +++++ ... => emqx_bridge_s3_upload_action_info.erl} | 8 +- .../test/emqx_bridge_s3_SUITE.erl | 112 ++-- .../emqx_bridge_s3_aggreg_buffer_SUITE.erl | 181 +++++++ .../test/emqx_bridge_s3_aggreg_csv_tests.erl | 72 +++ .../emqx_bridge_s3_aggreg_upload_SUITE.erl | 372 ++++++++++++++ .../test/emqx_bridge_s3_test_helpers.erl | 52 ++ 23 files changed, 2338 insertions(+), 199 deletions(-) create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl create mode 100644 apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl rename apps/emqx_bridge_s3/src/{emqx_bridge_s3_action_info.erl => emqx_bridge_s3_upload_action_info.erl} (69%) create mode 100644 apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl create mode 100644 apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl create mode 100644 apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl create mode 100644 apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index a8aaf9fdd..dc71d5806 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -118,7 +118,8 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_pulsar_action_info, emqx_bridge_greptimedb_action_info, emqx_bridge_tdengine_action_info, - emqx_bridge_s3_action_info + emqx_bridge_s3_upload_action_info, + emqx_bridge_s3_aggreg_upload_action_info ]. -else. hard_coded_action_info_modules_ee() -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src index a6000067a..5cdf3fb82 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src @@ -10,9 +10,15 @@ emqx_s3 ]}, {env, [ - {emqx_action_info_modules, [emqx_bridge_s3_action_info]}, - {emqx_connector_info_modules, [emqx_bridge_s3_connector_info]} + {emqx_action_info_modules, [ + emqx_bridge_s3_upload_action_info, + emqx_bridge_s3_aggreg_upload_action_info + ]}, + {emqx_connector_info_modules, [ + emqx_bridge_s3_connector_info + ]} ]}, + {mod, {emqx_bridge_s3_app, []}}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl index 79cc560d2..49f033554 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl @@ -19,7 +19,6 @@ ]). -export([ - bridge_v2_examples/1, connector_examples/1 ]). @@ -39,58 +38,11 @@ fields(Field) when Field == "post_connector" -> emqx_connector_schema:api_fields(Field, ?CONNECTOR, fields(s3_connector_config)); -fields(Field) when - Field == "get_bridge_v2"; - Field == "put_bridge_v2"; - Field == "post_bridge_v2" --> - emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); -fields(action) -> - {?ACTION, - hoconsc:mk( - hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), - #{ - desc => <<"S3 Action Config">>, - required => false - } - )}; fields("config_connector") -> emqx_connector_schema:common_fields() ++ fields(s3_connector_config); -fields(?ACTION) -> - emqx_bridge_v2_schema:make_producer_action_schema( - hoconsc:mk( - ?R_REF(s3_upload_parameters), - #{ - required => true, - desc => ?DESC(s3_upload) - } - ), - #{ - resource_opts_ref => ?R_REF(s3_action_resource_opts) - } - ); fields(s3_connector_config) -> emqx_s3_schema:fields(s3_client) ++ emqx_connector_schema:resource_opts_ref(?MODULE, s3_connector_resource_opts); -fields(s3_upload_parameters) -> - emqx_s3_schema:fields(s3_upload) ++ - [ - {content, - hoconsc:mk( - emqx_schema:template(), - #{ - required => false, - default => <<"${.}">>, - desc => ?DESC(s3_object_content) - } - )} - ]; -fields(s3_action_resource_opts) -> - UnsupportedOpts = [batch_size, batch_time], - lists:filter( - fun({N, _}) -> not lists:member(N, UnsupportedOpts) end, - emqx_bridge_v2_schema:action_resource_opts_fields() - ); fields(s3_connector_resource_opts) -> CommonOpts = emqx_connector_schema:common_resource_opts_subfields(), lists:filter( @@ -100,14 +52,6 @@ fields(s3_connector_resource_opts) -> desc("config_connector") -> ?DESC(config_connector); -desc(?ACTION) -> - ?DESC(s3_upload); -desc(s3_upload) -> - ?DESC(s3_upload); -desc(s3_upload_parameters) -> - ?DESC(s3_upload_parameters); -desc(s3_action_resource_opts) -> - ?DESC(emqx_resource_schema, resource_opts); desc(s3_connector_resource_opts) -> ?DESC(emqx_resource_schema, resource_opts); desc(_Name) -> @@ -115,54 +59,6 @@ desc(_Name) -> %% Examples -bridge_v2_examples(Method) -> - [ - #{ - <<"s3">> => #{ - summary => <<"S3 Simple Upload">>, - value => action_example(Method) - } - } - ]. - -action_example(post) -> - maps:merge( - action_example(put), - #{ - type => atom_to_binary(?ACTION), - name => <<"my_s3_action">> - } - ); -action_example(get) -> - maps:merge( - action_example(put), - #{ - status => <<"connected">>, - node_status => [ - #{ - node => <<"emqx@localhost">>, - status => <<"connected">> - } - ] - } - ); -action_example(put) -> - #{ - enable => true, - connector => <<"my_s3_connector">>, - description => <<"My action">>, - parameters => #{ - bucket => <<"${clientid}">>, - key => <<"${topic}">>, - content => <<"${payload}">>, - acl => <<"public_read">> - }, - resource_opts => #{ - query_mode => <<"sync">>, - inflight_window => 10 - } - }. - connector_examples(Method) -> [ #{ diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl b/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl index 6d500d056..62a80d260 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl @@ -5,7 +5,12 @@ -ifndef(__EMQX_BRIDGE_S3_HRL__). -define(__EMQX_BRIDGE_S3_HRL__, true). --define(ACTION, s3). +%% Actions +-define(ACTION_UPLOAD, s3). +-define(BRIDGE_TYPE_UPLOAD, <<"s3">>). +-define(ACTION_AGGREGATED_UPLOAD, s3_aggregated_upload). +-define(BRIDGE_TYPE_AGGREGATED_UPLOAD, <<"s3_aggregated_upload">>). + -define(CONNECTOR, s3). -endif. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl new file mode 100644 index 000000000..503b864a7 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl @@ -0,0 +1,138 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module provides pretty stupid interface for writing and reading +%% Erlang terms to/from a file descriptor (i.e. IO device), with a goal +%% of being able to write and read terms in a streaming fashion, and to +%% survive partial corruption. +%% +%% Layout of the file is as follows: +%% ``` +%% ETF(Header { Metadata }) +%% ETF(Record1 ByteSize) +%% ETF(Record1) +%% ETF(Record2 ByteSize) +%% ETF(Record2) +%% ... +%% ``` +%% ^ ETF = Erlang External Term Format (i.e. `erlang:term_to_binary/1`). +-module(emqx_bridge_s3_aggreg_buffer). + +-export([ + new_writer/2, + write/2, + takeover/1 +]). + +-export([ + new_reader/1, + read/1 +]). + +-export_type([writer/0, reader/0]). + +-record(reader, { + fd :: file:io_device() | eof, + buffer :: binary(), + hasread = 0 :: non_neg_integer() +}). + +-type writer() :: file:io_device(). +-type reader() :: #reader{}. + +%% + +-define(VSN, 1). +-define(HEADER(MD), [?MODULE, ?VSN, MD]). + +-define(READAHEAD_BYTES, 64 * 4096). +-define(SANE_TERM_SIZE, 256 * 1024 * 1024). + +%% + +-spec new_writer(file:io_device(), _Meta) -> writer(). +new_writer(FD, Meta) -> + %% TODO: Validate header is not too big? + Header = term_to_iovec(?HEADER(Meta)), + case file:write(FD, Header) of + ok -> + FD; + {error, Reason} -> + error({buffer_write_failed, Reason}) + end. + +-spec write(_Term, writer()) -> ok | {error, file:posix()}. +write(Term, FD) -> + IOData = term_to_iovec(Term), + Marker = term_to_binary(iolist_size(IOData)), + file:write(FD, [Marker | IOData]). + +%% + +-spec new_reader(file:io_device()) -> {_Meta, reader()}. +new_reader(FD) -> + Reader0 = #reader{fd = FD, buffer = <<>>}, + Reader1 = read_buffered(?READAHEAD_BYTES, Reader0), + case read_next_term(Reader1) of + {?HEADER(MD), Reader} -> + {MD, Reader}; + {UnexpectedHeader, _Reader} -> + error({buffer_unexpected_header, UnexpectedHeader}); + eof -> + error({buffer_incomplete, header}) + end. + +-spec read(reader()) -> {_Term, reader()} | eof. +read(Reader0) -> + case read_next_term(read_buffered(_LargeEnough = 16, Reader0)) of + {Size, Reader1} when is_integer(Size) andalso Size > 0 andalso Size < ?SANE_TERM_SIZE -> + case read_next_term(read_buffered(Size, Reader1)) of + {Term, Reader} -> + {Term, Reader}; + eof -> + error({buffer_incomplete, Size}) + end; + {UnexpectedSize, _Reader} -> + error({buffer_unexpected_record_size, UnexpectedSize}); + eof -> + eof + end. + +-spec takeover(reader()) -> writer(). +takeover(#reader{fd = FD, hasread = HasRead}) -> + case file:position(FD, HasRead) of + {ok, HasRead} -> + case file:truncate(FD) of + ok -> + FD; + {error, Reason} -> + error({buffer_takeover_failed, Reason}) + end; + {error, Reason} -> + error({buffer_takeover_failed, Reason}) + end. + +read_next_term(#reader{fd = eof, buffer = <<>>}) -> + eof; +read_next_term(Reader = #reader{buffer = Buffer, hasread = HasRead}) -> + {Term, UsedBytes} = erlang:binary_to_term(Buffer, [safe, used]), + BufferSize = byte_size(Buffer), + BufferLeft = binary:part(Buffer, UsedBytes, BufferSize - UsedBytes), + {Term, Reader#reader{buffer = BufferLeft, hasread = HasRead + UsedBytes}}. + +read_buffered(_Size, Reader = #reader{fd = eof}) -> + Reader; +read_buffered(Size, Reader = #reader{fd = FD, buffer = Buffer0}) -> + BufferSize = byte_size(Buffer0), + ReadSize = erlang:max(Size, ?READAHEAD_BYTES), + case BufferSize < Size andalso file:read(FD, ReadSize) of + false -> + Reader; + {ok, Data} -> + Reader#reader{buffer = <>}; + eof -> + Reader#reader{fd = eof}; + {error, Reason} -> + error({buffer_read_failed, Reason}) + end. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl new file mode 100644 index 000000000..31ce1fcc7 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl @@ -0,0 +1,98 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% CSV container implementation for `emqx_bridge_s3_aggregator`. +-module(emqx_bridge_s3_aggreg_csv). + +%% Container API +-export([ + new/1, + fill/2, + close/1 +]). + +-export_type([container/0]). + +-record(csv, { + columns :: [binary()] | undefined, + order :: [binary()], + separator :: char() | iodata(), + delimiter :: char() | iodata(), + quoting_mp :: _ReMP +}). + +-type container() :: #csv{}. +-type options() :: #{column_order => [column()]}. + +-type record() :: emqx_bridge_s3_aggregator:record(). +-type column() :: binary(). + +%% + +-spec new(options()) -> container(). +new(Opts) -> + {ok, MP} = re:compile("[\\[\\],\\r\\n\"]", [unicode]), + #csv{ + order = maps:get(column_order, Opts, []), + separator = $,, + delimiter = $\n, + quoting_mp = MP + }. + +-spec fill([record()], container()) -> {iodata(), container()}. +fill(Records = [Record | _], CSV0 = #csv{columns = undefined}) -> + Columns = mk_columns(Record, CSV0), + Header = emit_header(Columns, CSV0), + {Writes, CSV} = fill(Records, CSV0#csv{columns = Columns}), + {[Header | Writes], CSV}; +fill(Records, CSV) -> + Writes = [emit_row(R, CSV) || R <- Records], + {Writes, CSV}. + +-spec close(container()) -> iodata(). +close(#csv{}) -> + []. + +%% + +mk_columns(Record, #csv{order = ColumnOrder}) -> + Columns = lists:sort(maps:keys(Record)), + OrderedFirst = [CO || CO <- ColumnOrder, lists:member(CO, Columns)], + Unoredered = Columns -- ColumnOrder, + OrderedFirst ++ Unoredered. + +-spec emit_header([column()], container()) -> iodata(). +emit_header([C], #csv{delimiter = Delim}) -> + [C, Delim]; +emit_header([C | Rest], CSV = #csv{separator = Sep}) -> + [C, Sep | emit_header(Rest, CSV)]; +emit_header([], #csv{delimiter = Delim}) -> + [Delim]. + +-spec emit_row(record(), container()) -> iodata(). +emit_row(Record, CSV = #csv{columns = Columns}) -> + emit_row(Record, Columns, CSV). + +emit_row(Record, [C], CSV = #csv{delimiter = Delim}) -> + [emit_cell(C, Record, CSV), Delim]; +emit_row(Record, [C | Rest], CSV = #csv{separator = Sep}) -> + [emit_cell(C, Record, CSV), Sep | emit_row(Record, Rest, CSV)]; +emit_row(#{}, [], #csv{delimiter = Delim}) -> + [Delim]. + +emit_cell(Column, Record, CSV) -> + case maps:get(Column, Record, undefined) of + undefined -> + _Empty = ""; + Value -> + encode_cell(emqx_template:to_string(Value), CSV) + end. + +encode_cell(V, #csv{quoting_mp = MP}) -> + case re:run(V, MP, []) of + nomatch -> + V; + _ -> + [$", re:replace(V, <<"\"">>, <<"\"\"">>, [global, unicode]), $"] + end. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl new file mode 100644 index 000000000..c548226ab --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl @@ -0,0 +1,162 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module takes aggregated records from a buffer and delivers them to S3, +%% wrapped in a configurable container (though currently there's only CSV). +-module(emqx_bridge_s3_aggreg_delivery). + +-include_lib("snabbkaffe/include/trace.hrl"). +-include("emqx_bridge_s3_aggregator.hrl"). + +-export([start_link/3]). + +%% Internal exports +-export([run_delivery/3]). + +-behaviour(emqx_template). +-export([lookup/2]). + +-record(delivery, { + name :: _Name, + container :: emqx_bridge_s3_aggreg_csv:container(), + reader :: emqx_bridge_s3_aggreg_buffer:reader(), + upload :: emqx_s3_upload:t(), + empty :: boolean() +}). + +%% + +start_link(Name, Buffer, Opts) -> + proc_lib:start_link(?MODULE, run_delivery, [Name, Buffer, Opts]). + +%% + +run_delivery(Name, Buffer, Opts) -> + ?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}), + Reader = open_buffer(Buffer), + Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}), + ok = proc_lib:init_ack({ok, self()}), + loop_deliver(Delivery). + +init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) -> + #delivery{ + name = Name, + container = mk_container(ContainerOpts), + reader = Reader, + upload = mk_upload(Buffer, Opts), + empty = true + }. + +loop_deliver(Delivery = #delivery{reader = Reader0}) -> + case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Records = [#{} | _], Reader} -> + loop_deliver_records(Records, Delivery#delivery{reader = Reader}); + {[], Reader} -> + loop_deliver(Delivery#delivery{reader = Reader}); + eof -> + complete_delivery(Delivery); + {Unexpected, _Reader} -> + exit({buffer_unexpected_record, Unexpected}) + end. + +loop_deliver_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) -> + {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0), + {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), + loop_deliver_upload(Delivery#delivery{ + container = Container, + upload = Upload, + empty = false + }). + +loop_deliver_upload(Delivery = #delivery{upload = Upload0}) -> + case emqx_s3_upload:write(Upload0) of + {ok, Upload} -> + loop_deliver(Delivery#delivery{upload = Upload}); + {cont, Upload} -> + loop_deliver_upload(Delivery#delivery{upload = Upload}); + {error, Reason} -> + %% TODO: retries + _ = emqx_s3_upload:abort(Upload0), + exit({upload_failed, Reason}) + end. + +complete_delivery(#delivery{name = Name, empty = true}) -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}), + exit({shutdown, {skipped, empty}}); +complete_delivery(#delivery{name = Name, container = Container, upload = Upload0}) -> + Trailer = emqx_bridge_s3_aggreg_csv:close(Container), + {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0), + case emqx_s3_upload:complete(Upload) of + {ok, Completed} -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}), + ok; + {error, Reason} -> + %% TODO: retries + _ = emqx_s3_upload:abort(Upload), + exit({upload_failed, Reason}) + end. + +mk_container(#{type := csv, column_order := OrderOpt}) -> + %% TODO: Deduplicate? + ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt), + emqx_bridge_s3_aggreg_csv:new(#{column_order => ColumnOrder}). + +mk_upload( + Buffer, + Opts = #{ + bucket := Bucket, + upload_options := UploadOpts, + client_config := Config, + uploader_config := UploaderConfig + } +) -> + Client = emqx_s3_client:create(Bucket, Config), + Key = mk_object_key(Buffer, Opts), + emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig). + +mk_object_key(Buffer, #{action := Name, key := Template}) -> + emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}). + +open_buffer(#buffer{filename = Filename}) -> + case file:open(Filename, [read, binary, raw]) of + {ok, FD} -> + {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD), + Reader; + {error, Reason} -> + error({buffer_open_failed, Reason}) + end. + +%% + +-spec lookup(emqx_template:accessor(), {_Name, buffer()}) -> + {ok, integer() | string()} | {error, undefined}. +lookup([<<"action">>], {Name, _Buffer}) -> + {ok, mk_fs_safe_string(Name)}; +lookup(Accessor, {_Name, Buffer = #buffer{}}) -> + lookup_buffer_var(Accessor, Buffer); +lookup(_Accessor, _Context) -> + {error, undefined}. + +lookup_buffer_var([<<"datetime">>, Format], #buffer{since = Since}) -> + {ok, format_timestamp(Since, Format)}; +lookup_buffer_var([<<"datetime_until">>, Format], #buffer{until = Until}) -> + {ok, format_timestamp(Until, Format)}; +lookup_buffer_var([<<"sequence">>], #buffer{seq = Seq}) -> + {ok, Seq}; +lookup_buffer_var([<<"node">>], #buffer{}) -> + {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))}; +lookup_buffer_var(_Binding, _Context) -> + {error, undefined}. + +format_timestamp(Timestamp, <<"rfc3339utc">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"rfc3339">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"unix">>) -> + Timestamp. + +mk_fs_safe_string(String) -> + unicode:characters_to_binary(string:replace(String, ":", "_", all)). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl new file mode 100644 index 000000000..a62b47c9d --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl @@ -0,0 +1,240 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_bridge_s3.hrl"). + +-define(ACTION, ?ACTION_AGGREGATED_UPLOAD). + +-define(DEFAULT_BATCH_SIZE, 100). +-define(DEFAULT_BATCH_TIME, <<"10ms">>). + +-behaviour(hocon_schema). +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% Interpreting options +-export([ + mk_key_template/1 +]). + +%% emqx_bridge_v2_schema API +-export([bridge_v2_examples/1]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> + "bridge_s3". + +roots() -> + []. + +fields(Field) when + Field == "get_bridge_v2"; + Field == "put_bridge_v2"; + Field == "post_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); +fields(action) -> + {?ACTION, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), + #{ + desc => <<"S3 Aggregated Upload Action Config">>, + required => false + } + )}; +fields(?ACTION) -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + ?R_REF(s3_aggregated_upload_parameters), + #{ + required => true, + desc => ?DESC(s3_aggregated_upload) + } + ), + #{ + resource_opts_ref => ?R_REF(s3_aggreg_upload_resource_opts) + } + ); +fields(s3_aggregated_upload_parameters) -> + [ + {container, + hoconsc:mk( + %% TODO: Support selectors once there are more than one container. + hoconsc:union(fun + (all_union_members) -> [?REF(s3_container_csv)]; + ({value, _Valur}) -> [?REF(s3_container_csv)] + end), + #{ + required => true, + default => #{<<"type">> => <<"csv">>}, + desc => ?DESC(s3_aggregated_container) + } + )}, + {aggregation, + hoconsc:mk( + ?REF(s3_aggregation), + #{ + required => true, + desc => ?DESC(s3_aggregation) + } + )} + ] ++ + emqx_s3_schema:fields(s3_upload) ++ + emqx_s3_schema:fields(s3_uploader); +fields(s3_container_csv) -> + [ + {type, + hoconsc:mk( + csv, + #{ + required => true, + desc => ?DESC(s3_aggregated_container_csv) + } + )}, + {column_order, + hoconsc:mk( + hoconsc:array(string()), + #{ + required => false, + default => [], + desc => ?DESC(s3_aggregated_container_csv_column_order) + } + )} + ]; +fields(s3_aggregation) -> + [ + %% TODO: Needs bucketing? (e.g. messages falling in this 1h interval) + {time_interval, + hoconsc:mk( + emqx_schema:duration_s(), + #{ + required => false, + default => <<"1h">>, + desc => ?DESC(s3_aggregation_interval) + } + )}, + {max_records, + hoconsc:mk( + pos_integer(), + #{ + required => false, + default => <<"1000000">>, + desc => ?DESC(s3_aggregation_max_records) + } + )} + ]; +fields(s3_aggreg_upload_resource_opts) -> + %% NOTE: This action should benefit from generous batching defaults. + emqx_bridge_v2_schema:action_resource_opts_fields([ + {batch_size, #{default => ?DEFAULT_BATCH_SIZE}}, + {batch_time, #{default => ?DEFAULT_BATCH_TIME}} + ]). + +desc(?ACTION) -> + ?DESC(s3_aggregated_upload); +desc(s3_aggregated_upload_parameters) -> + ?DESC(s3_aggregated_upload_parameters); +desc(s3_aggreg_upload_resource_opts) -> + ?DESC(emqx_resource_schema, resource_opts); +desc(_Name) -> + undefined. + +%% Interpreting options + +-spec mk_key_template(string()) -> emqx_template:str(). +mk_key_template(Key) -> + Template = emqx_template:parse(Key), + {_, BindingErrors} = emqx_template:render(Template, #{}), + {UsedBindings, _} = lists:unzip(BindingErrors), + SuffixTemplate = mk_suffix_template(UsedBindings), + case emqx_template:is_const(SuffixTemplate) of + true -> + Template; + false -> + Template ++ SuffixTemplate + end. + +mk_suffix_template(UsedBindings) -> + RequiredBindings = ["action", "node", "datetime.", "sequence"], + SuffixBindings = [ + mk_default_binding(RB) + || RB <- RequiredBindings, + lists:all(fun(UB) -> string:prefix(UB, RB) == nomatch end, UsedBindings) + ], + SuffixTemplate = [["/", B] || B <- SuffixBindings], + emqx_template:parse(SuffixTemplate). + +mk_default_binding("datetime.") -> + "${datetime.rfc3339utc}"; +mk_default_binding(Binding) -> + "${" ++ Binding ++ "}". + +%% Examples + +bridge_v2_examples(Method) -> + [ + #{ + <<"s3_aggregated_upload">> => #{ + summary => <<"S3 Aggregated Upload">>, + value => s3_action_example(Method) + } + } + ]. + +s3_action_example(post) -> + maps:merge( + s3_action_example(put), + #{ + type => atom_to_binary(?ACTION_UPLOAD), + name => <<"my_s3_action">> + } + ); +s3_action_example(get) -> + maps:merge( + s3_action_example(put), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + } + ); +s3_action_example(put) -> + #{ + enable => true, + connector => <<"my_s3_connector">>, + description => <<"My action">>, + parameters => #{ + bucket => <<"mqtt-aggregated">>, + key => <<"${action}/${node}/${datetime.rfc3339utc}_N${sequence}.csv">>, + acl => <<"public_read">>, + aggregation => #{ + time_interval => <<"15m">>, + max_records => 100_000 + }, + <<"container">> => #{ + type => <<"csv">>, + column_order => [<<"clientid">>, <<"topic">>, <<"publish_received_at">>] + } + }, + resource_opts => #{ + health_check_interval => <<"10s">>, + query_mode => <<"async">>, + inflight_window => 100 + } + }. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl new file mode 100644 index 000000000..b179073e5 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl @@ -0,0 +1,21 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_action_info). + +-behaviour(emqx_action_info). + +-include("emqx_bridge_s3.hrl"). + +-export([ + action_type_name/0, + connector_type_name/0, + schema_module/0 +]). + +action_type_name() -> ?ACTION_AGGREGATED_UPLOAD. + +connector_type_name() -> s3. + +schema_module() -> emqx_bridge_s3_aggreg_upload. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl new file mode 100644 index 000000000..973187b7e --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_sup). + +-export([ + start_link/3, + start_link_delivery_sup/2 +]). + +-export([ + start_delivery/2, + start_delivery_proc/3 +]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPREF(NAME), {via, gproc, {n, l, {?MODULE, NAME}}}). + +%% + +start_link(Name, AggregOpts, DeliveryOpts) -> + supervisor:start_link(?MODULE, {root, Name, AggregOpts, DeliveryOpts}). + +start_link_delivery_sup(Name, DeliveryOpts) -> + supervisor:start_link(?SUPREF(Name), ?MODULE, {delivery, Name, DeliveryOpts}). + +%% + +start_delivery(Name, Buffer) -> + supervisor:start_child(?SUPREF(Name), [Buffer]). + +start_delivery_proc(Name, DeliveryOpts, Buffer) -> + emqx_bridge_s3_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts). + +%% + +init({root, Name, AggregOpts, DeliveryOpts}) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + AggregatorChildSpec = #{ + id => aggregator, + start => {emqx_bridge_s3_aggregator, start_link, [Name, AggregOpts]}, + type => worker, + restart => permanent + }, + DeliverySupChildSpec = #{ + id => delivery_sup, + start => {?MODULE, start_link_delivery_sup, [Name, DeliveryOpts]}, + type => supervisor, + restart => permanent + }, + {ok, {SupFlags, [DeliverySupChildSpec, AggregatorChildSpec]}}; +init({delivery, Name, DeliveryOpts}) -> + SupFlags = #{ + strategy => simple_one_for_one, + intensity => 100, + period => 5 + }, + ChildSpec = #{ + id => delivery, + start => {?MODULE, start_delivery_proc, [Name, DeliveryOpts]}, + type => worker, + restart => temporary, + shutdown => 1000 + }, + {ok, {SupFlags, [ChildSpec]}}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl new file mode 100644 index 000000000..c49a29cb8 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl @@ -0,0 +1,485 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module manages buffers for aggregating records and offloads them +%% to separate "delivery" processes when they are full or time interval +%% is over. +-module(emqx_bridge_s3_aggregator). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-include("emqx_bridge_s3_aggregator.hrl"). + +-export([ + start_link/2, + push_records/3, + tick/2, + take_error/1 +]). + +-behaviour(gen_server). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2 +]). + +-export_type([ + record/0, + timestamp/0 +]). + +%% Record. +-type record() :: #{binary() => _}. + +%% Unix timestamp, seconds since epoch. +-type timestamp() :: _Seconds :: non_neg_integer(). + +%% + +-define(VSN, 1). +-define(SRVREF(NAME), {via, gproc, {n, l, {?MODULE, NAME}}}). + +%% + +start_link(Name, Opts) -> + gen_server:start_link(?SRVREF(Name), ?MODULE, mk_state(Name, Opts), []). + +push_records(Name, Timestamp, Records = [_ | _]) -> + %% FIXME: Error feedback. + case pick_buffer(Name, Timestamp) of + undefined -> + BufferNext = next_buffer(Name, Timestamp), + write_records_limited(Name, BufferNext, Records); + Buffer -> + write_records_limited(Name, Buffer, Records) + end; +push_records(_Name, _Timestamp, []) -> + ok. + +tick(Name, Timestamp) -> + case pick_buffer(Name, Timestamp) of + #buffer{} -> + ok; + _Outdated -> + send_close_buffer(Name, Timestamp) + end. + +take_error(Name) -> + gen_server:call(?SRVREF(Name), take_error). + +%% + +write_records_limited(Name, Buffer = #buffer{max_records = undefined}, Records) -> + write_records(Name, Buffer, Records); +write_records_limited(Name, Buffer = #buffer{max_records = MaxRecords}, Records) -> + NR = length(Records), + case inc_num_records(Buffer, NR) of + NR -> + %% NOTE: Allow unconditionally if it's the first write. + write_records(Name, Buffer, Records); + NWritten when NWritten > MaxRecords -> + NextBuffer = rotate_buffer(Name, Buffer), + write_records_limited(Name, NextBuffer, Records); + _ -> + write_records(Name, Buffer, Records) + end. + +write_records(Name, Buffer = #buffer{fd = Writer}, Records) -> + case emqx_bridge_s3_aggreg_buffer:write(Records, Writer) of + ok -> + ?tp(s3_aggreg_records_written, #{action => Name, records => Records}), + ok; + {error, Reason} when Reason == terminated orelse Reason == closed -> + BufferNext = rotate_buffer(Name, Buffer), + write_records(Name, BufferNext, Records); + {error, _} = Error -> + Error + end. + +inc_num_records(#buffer{cnt_records = Counter}, Size) -> + inc_counter(Counter, Size). + +next_buffer(Name, Timestamp) -> + gen_server:call(?SRVREF(Name), {next_buffer, Timestamp}). + +rotate_buffer(Name, #buffer{fd = FD}) -> + gen_server:call(?SRVREF(Name), {rotate_buffer, FD}). + +send_close_buffer(Name, Timestamp) -> + gen_server:cast(?SRVREF(Name), {close_buffer, Timestamp}). + +%% + +-record(st, { + name :: _Name, + tab :: ets:tid() | undefined, + buffer :: buffer() | undefined, + queued :: buffer() | undefined, + deliveries = #{} :: #{reference() => buffer()}, + errors = queue:new() :: queue:queue(_Error), + interval :: emqx_schema:duration_s(), + max_records :: pos_integer(), + work_dir :: file:filename() +}). + +-type state() :: #st{}. + +mk_state(Name, Opts) -> + Interval = maps:get(time_interval, Opts), + MaxRecords = maps:get(max_records, Opts), + WorkDir = maps:get(work_dir, Opts), + ok = ensure_workdir(WorkDir), + #st{ + name = Name, + interval = Interval, + max_records = MaxRecords, + work_dir = WorkDir + }. + +ensure_workdir(WorkDir) -> + %% NOTE + %% Writing MANIFEST as a means to ensure the work directory is writable. It's not + %% (yet) read back because there's only one version of the implementation. + ok = filelib:ensure_path(WorkDir), + ok = write_manifest(WorkDir). + +write_manifest(WorkDir) -> + Manifest = #{<<"version">> => ?VSN}, + file:write_file(filename:join(WorkDir, "MANIFEST"), hocon_pp:do(Manifest, #{})). + +%% + +-spec init(state()) -> {ok, state()}. +init(St0 = #st{name = Name}) -> + _ = erlang:process_flag(trap_exit, true), + St1 = St0#st{tab = create_tab(Name)}, + St = recover(St1), + _ = announce_current_buffer(St), + {ok, St}. + +handle_call({next_buffer, Timestamp}, _From, St0) -> + St = #st{buffer = Buffer} = handle_next_buffer(Timestamp, St0), + {reply, Buffer, St, 0}; +handle_call({rotate_buffer, FD}, _From, St0) -> + St = #st{buffer = Buffer} = handle_rotate_buffer(FD, St0), + {reply, Buffer, St, 0}; +handle_call(take_error, _From, St0) -> + {MaybeError, St} = handle_take_error(St0), + {reply, MaybeError, St}. + +handle_cast({close_buffer, Timestamp}, St) -> + {noreply, handle_close_buffer(Timestamp, St)}; +handle_cast(_Cast, St) -> + {noreply, St}. + +handle_info(timeout, St) -> + {noreply, handle_queued_buffer(St)}; +handle_info({'DOWN', MRef, _, Pid, Reason}, St0 = #st{name = Name, deliveries = Ds0}) -> + case maps:take(MRef, Ds0) of + {Buffer, Ds} -> + St = St0#st{deliveries = Ds}, + {noreply, handle_delivery_exit(Buffer, Reason, St)}; + error -> + ?SLOG(notice, #{ + msg => "unexpected_down_signal", + action => Name, + pid => Pid, + reason => Reason + }), + {noreply, St0} + end; +handle_info(_Msg, St) -> + {noreply, St}. + +terminate(_Reason, #st{name = Name}) -> + cleanup_tab(Name). + +%% + +handle_next_buffer(Timestamp, St = #st{buffer = #buffer{until = Until}}) when Timestamp < Until -> + St; +handle_next_buffer(Timestamp, St0 = #st{buffer = Buffer = #buffer{since = PrevSince}}) -> + BufferClosed = close_buffer(Buffer), + St = enqueue_closed_buffer(BufferClosed, St0), + handle_next_buffer(Timestamp, PrevSince, St); +handle_next_buffer(Timestamp, St = #st{buffer = undefined}) -> + handle_next_buffer(Timestamp, Timestamp, St). + +handle_next_buffer(Timestamp, PrevSince, St0) -> + NextBuffer = allocate_next_buffer(Timestamp, PrevSince, St0), + St = St0#st{buffer = NextBuffer}, + _ = announce_current_buffer(St), + St. + +handle_rotate_buffer( + FD, + St0 = #st{buffer = Buffer = #buffer{since = Since, seq = Seq, fd = FD}} +) -> + BufferClosed = close_buffer(Buffer), + NextBuffer = allocate_buffer(Since, Seq + 1, St0), + St = enqueue_closed_buffer(BufferClosed, St0#st{buffer = NextBuffer}), + _ = announce_current_buffer(St), + St; +handle_rotate_buffer(_ClosedFD, St) -> + St. + +enqueue_closed_buffer(Buffer, St = #st{queued = undefined}) -> + St#st{queued = Buffer}; +enqueue_closed_buffer(Buffer, St0) -> + %% NOTE: Should never really happen unless interval / max records are too tight. + St = handle_queued_buffer(St0), + St#st{queued = Buffer}. + +handle_queued_buffer(St = #st{queued = undefined}) -> + St; +handle_queued_buffer(St = #st{queued = Buffer}) -> + enqueue_delivery(Buffer, St#st{queued = undefined}). + +allocate_next_buffer(Timestamp, PrevSince, St = #st{interval = Interval}) -> + Since = compute_since(Timestamp, PrevSince, Interval), + allocate_buffer(Since, 0, St). + +compute_since(Timestamp, PrevSince, Interval) -> + Timestamp - (Timestamp - PrevSince) rem Interval. + +allocate_buffer(Since, Seq, St = #st{name = Name}) -> + Buffer = #buffer{filename = Filename, cnt_records = Counter} = mk_buffer(Since, Seq, St), + {ok, FD} = file:open(Filename, [write, binary]), + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(FD, _Meta = []), + _ = add_counter(Counter), + ?tp(s3_aggreg_buffer_allocated, #{action => Name, filename => Filename}), + Buffer#buffer{fd = Writer}. + +recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) -> + {ok, FD} = file:open(Filename, [read, write, binary]), + case recover_buffer_writer(FD, Filename) of + {ok, Writer, NWritten} -> + _ = add_counter(Counter, NWritten), + Buffer#buffer{fd = Writer}; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "existing_buffer_recovery_failed", + filename => Filename, + reason => Reason, + details => "Buffer is corrupted beyond repair, will be discarded." + }), + _ = file:close(FD), + _ = file:delete(Filename), + undefined + end. + +recover_buffer_writer(FD, Filename) -> + try emqx_bridge_s3_aggreg_buffer:new_reader(FD) of + {_Meta, Reader} -> recover_buffer_writer(FD, Filename, Reader, 0) + catch + error:Reason -> + {error, Reason} + end. + +recover_buffer_writer(FD, Filename, Reader0, NWritten) -> + try emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Records, Reader} when is_list(Records) -> + recover_buffer_writer(FD, Filename, Reader, NWritten + length(Records)); + {Unexpected, _Reader} -> + %% Buffer is corrupted, should be discarded. + {error, {buffer_unexpected_record, Unexpected}}; + eof -> + %% Buffer is fine, continue writing at the end. + {ok, FD, NWritten} + catch + error:Reason -> + %% Buffer is truncated or corrupted somewhere in the middle. + %% Continue writing after the last valid record. + ?SLOG(warning, #{ + msg => "existing_buffer_recovered_partially", + filename => Filename, + reason => Reason, + details => + "Buffer is truncated or corrupted somewhere in the middle. " + "Corrupted records will be discarded." + }), + Writer = emqx_bridge_s3_aggreg_buffer:takeover(Reader0), + {ok, Writer, NWritten} + end. + +mk_buffer( + Since, + Seq, + #st{tab = Tab, interval = Interval, max_records = MaxRecords, work_dir = WorkDir} +) -> + Name = mk_filename(Since, Seq), + Counter = {Tab, {Since, Seq}}, + #buffer{ + since = Since, + until = Since + Interval, + seq = Seq, + filename = filename:join(WorkDir, Name), + max_records = MaxRecords, + cnt_records = Counter + }. + +handle_close_buffer( + Timestamp, + St0 = #st{buffer = Buffer = #buffer{until = Until}} +) when Timestamp >= Until -> + St = St0#st{buffer = undefined}, + _ = announce_current_buffer(St), + enqueue_delivery(close_buffer(Buffer), St); +handle_close_buffer(_Timestamp, St = #st{buffer = undefined}) -> + St. + +close_buffer(Buffer = #buffer{fd = FD}) -> + ok = file:close(FD), + Buffer#buffer{fd = undefined}. + +discard_buffer(#buffer{filename = Filename, cnt_records = Counter}) -> + %% NOTE: Hopefully, no process is touching this counter anymore. + _ = del_counter(Counter), + file:delete(Filename). + +pick_buffer(Name, Timestamp) -> + case lookup_current_buffer(Name) of + #buffer{until = Until} = Buffer when Timestamp < Until -> + Buffer; + #buffer{since = Since} when Timestamp < Since -> + %% TODO: Support timestamps going back. + error({invalid_timestamp, Timestamp}); + _Outdated -> + undefined + end. + +announce_current_buffer(#st{tab = Tab, buffer = Buffer}) -> + ets:insert(Tab, {buffer, Buffer}). + +lookup_current_buffer(Name) -> + ets:lookup_element(lookup_tab(Name), buffer, 2). + +%% + +enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) -> + {ok, Pid} = emqx_bridge_s3_aggreg_upload_sup:start_delivery(Name, Buffer), + MRef = erlang:monitor(process, Pid), + St#st{deliveries = Ds#{MRef => Buffer}}. + +handle_delivery_exit(Buffer, Normal, St = #st{name = Name}) when + Normal == normal; Normal == noproc +-> + ?SLOG(debug, #{ + msg => "aggregated_buffer_delivery_completed", + action => Name, + buffer => Buffer#buffer.filename + }), + ok = discard_buffer(Buffer), + St; +handle_delivery_exit(Buffer, {shutdown, {skipped, Reason}}, St = #st{name = Name}) -> + ?SLOG(info, #{ + msg => "aggregated_buffer_delivery_skipped", + action => Name, + buffer => {Buffer#buffer.since, Buffer#buffer.seq}, + reason => Reason + }), + ok = discard_buffer(Buffer), + St; +handle_delivery_exit(Buffer, Error, St = #st{name = Name}) -> + ?SLOG(error, #{ + msg => "aggregated_buffer_delivery_failed", + action => Name, + buffer => {Buffer#buffer.since, Buffer#buffer.seq}, + filename => Buffer#buffer.filename, + reason => Error + }), + enqueue_status_error(Error, St). + +enqueue_status_error({upload_failed, Error}, St = #st{errors = QErrors}) -> + %% TODO + %% This code feels too specific, errors probably need classification. + St#st{errors = queue:in(Error, QErrors)}; +enqueue_status_error(_AnotherError, St) -> + St. + +handle_take_error(St = #st{errors = QErrors0}) -> + case queue:out(QErrors0) of + {{value, Error}, QErrors} -> + {[Error], St#st{errors = QErrors}}; + {empty, QErrors} -> + {[], St#st{errors = QErrors}} + end. + +%% + +recover(St0 = #st{work_dir = WorkDir}) -> + {ok, Filenames} = file:list_dir(WorkDir), + ExistingBuffers = lists:flatmap(fun(FN) -> read_existing_file(FN, St0) end, Filenames), + case lists:reverse(lists:keysort(#buffer.since, ExistingBuffers)) of + [Buffer | ClosedBuffers] -> + St = lists:foldl(fun enqueue_delivery/2, St0, ClosedBuffers), + St#st{buffer = recover_buffer(Buffer)}; + [] -> + St0 + end. + +read_existing_file("MANIFEST", _St) -> + []; +read_existing_file(Name, St) -> + case parse_filename(Name) of + {Since, Seq} -> + [read_existing_buffer(Since, Seq, Name, St)]; + error -> + %% TODO: log? + [] + end. + +read_existing_buffer(Since, Seq, Name, St = #st{work_dir = WorkDir}) -> + Filename = filename:join(WorkDir, Name), + Buffer = mk_buffer(Since, Seq, St), + Buffer#buffer{filename = Filename}. + +%% + +mk_filename(Since, Seq) -> + "T" ++ integer_to_list(Since) ++ "_" ++ pad_number(Seq, 4). + +parse_filename(Filename) -> + case re:run(Filename, "^T(\\d+)_(\\d+)$", [{capture, all_but_first, list}]) of + {match, [Since, Seq]} -> + {list_to_integer(Since), list_to_integer(Seq)}; + nomatch -> + error + end. + +%% + +add_counter({Tab, Counter}) -> + add_counter({Tab, Counter}, 0). + +add_counter({Tab, Counter}, N) -> + ets:insert(Tab, {Counter, N}). + +inc_counter({Tab, Counter}, Size) -> + ets:update_counter(Tab, Counter, {2, Size}). + +del_counter({Tab, Counter}) -> + ets:delete(Tab, Counter). + +%% + +create_tab(Name) -> + Tab = ets:new(?MODULE, [public, set, {write_concurrency, auto}]), + ok = persistent_term:put({?MODULE, Name}, Tab), + Tab. + +lookup_tab(Name) -> + persistent_term:get({?MODULE, Name}). + +cleanup_tab(Name) -> + persistent_term:erase({?MODULE, Name}). + +%% + +pad_number(I, L) -> + string:pad(integer_to_list(I), L, leading, $0). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl new file mode 100644 index 000000000..7ac62c6b5 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl @@ -0,0 +1,15 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-record(buffer, { + since :: emqx_bridge_s3_aggregator:timestamp(), + until :: emqx_bridge_s3_aggregator:timestamp(), + seq :: non_neg_integer(), + filename :: file:filename(), + fd :: file:io_device() | undefined, + max_records :: pos_integer() | undefined, + cnt_records :: {ets:tab(), _Counter} | undefined +}). + +-type buffer() :: #buffer{}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl new file mode 100644 index 000000000..e5b77f9d6 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_app). + +-behaviour(application). +-export([start/2, stop/1]). + +%% + +start(_StartType, _StartArgs) -> + emqx_bridge_s3_sup:start_link(). + +stop(_State) -> + ok. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index 5d3ed19f8..972294ef7 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -7,6 +7,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include("emqx_bridge_s3.hrl"). -behaviour(emqx_resource). -export([ @@ -17,7 +18,7 @@ on_remove_channel/3, on_get_channels/1, on_query/3, - % on_batch_query/3, + on_batch_query/3, on_get_status/2, on_get_channel_status/3 ]). @@ -31,12 +32,31 @@ }. -type channel_config() :: #{ - parameters := #{ - bucket := string(), - key := string(), - content := string(), - acl => emqx_s3:acl() - } + bridge_type := binary(), + parameters := s3_upload_parameters() | s3_aggregated_upload_parameters() +}. + +-type s3_upload_parameters() :: #{ + bucket := string(), + key := string(), + content := string(), + acl => emqx_s3:acl() +}. + +-type s3_aggregated_upload_parameters() :: #{ + bucket := string(), + key := string(), + acl => emqx_s3:acl(), + aggregation => #{ + time_interval := emqx_schema:duration_s(), + max_records := pos_integer() + }, + container := #{ + type := csv, + column_order => [string()] + }, + min_part_size := emqx_schema:bytesize(), + max_part_size := emqx_schema:bytesize() }. -type channel_state() :: #{ @@ -123,12 +143,13 @@ on_get_status(_InstId, State = #{client_config := Config}) -> -spec on_add_channel(_InstanceId :: resource_id(), state(), channel_id(), channel_config()) -> {ok, state()} | {error, _Reason}. on_add_channel(_InstId, State = #{channels := Channels}, ChannelId, Config) -> - ChannelState = init_channel_state(Config), + ChannelState = start_channel(State, Config), {ok, State#{channels => Channels#{ChannelId => ChannelState}}}. -spec on_remove_channel(_InstanceId :: resource_id(), state(), channel_id()) -> {ok, state()}. on_remove_channel(_InstId, State = #{channels := Channels}, ChannelId) -> + ok = stop_channel(maps:get(ChannelId, Channels, undefined)), {ok, State#{channels => maps:remove(ChannelId, Channels)}}. -spec on_get_channels(_InstanceId :: resource_id()) -> @@ -138,27 +159,122 @@ on_get_channels(InstId) -> -spec on_get_channel_status(_InstanceId :: resource_id(), channel_id(), state()) -> channel_status(). -on_get_channel_status(_InstId, ChannelId, #{channels := Channels}) -> +on_get_channel_status(_InstId, ChannelId, State = #{channels := Channels}) -> case maps:get(ChannelId, Channels, undefined) of - _ChannelState = #{} -> - %% TODO - %% Since bucket name may be templated, we can't really provide any - %% additional information regarding the channel health. - ?status_connected; + ChannelState = #{} -> + channel_status(ChannelState, State); undefined -> ?status_disconnected end. -init_channel_state(#{parameters := Parameters}) -> +start_channel(_State, #{ + bridge_type := ?BRIDGE_TYPE_UPLOAD, + parameters := Parameters = #{ + bucket := Bucket, + key := Key, + content := Content + } +}) -> #{ - bucket => emqx_template:parse(maps:get(bucket, Parameters)), - key => emqx_template:parse(maps:get(key, Parameters)), - content => emqx_template:parse(maps:get(content, Parameters)), - upload_options => #{ - acl => maps:get(acl, Parameters, undefined) - } + type => ?ACTION_UPLOAD, + bucket => emqx_template:parse(Bucket), + key => emqx_template:parse(Key), + content => emqx_template:parse(Content), + upload_options => upload_options(Parameters) + }; +start_channel(State, #{ + bridge_type := Type = ?BRIDGE_TYPE_AGGREGATED_UPLOAD, + bridge_name := Name, + parameters := Parameters = #{ + aggregation := #{ + time_interval := TimeInterval, + max_records := MaxRecords + }, + container := Container, + bucket := Bucket, + key := Key + } +}) -> + AggregOpts = #{ + time_interval => TimeInterval, + max_records => MaxRecords, + work_dir => work_dir(Type, Name) + }, + DeliveryOpts = #{ + bucket => Bucket, + key => emqx_bridge_s3_aggreg_upload:mk_key_template(Key), + container => Container, + upload_options => upload_options(Parameters), + client_config => maps:get(client_config, State), + uploader_config => maps:with([min_part_size, max_part_size], Parameters) + }, + _ = emqx_bridge_s3_sup:delete_child({Type, Name}), + {ok, SupPid} = emqx_bridge_s3_sup:start_child(#{ + id => {Type, Name}, + start => {emqx_bridge_s3_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]}, + type => supervisor, + restart => permanent + }), + #{ + type => ?ACTION_AGGREGATED_UPLOAD, + name => Name, + bucket => Bucket, + supervisor => SupPid, + on_stop => fun() -> emqx_bridge_s3_sup:delete_child({Type, Name}) end }. +upload_options(Parameters) -> + #{acl => maps:get(acl, Parameters, undefined)}. + +work_dir(Type, Name) -> + filename:join([emqx:data_dir(), bridge, Type, Name]). + +stop_channel(#{on_stop := OnStop}) -> + OnStop(); +stop_channel(_ChannelState) -> + ok. + +channel_status(#{type := ?ACTION_UPLOAD}, _State) -> + %% TODO + %% Since bucket name may be templated, we can't really provide any additional + %% information regarding the channel health. + ?status_connected; +channel_status(#{type := ?ACTION_AGGREGATED_UPLOAD, name := Name, bucket := Bucket}, State) -> + %% NOTE: This will effectively trigger uploads of buffers yet to be uploaded. + Timestamp = erlang:system_time(second), + ok = emqx_bridge_s3_aggregator:tick(Name, Timestamp), + ok = check_bucket_accessible(Bucket, State), + ok = check_aggreg_upload_errors(Name), + ?status_connected. + +check_bucket_accessible(Bucket, #{client_config := Config}) -> + case emqx_s3_client:aws_config(Config) of + {error, Reason} -> + throw({unhealthy_target, Reason}); + AWSConfig -> + try erlcloud_s3:list_objects(Bucket, [{max_keys, 1}], AWSConfig) of + Props when is_list(Props) -> + ok + catch + error:{aws_error, {http_error, 404, _, _Reason}} -> + throw({unhealthy_target, "Bucket does not exist"}); + error:{aws_error, {socket_error, Reason}} -> + throw({unhealthy_target, emqx_utils:format(Reason)}) + end + end. + +check_aggreg_upload_errors(Name) -> + case emqx_bridge_s3_aggregator:take_error(Name) of + [Error] -> + %% TODO + %% This approach means that, for example, 3 upload failures will cause + %% the channel to be marked as unhealthy for 3 consecutive health checks. + ErrorMessage = emqx_utils:format(Error), + throw({unhealthy_target, ErrorMessage}); + [] -> + ok + end. + %% Queries -type query() :: {_Tag :: channel_id(), _Data :: emqx_jsonish:t()}. @@ -167,8 +283,21 @@ init_channel_state(#{parameters := Parameters}) -> {ok, _Result} | {error, _Reason}. on_query(InstId, {Tag, Data}, #{client_config := Config, channels := Channels}) -> case maps:get(Tag, Channels, undefined) of - ChannelState = #{} -> + ChannelState = #{type := ?ACTION_UPLOAD} -> run_simple_upload(InstId, Tag, Data, ChannelState, Config); + ChannelState = #{type := ?ACTION_AGGREGATED_UPLOAD} -> + run_aggregated_upload(InstId, [Data], ChannelState); + undefined -> + {error, {unrecoverable_error, {invalid_message_tag, Tag}}} + end. + +-spec on_batch_query(_InstanceId :: resource_id(), [query()], state()) -> + {ok, _Result} | {error, _Reason}. +on_batch_query(InstId, [{Tag, Data0} | Rest], #{channels := Channels}) -> + case maps:get(Tag, Channels, undefined) of + ChannelState = #{type := ?ACTION_AGGREGATED_UPLOAD} -> + Records = [Data0 | [Data || {_, Data} <- Rest]], + run_aggregated_upload(InstId, Records, ChannelState); undefined -> {error, {unrecoverable_error, {invalid_message_tag, Tag}}} end. @@ -206,6 +335,16 @@ run_simple_upload( {error, map_error(Reason)} end. +run_aggregated_upload(InstId, Records, #{name := Name}) -> + Timestamp = erlang:system_time(second), + case emqx_bridge_s3_aggregator:push_records(Name, Timestamp, Records) of + ok -> + ?tp(s3_bridge_aggreg_push_ok, #{instance_id => InstId, name => Name}), + ok; + {error, Reason} -> + {error, {unrecoverable_error, Reason}} + end. + map_error({socket_error, _} = Reason) -> {recoverable_error, Reason}; map_error(Reason = {aws_error, Status, _, _Body}) when Status >= 500 -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl index be3c29bae..560334610 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl @@ -20,7 +20,7 @@ type_name() -> s3. bridge_types() -> - [s3]. + [s3, s3_aggregated_upload]. resource_callback_module() -> emqx_bridge_s3_connector. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl new file mode 100644 index 000000000..230711a76 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_sup). + +-export([ + start_link/0, + start_child/1, + delete_child/1 +]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPREF, ?MODULE). + +%% + +start_link() -> + supervisor:start_link({local, ?SUPREF}, ?MODULE, root). + +start_child(ChildSpec) -> + supervisor:start_child(?SUPREF, ChildSpec). + +delete_child(ChildId) -> + case supervisor:terminate_child(?SUPREF, ChildId) of + ok -> + supervisor:delete_child(?SUPREF, ChildId); + Error -> + Error + end. + +%% + +init(root) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 1, + period => 1 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl new file mode 100644 index 000000000..6a63321bd --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl @@ -0,0 +1,142 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_upload). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_bridge_s3.hrl"). + +-define(ACTION, ?ACTION_UPLOAD). + +-behaviour(hocon_schema). +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-export([ + bridge_v2_examples/1 +]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> + "bridge_s3". + +roots() -> + []. + +fields(Field) when + Field == "get_bridge_v2"; + Field == "put_bridge_v2"; + Field == "post_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); +fields(action) -> + {?ACTION, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), + #{ + desc => <<"S3 Upload Action Config">>, + required => false + } + )}; +fields(?ACTION) -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + ?R_REF(s3_upload_parameters), + #{ + required => true, + desc => ?DESC(s3_upload) + } + ), + #{ + resource_opts_ref => ?R_REF(s3_action_resource_opts) + } + ); +fields(s3_upload_parameters) -> + emqx_s3_schema:fields(s3_upload) ++ + [ + {content, + hoconsc:mk( + emqx_schema:template(), + #{ + required => false, + default => <<"${.}">>, + desc => ?DESC(s3_object_content) + } + )} + ]; +fields(s3_action_resource_opts) -> + UnsupportedOpts = [batch_size, batch_time], + lists:filter( + fun({N, _}) -> not lists:member(N, UnsupportedOpts) end, + emqx_bridge_v2_schema:action_resource_opts_fields() + ). + +desc(?ACTION) -> + ?DESC(s3_upload); +desc(s3_upload) -> + ?DESC(s3_upload); +desc(s3_upload_parameters) -> + ?DESC(s3_upload_parameters); +desc(s3_action_resource_opts) -> + ?DESC(emqx_resource_schema, resource_opts); +desc(_Name) -> + undefined. + +%% Examples + +bridge_v2_examples(Method) -> + [ + #{ + <<"s3">> => #{ + summary => <<"S3 Simple Upload">>, + value => s3_upload_action_example(Method) + } + } + ]. + +s3_upload_action_example(post) -> + maps:merge( + s3_upload_action_example(put), + #{ + type => atom_to_binary(?ACTION_UPLOAD), + name => <<"my_s3_action">> + } + ); +s3_upload_action_example(get) -> + maps:merge( + s3_upload_action_example(put), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + } + ); +s3_upload_action_example(put) -> + #{ + enable => true, + connector => <<"my_s3_connector">>, + description => <<"My action">>, + parameters => #{ + bucket => <<"${clientid}">>, + key => <<"${topic}">>, + content => <<"${payload}">>, + acl => <<"public_read">> + }, + resource_opts => #{ + query_mode => <<"sync">>, + inflight_window => 10 + } + }. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl similarity index 69% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl rename to apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl index 646173bf4..15a76ae7c 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl @@ -2,18 +2,20 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_action_info). +-module(emqx_bridge_s3_upload_action_info). -behaviour(emqx_action_info). +-include("emqx_bridge_s3.hrl"). + -export([ action_type_name/0, connector_type_name/0, schema_module/0 ]). -action_type_name() -> s3. +action_type_name() -> ?ACTION_UPLOAD. connector_type_name() -> s3. -schema_module() -> emqx_bridge_s3. +schema_module() -> emqx_bridge_s3_upload. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl index 738f9186f..322666b1f 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl @@ -11,8 +11,6 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). --import(emqx_utils_conv, [bin/1]). - %% See `emqx_bridge_s3.hrl`. -define(BRIDGE_TYPE, <<"s3">>). -define(CONNECTOR_TYPE, <<"s3">>). @@ -79,67 +77,56 @@ end_per_testcase(_TestCase, _Config) -> connector_config(Name, _Config) -> BaseConf = emqx_s3_test_helpers:base_raw_config(tcp), - parse_and_check_config(<<"connectors">>, ?CONNECTOR_TYPE, Name, #{ - <<"enable">> => true, - <<"description">> => <<"S3 Connector">>, - <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), - <<"port">> => maps:get(<<"port">>, BaseConf), - <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), - <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), - <<"transport_options">> => #{ - <<"headers">> => #{ - <<"content-type">> => <> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"connectors">>, ?CONNECTOR_TYPE, Name, #{ + <<"enable">> => true, + <<"description">> => <<"S3 Connector">>, + <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), + <<"port">> => maps:get(<<"port">>, BaseConf), + <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), + <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), + <<"transport_options">> => #{ + <<"headers">> => #{ + <<"content-type">> => <> + }, + <<"connect_timeout">> => <<"500ms">>, + <<"request_timeout">> => <<"1s">>, + <<"pool_size">> => 4, + <<"max_retries">> => 0, + <<"enable_pipelining">> => 1 }, - <<"connect_timeout">> => <<"500ms">>, - <<"request_timeout">> => <<"1s">>, - <<"pool_size">> => 4, - <<"max_retries">> => 0, - <<"enable_pipelining">> => 1 - }, - <<"resource_opts">> => #{ - <<"health_check_interval">> => <<"5s">>, - <<"start_timeout">> => <<"5s">> + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"5s">>, + <<"start_timeout">> => <<"5s">> + } } - }). + ). action_config(Name, ConnectorId) -> - parse_and_check_config(<<"actions">>, ?BRIDGE_TYPE, Name, #{ - <<"enable">> => true, - <<"connector">> => ConnectorId, - <<"parameters">> => #{ - <<"bucket">> => <<"${clientid}">>, - <<"key">> => <<"${topic}">>, - <<"content">> => <<"${payload}">>, - <<"acl">> => <<"public_read">> - }, - <<"resource_opts">> => #{ - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, - <<"health_check_interval">> => <<"3s">>, - <<"inflight_window">> => 40, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"60s">>, - <<"resume_interval">> => <<"3s">>, - <<"worker_pool_size">> => <<"4">> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"actions">>, ?BRIDGE_TYPE, Name, #{ + <<"enable">> => true, + <<"connector">> => ConnectorId, + <<"parameters">> => #{ + <<"bucket">> => <<"${clientid}">>, + <<"key">> => <<"${topic}">>, + <<"content">> => <<"${payload}">>, + <<"acl">> => <<"public_read">> + }, + <<"resource_opts">> => #{ + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"3s">>, + <<"inflight_window">> => 40, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"60s">>, + <<"resume_interval">> => <<"3s">>, + <<"worker_pool_size">> => <<"4">> + } } - }). - -parse_and_check_config(Root, Type, Name, ConfigIn) -> - Schema = - case Root of - <<"connectors">> -> emqx_connector_schema; - <<"actions">> -> emqx_bridge_v2_schema - end, - #{Root := #{Type := #{Name := Config}}} = - hocon_tconf:check_plain( - Schema, - #{Root => #{Type => #{Name => ConfigIn}}}, - #{required => false, atom_key => false} - ), - ct:pal("parsed config: ~p", [Config]), - ConfigIn. + ). t_start_stop(Config) -> emqx_bridge_v2_testlib:t_start_stop(Config, s3_bridge_stopped). @@ -190,7 +177,7 @@ t_sync_query(Config) -> ok = erlcloud_s3:create_bucket(Bucket, AwsConfig), ok = emqx_bridge_v2_testlib:t_sync_query( Config, - fun() -> mk_message(Bucket, Topic, Payload) end, + fun() -> emqx_bridge_s3_test_helpers:mk_message_event(Bucket, Topic, Payload) end, fun(Res) -> ?assertMatch(ok, Res) end, s3_bridge_connector_upload_ok ), @@ -224,15 +211,10 @@ t_query_retry_recoverable(Config) -> heal_failure, [timeout, ?PROXY_NAME, ProxyHost, ProxyPort] ), - Message = mk_message(Bucket, Topic, Payload), + Message = emqx_bridge_s3_test_helpers:mk_message_event(Bucket, Topic, Payload), %% Verify that the message is sent eventually. ok = emqx_bridge_v2:send_message(?BRIDGE_TYPE, BridgeName, Message, #{}), ?assertMatch( #{content := Payload}, maps:from_list(erlcloud_s3:get_object(Bucket, Topic, AwsConfig)) ). - -mk_message(ClientId, Topic, Payload) -> - Message = emqx_message:make(bin(ClientId), bin(Topic), Payload), - {Event, _} = emqx_rule_events:eventmsg_publish(Message), - Event. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl new file mode 100644 index 000000000..199e070d3 --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl @@ -0,0 +1,181 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_buffer_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%% CT Setup + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + WorkDir = emqx_cth_suite:work_dir(Config), + ok = filelib:ensure_path(WorkDir), + [{work_dir, WorkDir} | Config]. + +end_per_suite(_Config) -> + ok. + +%% Testcases + +t_write_read_cycle(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + {ok, WFD} = file:open(Filename, [write, binary]), + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms = [ + [], + [[[[[[[[]]]]]]]], + 123, + lists:seq(1, 100), + lists:seq(1, 1000), + lists:seq(1, 10000), + lists:seq(1, 100000), + #{<<"id">> => 123456789, <<"ts">> => <<"2028-02-29T12:34:56Z">>, <<"gauge">> => 42.42}, + {<<"text/plain">>, _Huge = rand:bytes(1048576)}, + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})} + ], + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + Terms + ), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary, raw]), + {MetadataRead, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + ?assertEqual(Metadata, MetadataRead), + TermsRead = read_until_eof(Reader), + ?assertEqual(Terms, TermsRead). + +t_read_empty(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary]), + ?assertError( + {buffer_incomplete, header}, + emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + ). + +t_read_garbage(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + ok = file:write(WFD, rand:bytes(1048576)), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary]), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + ). + +t_read_truncated(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms = [ + [[[[[[[[[[[]]]]]]]]]]], + lists:seq(1, 100000), + #{<<"id">> => 123456789, <<"ts">> => <<"2029-02-30T12:34:56Z">>, <<"gauge">> => 42.42}, + {<<"text/plain">>, _Huge = rand:bytes(1048576)} + ], + LastTerm = + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}, + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + Terms + ), + {ok, WPos} = file:position(WFD, cur), + ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(LastTerm, Writer)), + ok = file:close(WFD), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos + 1), + {ok, RFD1} = file:open(Filename, [read, binary]), + {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD1), + {ReadTerms1, Reader1} = read_terms(length(Terms), Reader0), + ?assertEqual(Terms, ReadTerms1), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader1) + ), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + {ok, RFD2} = file:open(Filename, [read, binary]), + {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD2), + {ReadTerms2, Reader3} = read_terms(_FitsInto = 3, Reader2), + ?assertEqual(lists:sublist(Terms, 3), ReadTerms2), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader3) + ). + +t_read_truncated_takeover_write(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + Writer1 = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms1 = [ + [[[[[[[[[[[]]]]]]]]]]], + lists:seq(1, 10000), + lists:duplicate(1000, ?FUNCTION_NAME), + {<<"text/plain">>, _Huge = rand:bytes(1048576)} + ], + Terms2 = [ + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}, + {<<"application/x-octet-stream">>, rand:bytes(102400)} + ], + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer1)) end, + Terms1 + ), + {ok, WPos} = file:position(WFD, cur), + ok = file:close(WFD), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + {ok, RWFD} = file:open(Filename, [read, write, binary]), + {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RWFD), + {ReadTerms1, Reader1} = read_terms(_Survived = 3, Reader0), + ?assertEqual( + lists:sublist(Terms1, 3), + ReadTerms1 + ), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader1) + ), + Writer2 = emqx_bridge_s3_aggreg_buffer:takeover(Reader1), + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer2)) end, + Terms2 + ), + ok = file:close(RWFD), + {ok, RFD} = file:open(Filename, [read, binary]), + {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + ReadTerms2 = read_until_eof(Reader2), + ?assertEqual( + lists:sublist(Terms1, 3) ++ Terms2, + ReadTerms2 + ). + +%% + +mk_filename(Name, Config) -> + filename:join(?config(work_dir, Config), Name). + +read_terms(0, Reader) -> + {[], Reader}; +read_terms(N, Reader0) -> + {Term, Reader1} = emqx_bridge_s3_aggreg_buffer:read(Reader0), + {Terms, Reader} = read_terms(N - 1, Reader1), + {[Term | Terms], Reader}. + +read_until_eof(Reader0) -> + case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Term, Reader} -> + [Term | read_until_eof(Reader)]; + eof -> + [] + end. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl new file mode 100644 index 000000000..6da70c6fe --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_csv_tests). + +-include_lib("eunit/include/eunit.hrl"). + +encoding_test() -> + CSV = emqx_bridge_s3_aggreg_csv:new(#{}), + ?assertEqual( + "A,B,Ç\n" + "1.2345,string,0.0\n" + "0.3333333333,\"[]\",-0.0\n" + "111111,🫠,0.0\n" + "111.111,\"\"\"quoted\"\"\",\"line\r\nbreak\"\n" + "222.222,,\n", + fill_close(CSV, [ + [ + #{<<"A">> => 1.2345, <<"B">> => "string", <<"Ç"/utf8>> => +0.0}, + #{<<"A">> => 1 / 3, <<"B">> => "[]", <<"Ç"/utf8>> => -0.0}, + #{<<"A">> => 111111, <<"B">> => "🫠", <<"Ç"/utf8>> => 0.0}, + #{<<"A">> => 111.111, <<"B">> => "\"quoted\"", <<"Ç"/utf8>> => "line\r\nbreak"}, + #{<<"A">> => 222.222, <<"B">> => "", <<"Ç"/utf8>> => undefined} + ] + ]) + ). + +column_order_test() -> + Order = [<<"ID">>, <<"TS">>], + CSV = emqx_bridge_s3_aggreg_csv:new(#{column_order => Order}), + ?assertEqual( + "ID,TS,A,B,D\n" + "1,2024-01-01,12.34,str,\"[]\"\n" + "2,2024-01-02,23.45,ing,\n" + "3,,45,,'\n" + "4,2024-01-04,,,\n", + fill_close(CSV, [ + [ + #{ + <<"A">> => 12.34, + <<"B">> => "str", + <<"ID">> => 1, + <<"TS">> => "2024-01-01", + <<"D">> => <<"[]">> + }, + #{ + <<"TS">> => "2024-01-02", + <<"C">> => <<"null">>, + <<"ID">> => 2, + <<"A">> => 23.45, + <<"B">> => "ing" + } + ], + [ + #{<<"A">> => 45, <<"D">> => <<"'">>, <<"ID">> => 3}, + #{<<"ID">> => 4, <<"TS">> => "2024-01-04"} + ] + ]) + ). + +fill_close(CSV, LRecords) -> + string(fill_close_(CSV, LRecords)). + +fill_close_(CSV0, [Records | LRest]) -> + {Writes, CSV} = emqx_bridge_s3_aggreg_csv:fill(Records, CSV0), + [Writes | fill_close_(CSV, LRest)]; +fill_close_(CSV, []) -> + [emqx_bridge_s3_aggreg_csv:close(CSV)]. + +string(Writes) -> + unicode:characters_to_list(Writes). diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl new file mode 100644 index 000000000..45a830294 --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -0,0 +1,372 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +-import(emqx_utils_conv, [bin/1]). + +%% See `emqx_bridge_s3.hrl`. +-define(BRIDGE_TYPE, <<"s3_aggregated_upload">>). +-define(CONNECTOR_TYPE, <<"s3">>). + +-define(PROXY_NAME, "minio_tcp"). + +-define(CONF_TIME_INTERVAL, 4000). +-define(CONF_MAX_RECORDS, 100). +-define(CONF_COLUMN_ORDER, ?CONF_COLUMN_ORDER([])). +-define(CONF_COLUMN_ORDER(T), [ + <<"publish_received_at">>, + <<"clientid">>, + <<"topic">>, + <<"payload">> + | T +]). + +-define(LIMIT_TOLERANCE, 1.1). + +%% CT Setup + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + _ = emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_s3, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, _} = emqx_common_test_http:create_default_app(), + [ + {apps, Apps}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {proxy_name, ?PROXY_NAME} + | Config + ]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)). + +%% Testcases + +init_per_testcase(TestCase, Config) -> + ct:timetrap(timer:seconds(10)), + ok = snabbkaffe:start_trace(), + TS = erlang:system_time(), + Name = iolist_to_binary(io_lib:format("~s-~p", [TestCase, TS])), + Bucket = unicode:characters_to_list(string:replace(Name, "_", "-", all)), + ConnectorConfig = connector_config(Name, Config), + ActionConfig = action_config(Name, Name, Bucket), + ok = emqx_bridge_s3_test_helpers:create_bucket(Bucket), + [ + {connector_type, ?CONNECTOR_TYPE}, + {connector_name, Name}, + {connector_config, ConnectorConfig}, + {bridge_type, ?BRIDGE_TYPE}, + {bridge_name, Name}, + {bridge_config, ActionConfig}, + {s3_bucket, Bucket} + | Config + ]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + ok. + +connector_config(Name, _Config) -> + BaseConf = emqx_s3_test_helpers:base_raw_config(tcp), + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"connectors">>, ?CONNECTOR_TYPE, Name, #{ + <<"enable">> => true, + <<"description">> => <<"S3 Connector">>, + <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), + <<"port">> => maps:get(<<"port">>, BaseConf), + <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), + <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), + <<"transport_options">> => #{ + <<"connect_timeout">> => <<"500ms">>, + <<"request_timeout">> => <<"1s">>, + <<"pool_size">> => 4, + <<"max_retries">> => 0 + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"1s">> + } + } + ). + +action_config(Name, ConnectorId, Bucket) -> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"actions">>, ?BRIDGE_TYPE, Name, #{ + <<"enable">> => true, + <<"connector">> => ConnectorId, + <<"parameters">> => #{ + <<"bucket">> => unicode:characters_to_binary(Bucket), + <<"key">> => <<"${action}/${node}/${datetime.rfc3339}">>, + <<"acl">> => <<"public_read">>, + <<"aggregation">> => #{ + <<"time_interval">> => <<"4s">>, + <<"max_records">> => ?CONF_MAX_RECORDS + }, + <<"container">> => #{ + <<"type">> => <<"csv">>, + <<"column_order">> => ?CONF_COLUMN_ORDER + } + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"1s">>, + <<"max_buffer_bytes">> => <<"64MB">>, + <<"query_mode">> => <<"async">>, + <<"worker_pool_size">> => 4 + } + } + ). + +t_start_stop(Config) -> + emqx_bridge_v2_testlib:t_start_stop(Config, s3_bridge_stopped). + +t_create_via_http(Config) -> + emqx_bridge_v2_testlib:t_create_via_http(Config). + +t_on_get_status(Config) -> + emqx_bridge_v2_testlib:t_on_get_status(Config, #{}). + +t_aggreg_upload(Config) -> + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + BridgeNameString = unicode:characters_to_list(BridgeName), + NodeString = atom_to_list(node()), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Prepare some sample messages that look like Rule SQL productions. + MessageEvents = lists:map(fun mk_message_event/1, [ + {<<"C1">>, T1 = <<"a/b/c">>, P1 = <<"{\"hello\":\"world\"}">>}, + {<<"C2">>, T2 = <<"foo/bar">>, P2 = <<"baz">>}, + {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>} + ]), + ok = send_messages(BridgeName, MessageEvents), + %% Wait until the delivery is completed. + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check the uploaded objects. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + ?assertMatch( + [BridgeNameString, NodeString, _Datetime, _Seq = "0"], + string:split(Key, "/", all) + ), + _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + %% Verify that column order is respected. + ?assertMatch( + {ok, [ + ?CONF_COLUMN_ORDER(_), + [TS, <<"C1">>, T1, P1 | _], + [TS, <<"C2">>, T2, P2 | _], + [TS, <<"C3">>, T3, P3 | _] + ]}, + erl_csv:decode(Content) + ). + +t_aggreg_upload_restart(Config) -> + %% NOTE + %% This test verifies that the bridge will reuse existing aggregation buffer + %% after a restart. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send some sample messages that look like Rule SQL productions. + MessageEvents = lists:map(fun mk_message_event/1, [ + {<<"C1">>, T1 = <<"a/b/c">>, P1 = <<"{\"hello\":\"world\"}">>}, + {<<"C2">>, T2 = <<"foo/bar">>, P2 = <<"baz">>}, + {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>} + ]), + ok = send_messages(BridgeName, MessageEvents), + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + %% Restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Send some more messages. + ok = send_messages(BridgeName, MessageEvents), + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check there's still only one upload. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + %% Verify that column order is respected. + ?assertMatch( + {ok, [ + _Header = [_ | _], + [TS1, <<"C1">>, T1, P1 | _], + [TS1, <<"C2">>, T2, P2 | _], + [TS1, <<"C3">>, T3, P3 | _], + [TS2, <<"C1">>, T1, P1 | _], + [TS2, <<"C2">>, T2, P2 | _], + [TS2, <<"C3">>, T3, P3 | _] + ]}, + erl_csv:decode(Content) + ). + +t_aggreg_upload_restart_corrupted(Config) -> + %% NOTE + %% This test verifies that the bridge can recover from a buffer file corruption, + %% and does so while preserving uncompromised data. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + BatchSize = ?CONF_MAX_RECORDS div 2, + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send some sample messages that look like Rule SQL productions. + Messages1 = [ + {integer_to_binary(N), <<"a/b/c">>, <<"{\"hello\":\"world\"}">>} + || N <- lists:seq(1, BatchSize) + ], + %% Ensure that they span multiple batch queries. + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages1), 1), + {ok, _} = ?block_until( + #{?snk_kind := s3_aggreg_records_written, action := BridgeName}, + infinity, + 0 + ), + %% Find out the buffer file. + {ok, #{filename := Filename}} = ?block_until( + #{?snk_kind := s3_aggreg_buffer_allocated, action := BridgeName} + ), + %% Stop the bridge, corrupt the buffer file, and restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + BufferFileSize = filelib:file_size(Filename), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, BufferFileSize div 2), + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Send some more messages. + Messages2 = [ + {integer_to_binary(N), <<"c/d/e">>, <<"{\"hello\":\"world\"}">>} + || N <- lists:seq(1, BatchSize) + ], + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages2), 0), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check that upload contains part of the first batch and all of the second batch. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + CSV = [_Header | Rows] = fetch_parse_csv(Bucket, Key), + NRows = length(Rows), + ?assert( + NRows > BatchSize, + CSV + ), + ?assertEqual( + lists:sublist(Messages1, NRows - BatchSize) ++ Messages2, + [{ClientID, Topic, Payload} || [_TS, ClientID, Topic, Payload | _] <- Rows], + CSV + ). + +t_aggreg_next_rotate(Config) -> + %% NOTE + %% This is essentially a stress test that tries to verify that buffer rotation + %% and windowing work correctly under high rate, high concurrency conditions. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + NSenders = 4, + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Start separate processes to send messages. + Senders = [ + spawn_link(fun() -> run_message_sender(BridgeName, N) end) + || N <- lists:seq(1, NSenders) + ], + %% Give them some time to send messages so that rotation and windowing will happen. + ok = timer:sleep(round(?CONF_TIME_INTERVAL * 1.5)), + %% Stop the senders. + _ = [Sender ! {stop, self()} || Sender <- Senders], + NSent = receive_sender_reports(Senders), + %% Wait for the last delivery to complete. + ok = timer:sleep(round(?CONF_TIME_INTERVAL * 0.5)), + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}, infinity, 0), + %% There should be at least 2 time windows of aggregated records. + Uploads = [K || #{key := K} <- emqx_bridge_s3_test_helpers:list_objects(Bucket)], + DTs = [DT || K <- Uploads, [_Action, _Node, DT | _] <- [string:split(K, "/", all)]], + ?assert( + ordsets:size(ordsets:from_list(DTs)) > 1, + Uploads + ), + %% Uploads should not contain more than max allowed records. + CSVs = [{K, fetch_parse_csv(Bucket, K)} || K <- Uploads], + NRecords = [{K, length(CSV) - 1} || {K, CSV} <- CSVs], + ?assertEqual( + [], + [{K, NR} || {K, NR} <- NRecords, NR > ?CONF_MAX_RECORDS * ?LIMIT_TOLERANCE] + ), + %% No message should be lost. + ?assertEqual( + NSent, + lists:sum([NR || {_, NR} <- NRecords]) + ). + +run_message_sender(BridgeName, N) -> + ClientID = integer_to_binary(N), + Topic = <<"a/b/c/", ClientID/binary>>, + run_message_sender(BridgeName, N, ClientID, Topic, N, 0). + +run_message_sender(BridgeName, N, ClientID, Topic, Delay, NSent) -> + Payload = integer_to_binary(N * 1_000_000 + NSent), + Message = emqx_bridge_s3_test_helpers:mk_message_event(ClientID, Topic, Payload), + _ = send_message(BridgeName, Message), + receive + {stop, From} -> + From ! {sent, self(), NSent + 1} + after Delay -> + run_message_sender(BridgeName, N, ClientID, Topic, Delay, NSent + 1) + end. + +receive_sender_reports([Sender | Rest]) -> + receive + {sent, Sender, NSent} -> NSent + receive_sender_reports(Rest) + end; +receive_sender_reports([]) -> + 0. + +%% + +mk_message_event({ClientID, Topic, Payload}) -> + emqx_bridge_s3_test_helpers:mk_message_event(ClientID, Topic, Payload). + +send_messages(BridgeName, MessageEvents) -> + lists:foreach( + fun(M) -> send_message(BridgeName, M) end, + MessageEvents + ). + +send_messages_delayed(BridgeName, MessageEvents, Delay) -> + lists:foreach( + fun(M) -> + send_message(BridgeName, M), + timer:sleep(Delay) + end, + MessageEvents + ). + +send_message(BridgeName, Message) -> + ?assertEqual(ok, emqx_bridge_v2:send_message(?BRIDGE_TYPE, BridgeName, Message, #{})). + +fetch_parse_csv(Bucket, Key) -> + #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + {ok, CSV} = erl_csv:decode(Content), + CSV. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl new file mode 100644 index 000000000..de19c8028 --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_utils_conv, [bin/1]). + +parse_and_check_config(Root, Type, Name, Config) -> + Schema = + case Root of + <<"connectors">> -> emqx_connector_schema; + <<"actions">> -> emqx_bridge_v2_schema + end, + #{Root := #{Type := #{Name := _ConfigParsed}}} = + hocon_tconf:check_plain( + Schema, + #{Root => #{Type => #{Name => Config}}}, + #{required => false, atom_key => false} + ), + Config. + +mk_message_event(ClientId, Topic, Payload) -> + Message = emqx_message:make(bin(ClientId), bin(Topic), Payload), + {Event, _} = emqx_rule_events:eventmsg_publish(Message), + emqx_utils_maps:binary_key_map(Event). + +create_bucket(Bucket) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + erlcloud_s3:create_bucket(Bucket, AwsConfig). + +list_objects(Bucket) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + Response = erlcloud_s3:list_objects(Bucket, AwsConfig), + false = proplists:get_value(is_truncated, Response), + Contents = proplists:get_value(contents, Response), + lists:map(fun maps:from_list/1, Contents). + +get_object(Bucket, Key) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + maps:from_list(erlcloud_s3:get_object(Bucket, Key, AwsConfig)). + +%% File utilities + +truncate_at(Filename, Pos) -> + {ok, FD} = file:open(Filename, [read, write, binary]), + {ok, Pos} = file:position(FD, Pos), + ok = file:truncate(FD), + ok = file:close(FD). From 5b15b2d6410d8ed0c04d28948fbce8bdb3144834 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 26 Apr 2024 12:50:41 +0200 Subject: [PATCH 79/86] docs(s3-bridge): document aggregated upload action schema --- .../src/emqx_bridge_s3_aggreg_upload.erl | 69 ++++++++++--------- .../src/emqx_bridge_s3_upload.erl | 11 +-- rel/i18n/emqx_bridge_s3.hocon | 15 ---- rel/i18n/emqx_bridge_s3_aggreg_upload.hocon | 64 +++++++++++++++++ rel/i18n/emqx_bridge_s3_upload.hocon | 18 +++++ 5 files changed, 126 insertions(+), 51 deletions(-) create mode 100644 rel/i18n/emqx_bridge_s3_aggreg_upload.hocon create mode 100644 rel/i18n/emqx_bridge_s3_upload.hocon diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl index a62b47c9d..497f59ca9 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl @@ -60,7 +60,7 @@ fields(?ACTION) -> ?R_REF(s3_aggregated_upload_parameters), #{ required => true, - desc => ?DESC(s3_aggregated_upload) + desc => ?DESC(s3_aggregated_upload_parameters) } ), #{ @@ -68,32 +68,36 @@ fields(?ACTION) -> } ); fields(s3_aggregated_upload_parameters) -> - [ - {container, - hoconsc:mk( - %% TODO: Support selectors once there are more than one container. - hoconsc:union(fun - (all_union_members) -> [?REF(s3_container_csv)]; - ({value, _Valur}) -> [?REF(s3_container_csv)] - end), - #{ - required => true, - default => #{<<"type">> => <<"csv">>}, - desc => ?DESC(s3_aggregated_container) - } - )}, - {aggregation, - hoconsc:mk( - ?REF(s3_aggregation), - #{ - required => true, - desc => ?DESC(s3_aggregation) - } - )} - ] ++ - emqx_s3_schema:fields(s3_upload) ++ - emqx_s3_schema:fields(s3_uploader); -fields(s3_container_csv) -> + lists:append([ + [ + {container, + hoconsc:mk( + %% TODO: Support selectors once there are more than one container. + hoconsc:union(fun + (all_union_members) -> [?REF(s3_aggregated_container_csv)]; + ({value, _Valur}) -> [?REF(s3_aggregated_container_csv)] + end), + #{ + required => true, + default => #{<<"type">> => <<"csv">>}, + desc => ?DESC(s3_aggregated_container) + } + )}, + {aggregation, + hoconsc:mk( + ?REF(s3_aggregation), + #{ + required => true, + desc => ?DESC(s3_aggregation) + } + )} + ], + emqx_resource_schema:override(emqx_s3_schema:fields(s3_upload), [ + {key, #{desc => ?DESC(s3_aggregated_upload_key)}} + ]), + emqx_s3_schema:fields(s3_uploader) + ]); +fields(s3_aggregated_container_csv) -> [ {type, hoconsc:mk( @@ -142,10 +146,13 @@ fields(s3_aggreg_upload_resource_opts) -> {batch_time, #{default => ?DEFAULT_BATCH_TIME}} ]). -desc(?ACTION) -> - ?DESC(s3_aggregated_upload); -desc(s3_aggregated_upload_parameters) -> - ?DESC(s3_aggregated_upload_parameters); +desc(Name) when + Name == s3_aggregated_upload; + Name == s3_aggregated_upload_parameters; + Name == s3_aggregation; + Name == s3_aggregated_container_csv +-> + ?DESC(Name); desc(s3_aggreg_upload_resource_opts) -> ?DESC(emqx_resource_schema, resource_opts); desc(_Name) -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl index 6a63321bd..44e2360b8 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl @@ -80,12 +80,13 @@ fields(s3_action_resource_opts) -> emqx_bridge_v2_schema:action_resource_opts_fields() ). -desc(?ACTION) -> +desc(s3) -> ?DESC(s3_upload); -desc(s3_upload) -> - ?DESC(s3_upload); -desc(s3_upload_parameters) -> - ?DESC(s3_upload_parameters); +desc(Name) when + Name == s3_upload; + Name == s3_upload_parameters +-> + ?DESC(Name); desc(s3_action_resource_opts) -> ?DESC(emqx_resource_schema, resource_opts); desc(_Name) -> diff --git a/rel/i18n/emqx_bridge_s3.hocon b/rel/i18n/emqx_bridge_s3.hocon index fe10313e0..531ba9823 100644 --- a/rel/i18n/emqx_bridge_s3.hocon +++ b/rel/i18n/emqx_bridge_s3.hocon @@ -5,19 +5,4 @@ config_connector.label: config_connector.desc: """Configuration for a connector to S3 API compatible storage service.""" -s3_upload.label: -"""S3 Simple Upload""" -s3_upload.desc: -"""Action to upload a single object to the S3 service.""" - -s3_upload_parameters.label: -"""S3 Upload action parameters""" -s3_upload_parameters.desc: -"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content.""" - -s3_object_content.label: -"""S3 Object Content""" -s3_object_content.desc: -"""Content of the S3 object being uploaded. Supports templates.""" - } diff --git a/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon b/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon new file mode 100644 index 000000000..07239a32d --- /dev/null +++ b/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon @@ -0,0 +1,64 @@ +emqx_bridge_s3_aggreg_upload { + +s3_aggregated_upload.label: +"""S3 Aggregated Upload""" +s3_aggregated_upload.desc: +"""Action that enables time-based aggregation of incoming events and uploading them to the S3 service as a single object.""" + +s3_aggregated_upload_parameters.label: +"""S3 Aggregated Upload action parameters""" +s3_aggregated_upload_parameters.desc: +"""Set of parameters for the aggregated upload action.""" + +s3_aggregation.label: +"""Aggregation parameters""" +s3_aggregation.desc: +"""Set of parameters governing the aggregation process.""" + +s3_aggregation_interval.label: +"""Time interval""" +s3_aggregation_interval.desc: +"""Amount of time events will be aggregated in a single object before uploading.""" + +s3_aggregation_max_records.label: +"""Maximum number of records""" +s3_aggregation_max_records.desc: +"""Number of records (events) allowed per each aggregated object. Each aggregated upload will contain no more than that number of events, but may contain less.
+If event rate is high enough, there obviously may be more than one aggregated upload during the same time interval. These uploads will have different, but consecutive sequence numbers, which will be a part of S3 object key.""" + +s3_aggregated_container.label: +"""Container for aggregated events""" +s3_aggregated_container.desc: +"""Settings governing the file format of an upload containing aggregated events.""" + +s3_aggregated_container_csv.label: +"""CSV container""" +s3_aggregated_container_csv.desc: +"""Records (events) will be aggregated and uploaded as a CSV file.""" + +s3_aggregated_container_csv_column_order.label: +"""CSV column order""" +s3_aggregated_container_csv_column_order.desc: +"""Event fields that will be ordered first as columns in the resulting CSV file.
+Regardless of this setting, resulting CSV will contain all the fields of aggregated events, but all the columns not explicitly mentioned here will be ordered after the ones listed here in the lexicographical order.""" + +s3_aggregated_upload_key.label: +"""S3 object key template""" +s3_aggregated_upload_key.desc: +"""Template for the S3 object key of an aggregated upload.
+Template may contain placeholders for the following variables: +
    +
  • ${action}: name of the action (required).
  • +
  • ${node}: name of the EMQX node conducting the upload (required).
  • +
  • ${datetime.{format}}: date and time when aggregation started, formatted according to the {format} string (required): +
      +
    • ${datetime.rfc3339utc}: RFC3339-formatted date and time in UTC,
    • +
    • ${datetime.rfc3339}: RFC3339-formatted date and time in local timezone,
    • +
    • ${datetime.unix}: Unix timestamp.
    • +
    +
  • +
  • ${datetime_until.{format}}: date and time when aggregation ended, with the same formatting options.
  • +
  • ${sequence}: sequence number of the aggregated upload within the same time interval (required).
  • +
+All other placeholders are considered invalid. Note that placeholders marked as required will be added as a path suffix to the S3 object key if they are missing from the template.""" +} diff --git a/rel/i18n/emqx_bridge_s3_upload.hocon b/rel/i18n/emqx_bridge_s3_upload.hocon new file mode 100644 index 000000000..7d08cfaa5 --- /dev/null +++ b/rel/i18n/emqx_bridge_s3_upload.hocon @@ -0,0 +1,18 @@ +emqx_bridge_s3_upload { + +s3_upload.label: +"""S3 Simple Upload""" +s3_upload.desc: +"""Action to upload a single object to the S3 service.""" + +s3_upload_parameters.label: +"""S3 Upload action parameters""" +s3_upload_parameters.desc: +"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content.""" + +s3_object_content.label: +"""S3 Object Content""" +s3_object_content.desc: +"""Content of the S3 object being uploaded. Supports templates.""" + +} From 339036045dcc57aff165b185742123dbbdcec38f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 26 Apr 2024 13:41:06 +0200 Subject: [PATCH 80/86] feat(s3-aggreg): support custom and default S3 object HTTP headers I.e. configured container decides default `Content-Type` header. --- .../src/emqx_bridge_s3_aggreg_upload.erl | 34 +++++++++++++++++-- .../src/emqx_bridge_s3_connector.erl | 7 ++-- .../emqx_bridge_s3_aggreg_upload_SUITE.erl | 11 ++++-- apps/emqx_s3/src/emqx_s3_schema.erl | 8 +++++ rel/i18n/emqx_s3_schema.hocon | 7 ++++ 5 files changed, 58 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl index 497f59ca9..02d43f4f0 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl @@ -23,7 +23,8 @@ %% Interpreting options -export([ - mk_key_template/1 + mk_key_template/1, + mk_upload_options/1 ]). %% emqx_bridge_v2_schema API @@ -160,8 +161,8 @@ desc(_Name) -> %% Interpreting options --spec mk_key_template(string()) -> emqx_template:str(). -mk_key_template(Key) -> +-spec mk_key_template(_Parameters :: map()) -> emqx_template:str(). +mk_key_template(#{key := Key}) -> Template = emqx_template:parse(Key), {_, BindingErrors} = emqx_template:render(Template, #{}), {UsedBindings, _} = lists:unzip(BindingErrors), @@ -188,6 +189,33 @@ mk_default_binding("datetime.") -> mk_default_binding(Binding) -> "${" ++ Binding ++ "}". +-spec mk_upload_options(_Parameters :: map()) -> emqx_s3_client:upload_options(). +mk_upload_options(Parameters) -> + Headers = mk_upload_headers(Parameters), + #{ + headers => Headers, + acl => maps:get(acl, Parameters, undefined) + }. + +mk_upload_headers(Parameters = #{container := Container}) -> + Headers = normalize_headers(maps:get(headers, Parameters, #{})), + ContainerHeaders = mk_container_headers(Container), + maps:merge(ContainerHeaders, Headers). + +normalize_headers(Headers) -> + maps:fold( + fun(Header, Value, Acc) -> + maps:put(string:lowercase(emqx_utils_conv:str(Header)), Value, Acc) + end, + #{}, + Headers + ). + +mk_container_headers(#{type := csv}) -> + #{"content-type" => "text/csv"}; +mk_container_headers(#{}) -> + #{}. + %% Examples bridge_v2_examples(Method) -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index 972294ef7..d135a087a 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -191,8 +191,7 @@ start_channel(State, #{ max_records := MaxRecords }, container := Container, - bucket := Bucket, - key := Key + bucket := Bucket } }) -> AggregOpts = #{ @@ -202,9 +201,9 @@ start_channel(State, #{ }, DeliveryOpts = #{ bucket => Bucket, - key => emqx_bridge_s3_aggreg_upload:mk_key_template(Key), + key => emqx_bridge_s3_aggreg_upload:mk_key_template(Parameters), container => Container, - upload_options => upload_options(Parameters), + upload_options => emqx_bridge_s3_aggreg_upload:mk_upload_options(Parameters), client_config => maps:get(client_config, State), uploader_config => maps:with([min_part_size, max_part_size], Parameters) }, diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 45a830294..c7eeee2bb 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -70,7 +70,7 @@ end_per_suite(Config) -> %% Testcases init_per_testcase(TestCase, Config) -> - ct:timetrap(timer:seconds(10)), + ct:timetrap(timer:seconds(15)), ok = snabbkaffe:start_trace(), TS = erlang:system_time(), Name = iolist_to_binary(io_lib:format("~s-~p", [TestCase, TS])), @@ -124,6 +124,9 @@ action_config(Name, ConnectorId, Bucket) -> <<"bucket">> => unicode:characters_to_binary(Bucket), <<"key">> => <<"${action}/${node}/${datetime.rfc3339}">>, <<"acl">> => <<"public_read">>, + <<"headers">> => #{ + <<"X-AMZ-Meta-Version">> => <<"42">> + }, <<"aggregation">> => #{ <<"time_interval">> => <<"4s">>, <<"max_records">> => ?CONF_MAX_RECORDS @@ -173,7 +176,11 @@ t_aggreg_upload(Config) -> [BridgeNameString, NodeString, _Datetime, _Seq = "0"], string:split(Key, "/", all) ), - _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + ?assertMatch( + #{content_type := "text/csv", "x-amz-meta-version" := "42"}, + Upload + ), %% Verify that column order is respected. ?assertMatch( {ok, [ diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 1199948d0..1fa6d31cd 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -102,6 +102,14 @@ fields(s3_upload) -> desc => ?DESC("acl"), required => false } + )}, + {headers, + hoconsc:mk( + map(), + #{ + required => false, + desc => ?DESC("upload_headers") + } )} ]; fields(s3_uploader) -> diff --git a/rel/i18n/emqx_s3_schema.hocon b/rel/i18n/emqx_s3_schema.hocon index 44f4bbc56..b8b963539 100644 --- a/rel/i18n/emqx_s3_schema.hocon +++ b/rel/i18n/emqx_s3_schema.hocon @@ -18,6 +18,13 @@ key.desc: key.label: """Object Key""" +upload_headers.label: +"""S3 object HTTP headers""" + +upload_headers.desc: +"""HTTP headers to include in the S3 object upload request.
+Useful to specify content type, content encoding, etc. of the S3 object.""" + host.desc: """The host of the S3 endpoint.""" From f6e5eea4f7b25e74fcd6264bb820da4c2a893443 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 26 Apr 2024 20:00:28 +0200 Subject: [PATCH 81/86] feat(s3-aggreg): handle delivery shutdowns gracefully --- .../src/emqx_bridge_s3_aggreg_delivery.erl | 164 ++++++++++++------ .../src/emqx_bridge_s3_aggregator.erl | 1 + .../emqx_bridge_s3_aggreg_upload_SUITE.erl | 37 +++- .../test/emqx_bridge_s3_test_helpers.erl | 6 + apps/emqx_s3/src/emqx_s3_client.erl | 19 +- 5 files changed, 168 insertions(+), 59 deletions(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl index c548226ab..02099dbec 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl @@ -12,11 +12,21 @@ -export([start_link/3]). %% Internal exports --export([run_delivery/3]). +-export([ + init/4, + loop/3 +]). -behaviour(emqx_template). -export([lookup/2]). +%% Sys +-export([ + system_continue/3, + system_terminate/4, + format_status/2 +]). + -record(delivery, { name :: _Name, container :: emqx_bridge_s3_aggreg_csv:container(), @@ -25,19 +35,23 @@ empty :: boolean() }). +-type state() :: #delivery{}. + %% start_link(Name, Buffer, Opts) -> - proc_lib:start_link(?MODULE, run_delivery, [Name, Buffer, Opts]). + proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]). %% -run_delivery(Name, Buffer, Opts) -> +-spec init(pid(), _Name, buffer(), _Opts :: map()) -> no_return(). +init(Parent, Name, Buffer, Opts) -> ?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}), Reader = open_buffer(Buffer), Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}), + _ = erlang:process_flag(trap_exit, true), ok = proc_lib:init_ack({ok, self()}), - loop_deliver(Delivery). + loop(Delivery, Parent, []). init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) -> #delivery{ @@ -48,53 +62,13 @@ init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) -> empty = true }. -loop_deliver(Delivery = #delivery{reader = Reader0}) -> - case emqx_bridge_s3_aggreg_buffer:read(Reader0) of - {Records = [#{} | _], Reader} -> - loop_deliver_records(Records, Delivery#delivery{reader = Reader}); - {[], Reader} -> - loop_deliver(Delivery#delivery{reader = Reader}); - eof -> - complete_delivery(Delivery); - {Unexpected, _Reader} -> - exit({buffer_unexpected_record, Unexpected}) - end. - -loop_deliver_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) -> - {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0), - {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), - loop_deliver_upload(Delivery#delivery{ - container = Container, - upload = Upload, - empty = false - }). - -loop_deliver_upload(Delivery = #delivery{upload = Upload0}) -> - case emqx_s3_upload:write(Upload0) of - {ok, Upload} -> - loop_deliver(Delivery#delivery{upload = Upload}); - {cont, Upload} -> - loop_deliver_upload(Delivery#delivery{upload = Upload}); +open_buffer(#buffer{filename = Filename}) -> + case file:open(Filename, [read, binary, raw]) of + {ok, FD} -> + {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD), + Reader; {error, Reason} -> - %% TODO: retries - _ = emqx_s3_upload:abort(Upload0), - exit({upload_failed, Reason}) - end. - -complete_delivery(#delivery{name = Name, empty = true}) -> - ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}), - exit({shutdown, {skipped, empty}}); -complete_delivery(#delivery{name = Name, container = Container, upload = Upload0}) -> - Trailer = emqx_bridge_s3_aggreg_csv:close(Container), - {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0), - case emqx_s3_upload:complete(Upload) of - {ok, Completed} -> - ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}), - ok; - {error, Reason} -> - %% TODO: retries - _ = emqx_s3_upload:abort(Upload), - exit({upload_failed, Reason}) + error({buffer_open_failed, Reason}) end. mk_container(#{type := csv, column_order := OrderOpt}) -> @@ -118,15 +92,91 @@ mk_upload( mk_object_key(Buffer, #{action := Name, key := Template}) -> emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}). -open_buffer(#buffer{filename = Filename}) -> - case file:open(Filename, [read, binary, raw]) of - {ok, FD} -> - {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD), - Reader; - {error, Reason} -> - error({buffer_open_failed, Reason}) +%% + +-spec loop(state(), pid(), [sys:debug_option()]) -> no_return(). +loop(Delivery, Parent, Debug) -> + %% NOTE: This function is mocked in tests. + receive + Msg -> handle_msg(Msg, Delivery, Parent, Debug) + after 0 -> + process_delivery(Delivery, Parent, Debug) end. +process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) -> + case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Records = [#{} | _], Reader} -> + Delivery1 = Delivery0#delivery{reader = Reader}, + Delivery2 = process_append_records(Records, Delivery1), + Delivery = process_write(Delivery2), + loop(Delivery, Parent, Debug); + {[], Reader} -> + Delivery = Delivery0#delivery{reader = Reader}, + loop(Delivery, Parent, Debug); + eof -> + process_complete(Delivery0); + {Unexpected, _Reader} -> + exit({buffer_unexpected_record, Unexpected}) + end. + +process_append_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) -> + {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0), + {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), + Delivery#delivery{ + container = Container, + upload = Upload, + empty = false + }. + +process_write(Delivery = #delivery{upload = Upload0}) -> + case emqx_s3_upload:write(Upload0) of + {ok, Upload} -> + Delivery#delivery{upload = Upload}; + {cont, Upload} -> + process_write(Delivery#delivery{upload = Upload}); + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload0), + exit({upload_failed, Reason}) + end. + +process_complete(#delivery{name = Name, empty = true}) -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}), + exit({shutdown, {skipped, empty}}); +process_complete(#delivery{name = Name, container = Container, upload = Upload0}) -> + Trailer = emqx_bridge_s3_aggreg_csv:close(Container), + {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0), + case emqx_s3_upload:complete(Upload) of + {ok, Completed} -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}), + ok; + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload), + exit({upload_failed, Reason}) + end. + +%% + +handle_msg({system, From, Msg}, Delivery, Parent, Debug) -> + sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery); +handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) -> + system_terminate(Reason, Parent, Debug, Delivery); +handle_msg(_Msg, Delivery, Parent, Debug) -> + loop(Parent, Debug, Delivery). + +-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return(). +system_continue(Parent, Debug, Delivery) -> + loop(Delivery, Parent, Debug). + +-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _. +system_terminate(_Reason, _Parent, _Debug, #delivery{upload = Upload}) -> + emqx_s3_upload:abort(Upload). + +-spec format_status(normal, Args :: [term()]) -> _StateFormatted. +format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) -> + Delivery#delivery{ + upload = emqx_s3_upload:format(Delivery#delivery.upload) + }. + %% -spec lookup(emqx_template:accessor(), {_Name, buffer()}) -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl index c49a29cb8..9d1ac5575 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl @@ -393,6 +393,7 @@ handle_delivery_exit(Buffer, Error, St = #st{name = Name}) -> filename => Buffer#buffer.filename, reason => Error }), + %% TODO: Retries? enqueue_status_error(Error, St). enqueue_status_error({upload_failed, Error}, St = #st{errors = QErrors}) -> diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index c7eeee2bb..07a4c2056 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -219,7 +219,6 @@ t_aggreg_upload_restart(Config) -> %% Check there's still only one upload. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), - %% Verify that column order is respected. ?assertMatch( {ok, [ _Header = [_ | _], @@ -285,6 +284,42 @@ t_aggreg_upload_restart_corrupted(Config) -> CSV ). +t_aggreg_pending_upload_restart(Config) -> + %% NOTE + %% This test verifies that the bridge will finish uploading a buffer file after + %% a restart. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send few large messages that will require multipart upload. + %% Ensure that they span multiple batch queries. + Payload = iolist_to_binary(lists:duplicate(128 * 1024, "PAYLOAD!")), + Messages = [{integer_to_binary(N), <<"a/b/c">>, Payload} || N <- lists:seq(1, 10)], + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages), 10), + %% Wait until the multipart upload is started. + {ok, #{key := ObjectKey}} = + ?block_until(#{?snk_kind := s3_client_multipart_started, bucket := Bucket}), + %% Stop the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + %% Verify that pending uploads have been gracefully aborted. + %% NOTE: Minio does not support multipart upload listing w/o prefix. + ?assertEqual( + [], + emqx_bridge_s3_test_helpers:list_pending_uploads(Bucket, ObjectKey) + ), + %% Restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check that delivery contains all the messages. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + [_Header | Rows] = fetch_parse_csv(Bucket, Key), + ?assertEqual( + Messages, + [{CID, Topic, PL} || [_TS, CID, Topic, PL | _] <- Rows] + ). + t_aggreg_next_rotate(Config) -> %% NOTE %% This is essentially a stress test that tries to verify that buffer rotation diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl index de19c8028..21729369b 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl @@ -43,6 +43,12 @@ get_object(Bucket, Key) -> AwsConfig = emqx_s3_test_helpers:aws_config(tcp), maps:from_list(erlcloud_s3:get_object(Bucket, Key, AwsConfig)). +list_pending_uploads(Bucket, Key) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + {ok, Props} = erlcloud_s3:list_multipart_uploads(Bucket, [{prefix, Key}], [], AwsConfig), + Uploads = proplists:get_value(uploads, Props), + lists:map(fun maps:from_list/1, Uploads). + %% File utilities truncate_at(Filename, Pos) -> diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index a415cf8d4..58029ae85 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -6,6 +6,7 @@ -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -include_lib("erlcloud/include/erlcloud_aws.hrl"). -export([ @@ -133,7 +134,13 @@ start_multipart( Headers = join_headers(BaseHeaders, maps:get(headers, UploadOpts, undefined)), case erlcloud_s3:start_multipart(Bucket, ECKey, ECOpts, Headers, AwsConfig) of {ok, Props} -> - {ok, response_property('uploadId', Props)}; + UploadId = response_property('uploadId', Props), + ?tp(s3_client_multipart_started, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), + {ok, UploadId}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} @@ -177,6 +184,11 @@ complete_multipart( ) of ok -> + ?tp(s3_client_multipart_completed, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), ok; {error, Reason} -> ?SLOG(debug, #{msg => "complete_multipart_fail", key => Key, reason => Reason}), @@ -193,6 +205,11 @@ abort_multipart( ) -> case erlcloud_s3:abort_multipart(Bucket, erlcloud_key(Key), UploadId, [], Headers, AwsConfig) of ok -> + ?tp(s3_client_multipart_aborted, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), ok; {error, Reason} -> ?SLOG(debug, #{msg => "abort_multipart_fail", key => Key, reason => Reason}), From a1a313d992b8b7b7ed31b46ba971011c6a09813d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 Apr 2024 09:53:21 +0200 Subject: [PATCH 82/86] fix(s3-aggeg): apply CSV column order setting consistently Otherwise, columns that are part of column order could appear and disappear from consecutive uploads, depending on if they are part of the very first buffered event or not. --- apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl | 3 +-- .../test/emqx_bridge_s3_aggreg_upload_SUITE.erl | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl index 31ce1fcc7..a6e25a87c 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl @@ -58,9 +58,8 @@ close(#csv{}) -> mk_columns(Record, #csv{order = ColumnOrder}) -> Columns = lists:sort(maps:keys(Record)), - OrderedFirst = [CO || CO <- ColumnOrder, lists:member(CO, Columns)], Unoredered = Columns -- ColumnOrder, - OrderedFirst ++ Unoredered. + ColumnOrder ++ Unoredered. -spec emit_header([column()], container()) -> iodata(). emit_header([C], #csv{delimiter = Delim}) -> diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 07a4c2056..85fdd844b 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -26,7 +26,8 @@ <<"publish_received_at">>, <<"clientid">>, <<"topic">>, - <<"payload">> + <<"payload">>, + <<"empty">> | T ]). @@ -185,9 +186,9 @@ t_aggreg_upload(Config) -> ?assertMatch( {ok, [ ?CONF_COLUMN_ORDER(_), - [TS, <<"C1">>, T1, P1 | _], - [TS, <<"C2">>, T2, P2 | _], - [TS, <<"C3">>, T3, P3 | _] + [TS, <<"C1">>, T1, P1, <<>> | _], + [TS, <<"C2">>, T2, P2, <<>> | _], + [TS, <<"C3">>, T3, P3, <<>> | _] ]}, erl_csv:decode(Content) ). From 83366cbed0ca14e82ee083dd90ab3a623b01d0b2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 Apr 2024 10:41:25 +0200 Subject: [PATCH 83/86] fix(s3-aggreg): ensure action works in Rule SQL contexts --- .../src/emqx_bridge_s3_aggreg_csv.erl | 12 ++--- .../emqx_bridge_s3_aggreg_upload_SUITE.erl | 50 +++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl index a6e25a87c..96d924912 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl @@ -57,7 +57,7 @@ close(#csv{}) -> %% mk_columns(Record, #csv{order = ColumnOrder}) -> - Columns = lists:sort(maps:keys(Record)), + Columns = [emqx_utils_conv:bin(C) || C <- lists:sort(maps:keys(Record))], Unoredered = Columns -- ColumnOrder, ColumnOrder ++ Unoredered. @@ -81,11 +81,11 @@ emit_row(#{}, [], #csv{delimiter = Delim}) -> [Delim]. emit_cell(Column, Record, CSV) -> - case maps:get(Column, Record, undefined) of - undefined -> - _Empty = ""; - Value -> - encode_cell(emqx_template:to_string(Value), CSV) + case emqx_template:lookup(Column, Record) of + {ok, Value} -> + encode_cell(emqx_template:to_string(Value), CSV); + {error, undefined} -> + _Empty = "" end. encode_cell(V, #csv{quoting_mp = MP}) -> diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 85fdd844b..6577b45ed 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -193,6 +193,56 @@ t_aggreg_upload(Config) -> erl_csv:decode(Content) ). +t_aggreg_upload_rule(Config) -> + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + ClientID = emqx_utils_conv:bin(?FUNCTION_NAME), + %% Create a bridge with the sample configuration and a simple SQL rule. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + ?assertMatch( + {ok, _Rule}, + emqx_bridge_v2_testlib:create_rule_and_action_http(?BRIDGE_TYPE, <<>>, Config, #{ + sql => << + "SELECT" + " *," + " strlen(payload) as psize," + " unix_ts_to_rfc3339(publish_received_at, 'millisecond') as publish_received_at" + " FROM 's3/#'" + >> + }) + ), + ok = lists:foreach(fun emqx:publish/1, [ + emqx_message:make(?FUNCTION_NAME, T1 = <<"s3/m1">>, P1 = <<"[HELLO]">>), + emqx_message:make(?FUNCTION_NAME, T2 = <<"s3/m2">>, P2 = <<"[WORLD]">>), + emqx_message:make(?FUNCTION_NAME, T3 = <<"s3/empty">>, P3 = <<>>), + emqx_message:make(?FUNCTION_NAME, <<"not/s3">>, <<"should not be here">>) + ]), + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check the uploaded objects. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + _CSV = [Header | Rows] = fetch_parse_csv(Bucket, Key), + %% Verify that column order is respected and event fields are preserved. + ?assertMatch(?CONF_COLUMN_ORDER(_), Header), + ?assertEqual( + [<<"event">>, <<"qos">>, <<"psize">>], + [C || C <- [<<"event">>, <<"qos">>, <<"psize">>], lists:member(C, Header)] + ), + %% Verify that all the matching messages are present. + ?assertMatch( + [ + [_TS1, ClientID, T1, P1 | _], + [_TS2, ClientID, T2, P2 | _], + [_TS3, ClientID, T3, P3 | _] + ], + Rows + ), + %% Verify that timestamp column now has RFC3339 format. + [_Row = [TS1 | _] | _Rest] = Rows, + ?assert( + is_integer(emqx_rule_funcs:rfc3339_to_unix_ts(TS1, millisecond)), + TS1 + ). + t_aggreg_upload_restart(Config) -> %% NOTE %% This test verifies that the bridge will reuse existing aggregation buffer From 4bea938ef25150370459262d1204f30d7039f6de Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 Apr 2024 14:41:42 +0200 Subject: [PATCH 84/86] fix(s3-csv): clarify naming of CSV container concepts Co-Authored-By: Ilya Averyanov --- .../src/emqx_bridge_s3_aggreg_csv.erl | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl index 96d924912..33895d8c1 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl @@ -16,14 +16,22 @@ -record(csv, { columns :: [binary()] | undefined, - order :: [binary()], - separator :: char() | iodata(), - delimiter :: char() | iodata(), + column_order :: [binary()], + %% A string or character that separates each field in a record from the next. + %% Default: "," + field_separator :: char() | iodata(), + %% A string or character that delimits boundaries of a record. + %% Default: "\n" + record_delimiter :: char() | iodata(), quoting_mp :: _ReMP }). -type container() :: #csv{}. --type options() :: #{column_order => [column()]}. + +-type options() :: #{ + %% Which columns have to be ordered first in the resulting CSV? + column_order => [column()] +}. -type record() :: emqx_bridge_s3_aggregator:record(). -type column() :: binary(). @@ -34,9 +42,9 @@ new(Opts) -> {ok, MP} = re:compile("[\\[\\],\\r\\n\"]", [unicode]), #csv{ - order = maps:get(column_order, Opts, []), - separator = $,, - delimiter = $\n, + column_order = maps:get(column_order, Opts, []), + field_separator = $,, + record_delimiter = $\n, quoting_mp = MP }. @@ -56,28 +64,28 @@ close(#csv{}) -> %% -mk_columns(Record, #csv{order = ColumnOrder}) -> +mk_columns(Record, #csv{column_order = ColumnOrder}) -> Columns = [emqx_utils_conv:bin(C) || C <- lists:sort(maps:keys(Record))], Unoredered = Columns -- ColumnOrder, ColumnOrder ++ Unoredered. -spec emit_header([column()], container()) -> iodata(). -emit_header([C], #csv{delimiter = Delim}) -> +emit_header([C], #csv{record_delimiter = Delim}) -> [C, Delim]; -emit_header([C | Rest], CSV = #csv{separator = Sep}) -> +emit_header([C | Rest], CSV = #csv{field_separator = Sep}) -> [C, Sep | emit_header(Rest, CSV)]; -emit_header([], #csv{delimiter = Delim}) -> +emit_header([], #csv{record_delimiter = Delim}) -> [Delim]. -spec emit_row(record(), container()) -> iodata(). emit_row(Record, CSV = #csv{columns = Columns}) -> emit_row(Record, Columns, CSV). -emit_row(Record, [C], CSV = #csv{delimiter = Delim}) -> +emit_row(Record, [C], CSV = #csv{record_delimiter = Delim}) -> [emit_cell(C, Record, CSV), Delim]; -emit_row(Record, [C | Rest], CSV = #csv{separator = Sep}) -> +emit_row(Record, [C | Rest], CSV = #csv{field_separator = Sep}) -> [emit_cell(C, Record, CSV), Sep | emit_row(Record, Rest, CSV)]; -emit_row(#{}, [], #csv{delimiter = Delim}) -> +emit_row(#{}, [], #csv{record_delimiter = Delim}) -> [Delim]. emit_cell(Column, Record, CSV) -> From 6d3add3646b3859a413c7f9e7b798a10e9ad6c24 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 29 Apr 2024 15:23:57 +0200 Subject: [PATCH 85/86] fix(s3-aggreg): do not handle `{error, closed}` on buffer write Because it's not really something `file:write/2` is supposed to return. --- apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl index 9d1ac5575..47ecdeb4a 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl @@ -94,7 +94,7 @@ write_records(Name, Buffer = #buffer{fd = Writer}, Records) -> ok -> ?tp(s3_aggreg_records_written, #{action => Name, records => Records}), ok; - {error, Reason} when Reason == terminated orelse Reason == closed -> + {error, terminated} -> BufferNext = rotate_buffer(Name, Buffer), write_records(Name, BufferNext, Records); {error, _} = Error -> From 1974ec15ecc28050315908244a8ef190cbccd76a Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 30 Apr 2024 12:12:35 +0200 Subject: [PATCH 86/86] fix(client_attrs): fix client_attrs extraction loop --- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/test/emqx_client_SUITE.erl | 6 +++++- apps/emqx_utils/test/emqx_variform_tests.erl | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index ec49d165c..05358f889 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1636,7 +1636,7 @@ maybe_set_client_initial_attrs(ConnPkt, #{zone := Zone} = ClientInfo) -> initialize_client_attrs(Inits, ClientInfo) -> lists:foldl( fun(#{expression := Variform, set_as_attr := Name}, Acc) -> - Attrs = maps:get(client_attrs, ClientInfo, #{}), + Attrs = maps:get(client_attrs, Acc, #{}), case emqx_variform:render(Variform, ClientInfo) of {ok, <<>>} -> ?SLOG( diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index f0afe6195..d47162ed7 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -422,6 +422,10 @@ t_client_attr_from_user_property(_Config) -> #{ expression => Compiled, set_as_attr => <<"group">> + }, + #{ + expression => Compiled, + set_as_attr => <<"group2">> } ]), SslConf = emqx_common_test_helpers:client_mtls('tlsv1.3'), @@ -436,7 +440,7 @@ t_client_attr_from_user_property(_Config) -> {ok, _} = emqtt:connect(Client), %% assert only two chars are extracted ?assertMatch( - #{clientinfo := #{client_attrs := #{<<"group">> := <<"g1">>}}}, + #{clientinfo := #{client_attrs := #{<<"group">> := <<"g1">>, <<"group2">> := <<"g1">>}}}, emqx_cm:get_chan_info(ClientId) ), emqtt:disconnect(Client). diff --git a/apps/emqx_utils/test/emqx_variform_tests.erl b/apps/emqx_utils/test/emqx_variform_tests.erl index 59c39cf1c..2e3c6c4d5 100644 --- a/apps/emqx_utils/test/emqx_variform_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_tests.erl @@ -275,7 +275,7 @@ to_range_badarg_test_() -> ]. iif_test_() -> - %% if clientid has to words separated by a -, take the suffix, and append with `/#` + %% if clientid has two words separated by a -, take the suffix, and append with `/#` Expr1 = "iif(nth(2,tokens(clientid,'-')),concat([nth(2,tokens(clientid,'-')),'/#']),'')", [ ?_assertEqual({ok, <<"yes-A">>}, render("iif(a,'yes-A','no-A')", #{a => <<"x">>})),