yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

docs(s3-bridge): document aggregated upload action schema

This commit is contained in:
Andrew Mayorov 2024-04-26 12:50:41 +02:00
parent ccbcc0c4e3
commit 5b15b2d641
No known key found for this signature in database
GPG Key ID: 2837C62ACFBFED5D
5 changed files with 126 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl
View File

@ -60,7 +60,7 @@ fields(?ACTION) ->
?R_REF(s3_aggregated_upload_parameters),
#{
required => true,
desc => ?DESC(s3_aggregated_upload)
desc => ?DESC(s3_aggregated_upload_parameters)
}
),
#{
@ -68,32 +68,36 @@ fields(?ACTION) ->
}
);
fields(s3_aggregated_upload_parameters) ->
[
{container,
hoconsc:mk(
%% TODO: Support selectors once there are more than one container.
hoconsc:union(fun
(all_union_members) -> [?REF(s3_container_csv)];
({value, _Valur}) -> [?REF(s3_container_csv)]
end),
#{
required => true,
default => #{<<"type">> => <<"csv">>},
desc => ?DESC(s3_aggregated_container)
}
)},
{aggregation,
hoconsc:mk(
?REF(s3_aggregation),
#{
required => true,
desc => ?DESC(s3_aggregation)
}
)}
] ++
emqx_s3_schema:fields(s3_upload) ++
emqx_s3_schema:fields(s3_uploader);
fields(s3_container_csv) ->
lists:append([
[
{container,
hoconsc:mk(
%% TODO: Support selectors once there are more than one container.
hoconsc:union(fun
(all_union_members) -> [?REF(s3_aggregated_container_csv)];
({value, _Valur}) -> [?REF(s3_aggregated_container_csv)]
end),
#{
required => true,
default => #{<<"type">> => <<"csv">>},
desc => ?DESC(s3_aggregated_container)
}
)},
{aggregation,
hoconsc:mk(
?REF(s3_aggregation),
#{
required => true,
desc => ?DESC(s3_aggregation)
}
)}
],
emqx_resource_schema:override(emqx_s3_schema:fields(s3_upload), [
{key, #{desc => ?DESC(s3_aggregated_upload_key)}}
]),
emqx_s3_schema:fields(s3_uploader)
]);
fields(s3_aggregated_container_csv) ->
[
{type,
hoconsc:mk(
@ -142,10 +146,13 @@ fields(s3_aggreg_upload_resource_opts) ->
{batch_time, #{default => ?DEFAULT_BATCH_TIME}}
]).
desc(?ACTION) ->
?DESC(s3_aggregated_upload);
desc(s3_aggregated_upload_parameters) ->
?DESC(s3_aggregated_upload_parameters);
desc(Name) when
Name == s3_aggregated_upload;
Name == s3_aggregated_upload_parameters;
Name == s3_aggregation;
Name == s3_aggregated_container_csv
->
?DESC(Name);
desc(s3_aggreg_upload_resource_opts) ->
?DESC(emqx_resource_schema, resource_opts);
desc(_Name) ->

11
apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl
View File

@ -80,12 +80,13 @@ fields(s3_action_resource_opts) ->
emqx_bridge_v2_schema:action_resource_opts_fields()
).
desc(?ACTION) ->
desc(s3) ->
?DESC(s3_upload);
desc(s3_upload) ->
?DESC(s3_upload);
desc(s3_upload_parameters) ->
?DESC(s3_upload_parameters);
desc(Name) when
Name == s3_upload;
Name == s3_upload_parameters
->
?DESC(Name);
desc(s3_action_resource_opts) ->
?DESC(emqx_resource_schema, resource_opts);
desc(_Name) ->

15
rel/i18n/emqx_bridge_s3.hocon
View File

@ -5,19 +5,4 @@ config_connector.label:
config_connector.desc:
"""Configuration for a connector to S3 API compatible storage service."""
s3_upload.label:
"""S3 Simple Upload"""
s3_upload.desc:
"""Action to upload a single object to the S3 service."""
s3_upload_parameters.label:
"""S3 Upload action parameters"""
s3_upload_parameters.desc:
"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content."""
s3_object_content.label:
"""S3 Object Content"""
s3_object_content.desc:
"""Content of the S3 object being uploaded. Supports templates."""
}

64
rel/i18n/emqx_bridge_s3_aggreg_upload.hocon Normal file
View File

@ -0,0 +1,64 @@
emqx_bridge_s3_aggreg_upload {
s3_aggregated_upload.label:
"""S3 Aggregated Upload"""
s3_aggregated_upload.desc:
"""Action that enables time-based aggregation of incoming events and uploading them to the S3 service as a single object."""
s3_aggregated_upload_parameters.label:
"""S3 Aggregated Upload action parameters"""
s3_aggregated_upload_parameters.desc:
"""Set of parameters for the aggregated upload action."""
s3_aggregation.label:
"""Aggregation parameters"""
s3_aggregation.desc:
"""Set of parameters governing the aggregation process."""
s3_aggregation_interval.label:
"""Time interval"""
s3_aggregation_interval.desc:
"""Amount of time events will be aggregated in a single object before uploading."""
s3_aggregation_max_records.label:
"""Maximum number of records"""
s3_aggregation_max_records.desc:
"""Number of records (events) allowed per each aggregated object. Each aggregated upload will contain no more than that number of events, but may contain less.<br/>
If event rate is high enough, there obviously may be more than one aggregated upload during the same time interval. These uploads will have different, but consecutive sequence numbers, which will be a part of S3 object key."""
s3_aggregated_container.label:
"""Container for aggregated events"""
s3_aggregated_container.desc:
"""Settings governing the file format of an upload containing aggregated events."""
s3_aggregated_container_csv.label:
"""CSV container"""
s3_aggregated_container_csv.desc:
"""Records (events) will be aggregated and uploaded as a CSV file."""
s3_aggregated_container_csv_column_order.label:
"""CSV column order"""
s3_aggregated_container_csv_column_order.desc:
"""Event fields that will be ordered first as columns in the resulting CSV file.<br/>
Regardless of this setting, resulting CSV will contain all the fields of aggregated events, but all the columns not explicitly mentioned here will be ordered after the ones listed here in the lexicographical order."""
s3_aggregated_upload_key.label:
"""S3 object key template"""
s3_aggregated_upload_key.desc:
"""Template for the S3 object key of an aggregated upload.<br/>
Template may contain placeholders for the following variables:
<ul>
<li><code>${action}</code>: name of the action (required).<li/>
<li><code>${node}</code>: name of the EMQX node conducting the upload (required).<li/>
<li><code>${datetime.{format}}</code>: date and time when aggregation started, formatted according to the <code>{format}</code> string (required):
<ul>
<li><code>${datetime.rfc3339utc}</code>: RFC3339-formatted date and time in UTC,<li/>
<li><code>${datetime.rfc3339}</code>: RFC3339-formatted date and time in local timezone,<li/>
<li><code>${datetime.unix}</code>: Unix timestamp.<li/>
</ul>
<li/>
<li><code>${datetime_until.{format}}</code>: date and time when aggregation ended, with the same formatting options.<li/>
<li><code>${sequence}</code>: sequence number of the aggregated upload within the same time interval (required).<li/>
</ul>
All other placeholders are considered invalid. Note that placeholders marked as required will be added as a path suffix to the S3 object key if they are missing from the template."""
}

18
rel/i18n/emqx_bridge_s3_upload.hocon Normal file
View File

@ -0,0 +1,18 @@
emqx_bridge_s3_upload {
s3_upload.label:
"""S3 Simple Upload"""
s3_upload.desc:
"""Action to upload a single object to the S3 service."""
s3_upload_parameters.label:
"""S3 Upload action parameters"""
s3_upload_parameters.desc:
"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content."""
s3_object_content.label:
"""S3 Object Content"""
s3_object_content.desc:
"""Content of the S3 object being uploaded. Supports templates."""
}