yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(s3-aggeg): apply CSV column order setting consistently 

Otherwise, columns that are part of column order could appear and
disappear from consecutive uploads, depending on if they are part
of the very first buffered event or not.
This commit is contained in:
Andrew Mayorov 2024-04-29 09:53:21 +02:00
parent f6e5eea4f7
commit a1a313d992
No known key found for this signature in database
GPG Key ID: 2837C62ACFBFED5D
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl
View File

@ -58,9 +58,8 @@ close(#csv{}) ->
mk_columns(Record, #csv{order = ColumnOrder}) -> mk_columns(Record, #csv{order = ColumnOrder}) ->
Columns = lists:sort(maps:keys(Record)), Columns = lists:sort(maps:keys(Record)),
OrderedFirst = [CO || CO <- ColumnOrder, lists:member(CO, Columns)],
Unoredered = Columns -- ColumnOrder, Unoredered = Columns -- ColumnOrder,
OrderedFirst ++ Unoredered. ColumnOrder ++ Unoredered.
-spec emit_header([column()], container()) -> iodata(). -spec emit_header([column()], container()) -> iodata().
emit_header([C], #csv{delimiter = Delim}) -> emit_header([C], #csv{delimiter = Delim}) ->

9
apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl
View File

@ -26,7 +26,8 @@
<<"publish_received_at">>, <<"publish_received_at">>,
<<"clientid">>, <<"clientid">>,
<<"topic">>, <<"topic">>,
<<"payload">> <<"payload">>,
<<"empty">>
| T | T
]). ]).
@ -185,9 +186,9 @@ t_aggreg_upload(Config) ->
?assertMatch( ?assertMatch(
{ok, [ {ok, [
?CONF_COLUMN_ORDER(_), ?CONF_COLUMN_ORDER(_),
[TS, <<"C1">>, T1, P1 | _], [TS, <<"C1">>, T1, P1, <<>> | _],
[TS, <<"C2">>, T2, P2 | _], [TS, <<"C2">>, T2, P2, <<>> | _],
[TS, <<"C3">>, T3, P3 | _] [TS, <<"C3">>, T3, P3, <<>> | _]
]}, ]},
erl_csv:decode(Content) erl_csv:decode(Content)
). ).