yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10775 from zmstone/0522-merge-release-50-to-master 

0522 merge release 50 to master
This commit is contained in:
zhongwencool 2023-05-23 13:43:33 +08:00 committed by GitHub
parent 1f44dd4579 1deb3c50c8
commit 3281b5fc33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
213 changed files with 20147 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/docker-compose-file/.env
View File

@ -7,6 +7,7 @@ INFLUXDB_TAG=2.5.0
TDENGINE_TAG=3.0.2.4
DYNAMO_TAG=1.21.0
CASSANDRA_TAG=3.11.6
MINIO_TAG=RELEASE.2023-03-20T20-16-18Z
OPENTS_TAG=9aa7f88
MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server

21
.ci/docker-compose-file/docker-compose-minio-tcp.yaml Normal file
View File

@ -0,0 +1,21 @@
version: '3.7'
services:
minio:
hostname: minio
image: quay.io/minio/minio:${MINIO_TAG}
command: server --address ":9000" --console-address ":9001" /minio-data
expose:
- "9000"
- "9001"
ports:
- "9000:9000"
- "9001:9001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 5s
retries: 3
networks:
emqx_bridge:

23
.ci/docker-compose-file/docker-compose-minio-tls.yaml Normal file
View File

@ -0,0 +1,23 @@
version: '3.7'
services:
minio_tls:
hostname: minio-tls
image: quay.io/minio/minio:${MINIO_TAG}
command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data
volumes:
- ./certs/server.crt:/etc/certs/public.crt
- ./certs/server.key:/etc/certs/private.key
expose:
- "9100"
- "9101"
ports:
- "9100:9100"
- "9101:9101"
healthcheck:
test: ["CMD", "curl", "-k", "-f", "https://localhost:9100/minio/health/live"]
interval: 30s
timeout: 5s
retries: 3
networks:
emqx_bridge:

18
.ci/docker-compose-file/docker-compose-toxiproxy.yaml
View File

@ -13,19 +13,37 @@ services:
volumes:
- "./toxiproxy.json:/config/toxiproxy.json"
ports:
# Toxiproxy management API
- 8474:8474
# InfluxDB
- 8086:8086
# InfluxDB TLS
- 8087:8087
# SQL Server
- 11433:1433
# MySQL
- 13306:3306
# MySQL TLS
- 13307:3307
# PostgreSQL
- 15432:5432
# PostgreSQL TLS
- 15433:5433
# TDEngine
- 16041:6041
# DynamoDB
- 18000:8000
# RocketMQ
- 19876:9876
# Cassandra
- 19042:9042
# Cassandra TLS
- 19142:9142
# S3
- 19000:19000
# S3 TLS
- 19100:19100
# IOTDB
- 14242:4242
- 28080:18080
command:

12
.ci/docker-compose-file/toxiproxy.json
View File

@ -131,5 +131,17 @@
"listen": "0.0.0.0:18080",
"upstream": "iotdb:18080",
"enabled": true
},
{
"name": "minio_tcp",
"listen": "0.0.0.0:19000",
"upstream": "minio:9000",
"enabled": true
},
{
"name": "minio_tls",
"listen": "0.0.0.0:19100",
"upstream": "minio-tls:9100",
"enabled": true
}
]

1
.github/CODEOWNERS vendored
View File

@ -8,6 +8,7 @@
/apps/emqx_connector/ @emqx/emqx-review-board @JimMoen
/apps/emqx_dashboard/ @emqx/emqx-review-board @JimMoen @lafirest
/apps/emqx_exhook/ @emqx/emqx-review-board @JimMoen @lafirest
/apps/emqx_ft/ @emqx/emqx-review-board @savonarola @keynslug
/apps/emqx_gateway/ @emqx/emqx-review-board @lafirest
/apps/emqx_management/ @emqx/emqx-review-board @lafirest @sstrigler
/apps/emqx_plugin_libs/ @emqx/emqx-review-board @lafirest

53
.github/workflows/run_conf_tests.yaml vendored Normal file
View File

@ -0,0 +1,53 @@
name: Run Configuration tests
concurrency:
group: test-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches:
- master
- 'ci/**'
tags:
- v*
- e*
pull_request:
env:
IS_CI: "yes"
jobs:
run_conf_tests:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
path: source
- name: build_package
working-directory: source
run: |
make ${{ matrix.profile }}
- name: run_tests
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
run: |
./scripts/conf-test/run.sh
- name: print_erlang_log
if: failure()
run: |
cat source/_build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.profile }}
path: source/_build/${{ matrix.profile }}/rel/emqx/logs

1
.github/workflows/run_test_cases.yaml vendored
View File

@ -195,6 +195,7 @@ jobs:
INFLUXDB_TAG: "2.5.0"
TDENGINE_TAG: "3.0.2.4"
OPENTS_TAG: "9aa7f88"
MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z"
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}

2
Makefile
View File

@ -16,7 +16,7 @@ endif
# Dashbord version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.2.4-1
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7-beta.3
# `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
# In make 4.4+, for backward-compatibility the value from the original environment is used.

2
apps/emqx/include/emqx_release.hrl
View File

@ -35,7 +35,7 @@
-define(EMQX_RELEASE_CE, "5.0.25").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.0.4-alpha.1").
-define(EMQX_RELEASE_EE, "5.0.4-alpha.2").
%% the HTTP API version
-define(EMQX_API_VERSION, "5.0").

4
apps/emqx/priv/bpapi.versions
View File

@ -9,12 +9,16 @@
{emqx_bridge,4}.
{emqx_broker,1}.
{emqx_cm,1}.
{emqx_cm,2}.
{emqx_conf,1}.
{emqx_conf,2}.
{emqx_dashboard,1}.
{emqx_delayed,1}.
{emqx_eviction_agent,1}.
{emqx_exhook,1}.
{emqx_ft_storage_exporter_fs,1}.
{emqx_ft_storage_fs,1}.
{emqx_ft_storage_fs_reader,1}.
{emqx_gateway_api_listeners,1}.
{emqx_gateway_cm,1}.
{emqx_gateway_http,1}.

456
apps/emqx/src/emqx_calendar.erl Normal file
View File

@ -0,0 +1,456 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_calendar).
-define(SECONDS_PER_MINUTE, 60).
-define(SECONDS_PER_HOUR, 3600).
-define(SECONDS_PER_DAY, 86400).
-define(DAYS_PER_YEAR, 365).
-define(DAYS_PER_LEAP_YEAR, 366).
-define(DAYS_FROM_0_TO_1970, 719528).
-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY).
-export([
formatter/1,
format/3,
format/4,
parse/3,
offset_second/1
]).
-define(DATE_PART, [
year,
month,
day,
hour,
minute,
second,
nanosecond,
millisecond,
microsecond
]).
-define(DATE_ZONE_NAME, [
timezone,
timezone1,
timezone2
]).
formatter(FormatterStr) when is_list(FormatterStr) ->
formatter(list_to_binary(FormatterStr));
formatter(FormatterBin) when is_binary(FormatterBin) ->
do_formatter(FormatterBin, []).
offset_second(Offset) ->
offset_second_(Offset).
format(Time, Unit, Formatter) ->
format(Time, Unit, undefined, Formatter).
format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) ->
format(Time, Unit, Offset, formatter(FormatterBin));
format(Time, Unit, Offset, Formatter) ->
do_format(Time, time_unit(Unit), offset_second(Offset), Formatter).
parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
parse(DateStr, Unit, formatter(FormatterBin));
parse(DateStr, Unit, Formatter) ->
do_parse(DateStr, Unit, Formatter).
%% -------------------------------------------------------------------------------------------------
%% internal
time_unit(second) -> second;
time_unit(millisecond) -> millisecond;
time_unit(microsecond) -> microsecond;
time_unit(nanosecond) -> nanosecond;
time_unit("second") -> second;
time_unit("millisecond") -> millisecond;
time_unit("microsecond") -> microsecond;
time_unit("nanosecond") -> nanosecond;
time_unit(<<"second">>) -> second;
time_unit(<<"millisecond">>) -> millisecond;
time_unit(<<"microsecond">>) -> microsecond;
time_unit(<<"nanosecond">>) -> nanosecond.
%% -------------------------------------------------------------------------------------------------
%% internal: format part
do_formatter(<<>>, Formatter) ->
lists:reverse(Formatter);
do_formatter(<<"%Y", Tail/binary>>, Formatter) ->
do_formatter(Tail, [year | Formatter]);
do_formatter(<<"%m", Tail/binary>>, Formatter) ->
do_formatter(Tail, [month | Formatter]);
do_formatter(<<"%d", Tail/binary>>, Formatter) ->
do_formatter(Tail, [day | Formatter]);
do_formatter(<<"%H", Tail/binary>>, Formatter) ->
do_formatter(Tail, [hour | Formatter]);
do_formatter(<<"%M", Tail/binary>>, Formatter) ->
do_formatter(Tail, [minute | Formatter]);
do_formatter(<<"%S", Tail/binary>>, Formatter) ->
do_formatter(Tail, [second | Formatter]);
do_formatter(<<"%N", Tail/binary>>, Formatter) ->
do_formatter(Tail, [nanosecond | Formatter]);
do_formatter(<<"%3N", Tail/binary>>, Formatter) ->
do_formatter(Tail, [millisecond | Formatter]);
do_formatter(<<"%6N", Tail/binary>>, Formatter) ->
do_formatter(Tail, [microsecond | Formatter]);
do_formatter(<<"%z", Tail/binary>>, Formatter) ->
do_formatter(Tail, [timezone | Formatter]);
do_formatter(<<"%:z", Tail/binary>>, Formatter) ->
do_formatter(Tail, [timezone1 | Formatter]);
do_formatter(<<"%::z", Tail/binary>>, Formatter) ->
do_formatter(Tail, [timezone2 | Formatter]);
do_formatter(<<Char:8, Tail/binary>>, [Str | Formatter]) when is_list(Str) ->
do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]);
do_formatter(<<Char:8, Tail/binary>>, Formatter) ->
do_formatter(Tail, [[Char] | Formatter]).
offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond;
offset_second_(undefined) ->
0;
offset_second_("local") ->
offset_second_(local);
offset_second_(<<"local">>) ->
offset_second_(local);
offset_second_(local) ->
UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second),
LocalTime = erlang:universaltime_to_localtime(UniversalTime),
LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime),
UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime),
LocalSecs - UniversalSecs;
offset_second_(Offset) when is_binary(Offset) ->
offset_second_(erlang:binary_to_list(Offset));
offset_second_("Z") ->
0;
offset_second_("z") ->
0;
offset_second_(Offset) when is_list(Offset) ->
Sign = hd(Offset),
((Sign == $+) orelse (Sign == $-)) orelse
error({bad_time_offset, Offset}),
Signs = #{$+ => 1, $- => -1},
PosNeg = maps:get(Sign, Signs),
[Sign | HM] = Offset,
{HourStr, MinuteStr, SecondStr} =
case string:tokens(HM, ":") of
[H, M] ->
{H, M, "0"};
[H, M, S] ->
{H, M, S};
[HHMM] when erlang:length(HHMM) == 4 ->
{string:sub_string(HHMM, 1, 2), string:sub_string(HHMM, 3, 4), "0"};
_ ->
error({bad_time_offset, Offset})
end,
Hour = erlang:list_to_integer(HourStr),
Minute = erlang:list_to_integer(MinuteStr),
Second = erlang:list_to_integer(SecondStr),
(Hour =< 23) orelse error({bad_time_offset_hour, Hour}),
(Minute =< 59) orelse error({bad_time_offset_minute, Minute}),
(Second =< 59) orelse error({bad_time_offset_second, Second}),
PosNeg * (Hour * 3600 + Minute * 60 + Second).
do_format(Time, Unit, Offset, Formatter) ->
Adjustment = erlang:convert_time_unit(Offset, second, Unit),
AdjustedTime = Time + Adjustment,
Factor = factor(Unit),
Secs = AdjustedTime div Factor,
DateTime = system_time_to_datetime(Secs),
{{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
Date = #{
year => padding(Year, 4),
month => padding(Month, 2),
day => padding(Day, 2),
hour => padding(Hour, 2),
minute => padding(Min, 2),
second => padding(Sec, 2),
millisecond => trans_x_second(Unit, millisecond, Time),
microsecond => trans_x_second(Unit, microsecond, Time),
nanosecond => trans_x_second(Unit, nanosecond, Time)
},
Timezones = formatter_timezones(Offset, Formatter, #{}),
DateWithZone = maps:merge(Date, Timezones),
[maps:get(Key, DateWithZone, Key) || Key <- Formatter].
formatter_timezones(_Offset, [], Zones) ->
Zones;
formatter_timezones(Offset, [Timezone | Formatter], Zones) ->
case lists:member(Timezone, [timezone, timezone1, timezone2]) of
true ->
NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)},
formatter_timezones(Offset, Formatter, NZones);
false ->
formatter_timezones(Offset, Formatter, Zones)
end.
offset_to_timezone(Offset, Timezone) ->
Sign =
case Offset >= 0 of
true ->
$+;
false ->
$-
end,
{H, M, S} = seconds_to_time(abs(Offset)),
%% TODO: Support zone define %:::z
%% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30).
case Timezone of
timezone ->
%% +0800
io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]);
timezone1 ->
%% +08:00
io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]);
timezone2 ->
%% +08:00:00
io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S])
end.
factor(second) -> 1;
factor(millisecond) -> 1000;
factor(microsecond) -> 1000000;
factor(nanosecond) -> 1000000000.
system_time_to_datetime(Seconds) ->
gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970).
gregorian_seconds_to_datetime(Secs) when Secs >= 0 ->
Days = Secs div ?SECONDS_PER_DAY,
Rest = Secs rem ?SECONDS_PER_DAY,
{gregorian_days_to_date(Days), seconds_to_time(Rest)}.
seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY ->
Secs0 = Secs rem ?SECONDS_PER_DAY,
Hour = Secs0 div ?SECONDS_PER_HOUR,
Secs1 = Secs0 rem ?SECONDS_PER_HOUR,
Minute = Secs1 div ?SECONDS_PER_MINUTE,
Second = Secs1 rem ?SECONDS_PER_MINUTE,
{Hour, Minute, Second}.
gregorian_days_to_date(Days) ->
{Year, DayOfYear} = day_to_year(Days),
{Month, DayOfMonth} = year_day_to_date(Year, DayOfYear),
{Year, Month, DayOfMonth}.
day_to_year(DayOfEpoch) when DayOfEpoch >= 0 ->
YMax = DayOfEpoch div ?DAYS_PER_YEAR,
YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR,
{Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)),
{Y1, DayOfEpoch - D1}.
year_day_to_date(Year, DayOfYear) ->
ExtraDay =
case is_leap_year(Year) of
true ->
1;
false ->
0
end,
{Month, Day} = year_day_to_date2(ExtraDay, DayOfYear),
{Month, Day + 1}.
dty(Min, Max, _D1, DMin, _DMax) when Min == Max ->
{Min, DMin};
dty(Min, Max, D1, DMin, DMax) ->
Diff = Max - Min,
Mid = Min + Diff * (D1 - DMin) div (DMax - DMin),
MidLength =
case is_leap_year(Mid) of
true ->
?DAYS_PER_LEAP_YEAR;
false ->
?DAYS_PER_YEAR
end,
case dy(Mid) of
D2 when D1 < D2 ->
NewMax = Mid - 1,
dty(Min, NewMax, D1, DMin, dy(NewMax));
D2 when D1 - D2 >= MidLength ->
NewMin = Mid + 1,
dty(NewMin, Max, D1, dy(NewMin), DMax);
D2 ->
{Mid, D2}
end.
dy(Y) when Y =< 0 ->
0;
dy(Y) ->
X = Y - 1,
X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR.
is_leap_year(Y) when is_integer(Y), Y >= 0 ->
is_leap_year1(Y).
is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 ->
true;
is_leap_year1(Year) when Year rem 400 =:= 0 ->
true;
is_leap_year1(_) ->
false.
year_day_to_date2(_, Day) when Day < 31 ->
{1, Day};
year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E ->
{2, Day - 31};
year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E ->
{3, Day - (59 + E)};
year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E ->
{4, Day - (90 + E)};
year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E ->
{5, Day - (120 + E)};
year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E ->
{6, Day - (151 + E)};
year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E ->
{7, Day - (181 + E)};
year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E ->
{8, Day - (212 + E)};
year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E ->
{9, Day - (243 + E)};
year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E ->
{10, Day - (273 + E)};
year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E ->
{11, Day - (304 + E)};
year_day_to_date2(E, Day) when 334 + E =< Day ->
{12, Day - (334 + E)}.
trans_x_second(FromUnit, ToUnit, Time) ->
XSecond = do_trans_x_second(FromUnit, ToUnit, Time),
Len =
case ToUnit of
millisecond -> 3;
microsecond -> 6;
nanosecond -> 9
end,
padding(XSecond, Len).
do_trans_x_second(second, _, _Time) -> 0;
do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000;
do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000;
do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000;
do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000;
do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000;
do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000;
do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000;
do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000;
do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000.
padding(Data, Len) when is_integer(Data) ->
padding(integer_to_list(Data), Len);
padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len ->
[$0 | padding(Data, Len - 1)];
padding(Data, _Len) ->
Data.
%% -------------------------------------------------------------------------------------------------
%% internal
%% parse part
do_parse(DateStr, Unit, Formatter) ->
DateInfo = do_parse_date_str(DateStr, Formatter, #{}),
{Precise, PrecisionUnit} = precision(DateInfo),
Counter =
fun
(year, V, Res) ->
Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise);
(month, V, Res) ->
Res + dm(V) * ?SECONDS_PER_DAY * Precise;
(day, V, Res) ->
Res + (V * ?SECONDS_PER_DAY * Precise);
(hour, V, Res) ->
Res + (V * ?SECONDS_PER_HOUR * Precise);
(minute, V, Res) ->
Res + (V * ?SECONDS_PER_MINUTE * Precise);
(second, V, Res) ->
Res + V * Precise;
(millisecond, V, Res) ->
case PrecisionUnit of
millisecond ->
Res + V;
microsecond ->
Res + (V * 1000);
nanosecond ->
Res + (V * 1000000)
end;
(microsecond, V, Res) ->
case PrecisionUnit of
microsecond ->
Res + V;
nanosecond ->
Res + (V * 1000)
end;
(nanosecond, V, Res) ->
Res + V;
(parsed_offset, V, Res) ->
Res - V
end,
Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise),
erlang:convert_time_unit(Count, PrecisionUnit, Unit).
precision(#{nanosecond := _}) -> {1000_000_000, nanosecond};
precision(#{microsecond := _}) -> {1000_000, microsecond};
precision(#{millisecond := _}) -> {1000, millisecond};
precision(#{second := _}) -> {1, second};
precision(_) -> {1, second}.
do_parse_date_str(<<>>, _, Result) ->
Result;
do_parse_date_str(_, [], Result) ->
Result;
do_parse_date_str(Date, [Key | Formatter], Result) ->
Size = date_size(Key),
<<DatePart:Size/binary-unit:8, Tail/binary>> = Date,
case lists:member(Key, ?DATE_PART) of
true ->
do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)});
false ->
case lists:member(Key, ?DATE_ZONE_NAME) of
true ->
do_parse_date_str(Tail, Formatter, Result#{
parsed_offset => offset_second(DatePart)
});
false ->
do_parse_date_str(Tail, Formatter, Result)
end
end.
date_size(Str) when is_list(Str) -> erlang:length(Str);
date_size(year) -> 4;
date_size(month) -> 2;
date_size(day) -> 2;
date_size(hour) -> 2;
date_size(minute) -> 2;
date_size(second) -> 2;
date_size(millisecond) -> 3;
date_size(microsecond) -> 6;
date_size(nanosecond) -> 9;
date_size(timezone) -> 5;
date_size(timezone1) -> 6;
date_size(timezone2) -> 9.
dm(1) -> 0;
dm(2) -> 31;
dm(3) -> 59;
dm(4) -> 90;
dm(5) -> 120;
dm(6) -> 151;
dm(7) -> 181;
dm(8) -> 212;
dm(9) -> 243;
dm(10) -> 273;
dm(11) -> 304;
dm(12) -> 334.

29
apps/emqx/src/emqx_channel.erl
View File

@ -715,9 +715,13 @@ do_publish(_PacketId, Msg = #message{qos = ?QOS_0}, Channel) ->
{ok, NChannel};
do_publish(PacketId, Msg = #message{qos = ?QOS_1}, Channel) ->
PubRes = emqx_broker:publish(Msg),
RC = puback_reason_code(PubRes),
NChannel = ensure_quota(PubRes, Channel),
handle_out(puback, {PacketId, RC}, NChannel);
RC = puback_reason_code(PacketId, Msg, PubRes),
case RC of
undefined ->
{ok, Channel};
_Value ->
do_finish_publish(PacketId, PubRes, RC, Channel)
end;
do_publish(
PacketId,
Msg = #message{qos = ?QOS_2},
@ -725,7 +729,7 @@ do_publish(
) ->
case emqx_session:publish(ClientInfo, PacketId, Msg, Session) of
{ok, PubRes, NSession} ->
RC = puback_reason_code(PubRes),
RC = pubrec_reason_code(PubRes),
NChannel0 = set_session(NSession, Channel),
NChannel1 = ensure_timer(await_timer, NChannel0),
NChannel2 = ensure_quota(PubRes, NChannel1),
@ -738,6 +742,10 @@ do_publish(
handle_out(disconnect, RC, Channel)
end.
do_finish_publish(PacketId, PubRes, RC, Channel) ->
NChannel = ensure_quota(PubRes, Channel),
handle_out(puback, {PacketId, RC}, NChannel).
ensure_quota(_, Channel = #channel{quota = infinity}) ->
Channel;
ensure_quota(PubRes, Channel = #channel{quota = Limiter}) ->
@ -757,9 +765,14 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) ->
ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter})
end.
-compile({inline, [puback_reason_code/1]}).
puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS;
puback_reason_code([_ | _]) -> ?RC_SUCCESS.
-compile({inline, [pubrec_reason_code/1]}).
pubrec_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS;
pubrec_reason_code([_ | _]) -> ?RC_SUCCESS.
puback_reason_code(PacketId, Msg, [] = PubRes) ->
emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_NO_MATCHING_SUBSCRIBERS);
puback_reason_code(PacketId, Msg, [_ | _] = PubRes) ->
emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_SUCCESS).
-compile({inline, [after_message_acked/3]}).
after_message_acked(ClientInfo, Msg, PubAckProps) ->
@ -1264,6 +1277,8 @@ handle_info(die_if_test = Info, Channel) ->
{ok, Channel};
handle_info({disconnect, ReasonCode, ReasonName, Props}, Channel) ->
handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel);
handle_info({puback, PacketId, PubRes, RC}, Channel) ->
do_finish_publish(PacketId, PubRes, RC, Channel);
handle_info(Info, Channel) ->
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{ok, Channel}.

32
apps/emqx/src/emqx_cm.erl
View File

@ -97,6 +97,7 @@
mark_channel_connected/1,
mark_channel_disconnected/1,
get_connected_client_count/0,
takeover_finish/2,
do_kick_session/3,
do_get_chan_stats/2,
@ -188,11 +189,13 @@ unregister_channel(ClientId) when is_binary(ClientId) ->
ok.
%% @private
do_unregister_channel(Chan) ->
do_unregister_channel({_ClientId, ChanPid} = Chan) ->
ok = emqx_cm_registry:unregister_channel(Chan),
true = ets:delete(?CHAN_CONN_TAB, Chan),
true = ets:delete(?CHAN_INFO_TAB, Chan),
ets:delete_object(?CHAN_TAB, Chan).
ets:delete_object(?CHAN_TAB, Chan),
ok = emqx_hooks:run('channel.unregistered', [ChanPid]),
true.
-spec connection_closed(emqx_types:clientid()) -> true.
connection_closed(ClientId) ->
@ -220,7 +223,7 @@ do_get_chan_info(ClientId, ChanPid) ->
-spec get_chan_info(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:infos()).
get_chan_info(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v1:get_chan_info(ClientId, ChanPid)).
wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)).
%% @doc Update infos of the channel.
-spec set_chan_info(emqx_types:clientid(), emqx_types:attrs()) -> boolean().
@ -250,7 +253,7 @@ do_get_chan_stats(ClientId, ChanPid) ->
-spec get_chan_stats(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:stats()).
get_chan_stats(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v1:get_chan_stats(ClientId, ChanPid)).
wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)).
%% @doc Set channel's stats.
-spec set_chan_stats(emqx_types:clientid(), emqx_types:stats()) -> boolean().
@ -312,13 +315,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) ->
}};
{living, ConnMod, ChanPid, Session} ->
ok = emqx_session:resume(ClientInfo, Session),
case
request_stepdown(
{takeover, 'end'},
ConnMod,
ChanPid
)
of
case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of
{ok, Pendings} ->
Session1 = emqx_persistent_session:persist(
ClientInfo, ConnInfo, Session
@ -408,6 +405,13 @@ takeover_session(ClientId) ->
takeover_session(ClientId, ChanPid)
end.
takeover_finish(ConnMod, ChanPid) ->
request_stepdown(
{takeover, 'end'},
ConnMod,
ChanPid
).
takeover_session(ClientId, Pid) ->
try
do_takeover_session(ClientId, Pid)
@ -437,7 +441,7 @@ do_takeover_session(ClientId, ChanPid) when node(ChanPid) == node() ->
end
end;
do_takeover_session(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v1:takeover_session(ClientId, ChanPid)).
wrap_rpc(emqx_cm_proto_v2:takeover_session(ClientId, ChanPid)).
%% @doc Discard all the sessions identified by the ClientId.
-spec discard_session(emqx_types:clientid()) -> ok.
@ -539,7 +543,7 @@ do_kick_session(Action, ClientId, ChanPid) ->
%% @private This function is shared for session 'kick' and 'discard' (as the first arg Action).
kick_session(Action, ClientId, ChanPid) ->
try
wrap_rpc(emqx_cm_proto_v1:kick_session(Action, ClientId, ChanPid))
wrap_rpc(emqx_cm_proto_v2:kick_session(Action, ClientId, ChanPid))
catch
Error:Reason ->
%% This should mostly be RPC failures.
@ -759,7 +763,7 @@ do_get_chann_conn_mod(ClientId, ChanPid) ->
end.
get_chann_conn_mod(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v1:get_chann_conn_mod(ClientId, ChanPid)).
wrap_rpc(emqx_cm_proto_v2:get_chann_conn_mod(ClientId, ChanPid)).
mark_channel_connected(ChanPid) ->
?tp(emqx_cm_connected_client_count_inc, #{chan_pid => ChanPid}),

90
apps/emqx/src/emqx_maybe.erl Normal file
View File

@ -0,0 +1,90 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_maybe).
-include_lib("emqx/include/types.hrl").
-export([to_list/1]).
-export([from_list/1]).
-export([define/2]).
-export([apply/2]).
-type t(T) :: maybe(T).
-export_type([t/1]).
-spec to_list(maybe(A)) -> [A].
to_list(undefined) ->
[];
to_list(Term) ->
[Term].
-spec from_list([A]) -> maybe(A).
from_list([]) ->
undefined;
from_list([Term]) ->
Term.
-spec define(maybe(A), B) -> A | B.
define(undefined, Term) ->
Term;
define(Term, _) ->
Term.
%% @doc Apply a function to a maybe argument.
-spec apply(fun((A) -> maybe(A)), maybe(A)) ->
maybe(A).
apply(_Fun, undefined) ->
undefined;
apply(Fun, Term) when is_function(Fun) ->
erlang:apply(Fun, [Term]).
%%
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
to_list_test_() ->
[
?_assertEqual([], to_list(undefined)),
?_assertEqual([42], to_list(42))
].
from_list_test_() ->
[
?_assertEqual(undefined, from_list([])),
?_assertEqual(3.1415, from_list([3.1415])),
?_assertError(_, from_list([1, 2, 3]))
].
define_test_() ->
[
?_assertEqual(42, define(42, undefined)),
?_assertEqual(<<"default">>, define(undefined, <<"default">>)),
?_assertEqual(undefined, define(undefined, undefined))
].
apply_test_() ->
[
?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)),
?_assertEqual(undefined, ?MODULE:apply(fun erlang:integer_to_binary/1, undefined)),
?_assertEqual(undefined, ?MODULE:apply(fun crash/1, undefined))
].
crash(_) ->
erlang:error(crashed).
-endif.

6
apps/emqx/src/emqx_schema.erl
View File

@ -28,6 +28,7 @@
-include("emqx_access_control.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("logger.hrl").
-type duration() :: integer().
-type duration_s() :: integer().
@ -3324,6 +3325,11 @@ naive_env_interpolation("$" ++ Maybe = Original) ->
{ok, Path} ->
filename:join([Path, Tail]);
error ->
?SLOG(warning, #{
msg => "failed_to_resolve_env_variable",
env => Env,
original => Original
}),
Original
end;
naive_env_interpolation(Other) ->

6
apps/emqx/src/emqx_tls_lib.erl
View File

@ -620,9 +620,11 @@ ensure_bin(A) when is_atom(A) -> atom_to_binary(A, utf8).
ensure_ssl_file_key(_SSL, []) ->
ok;
ensure_ssl_file_key(SSL, RequiredKeyPaths) ->
NotFoundRef = make_ref(),
Filter = fun(KeyPath) ->
NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef)
case emqx_utils_maps:deep_find(KeyPath, SSL) of
{not_found, _, _} -> true;
_ -> false
end
end,
case lists:filter(Filter, RequiredKeyPaths) of
[] -> ok;

4
apps/emqx/src/emqx_types.erl
View File

@ -101,6 +101,8 @@
-export_type([oom_policy/0]).
-export_type([takeover_data/0]).
-type proto_ver() ::
?MQTT_PROTO_V3
| ?MQTT_PROTO_V4
@ -242,3 +244,5 @@
max_heap_size => non_neg_integer(),
enable => boolean()
}.
-type takeover_data() :: map().

208
apps/emqx/src/emqx_wdgraph.erl Normal file
View File

@ -0,0 +1,208 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Weighted directed graph.
%%
%% Purely functional, built on top of a single `gb_tree`.
%% Weights are currently assumed to be non-negative numbers, hovewer
%% presumably anything that is ≥ 0 should work (but won't typecheck 🥲).
-module(emqx_wdgraph).
-export([new/0]).
-export([insert_edge/5]).
-export([find_edge/3]).
-export([get_edges/2]).
-export([fold/3]).
-export([find_shortest_path/3]).
-export_type([t/0]).
-export_type([t/2]).
-export_type([weight/0]).
-type gnode() :: term().
-type weight() :: _NonNegative :: number().
-type label() :: term().
-opaque t() :: t(gnode(), label()).
-opaque t(Node, Label) :: gb_trees:tree({Node}, [{Node, weight(), Label}]).
%%
-spec new() -> t(_, _).
new() ->
gb_trees:empty().
%% Add an edge.
%% Nodes are not expected to exist beforehand, and created lazily.
%% There could be only one edge between each pair of nodes, this function
%% replaces any existing edge in the graph.
-spec insert_edge(Node, Node, weight(), Label, t(Node, Label)) -> t(Node, Label).
insert_edge(From, To, Weight, EdgeLabel, G) ->
Edges = tree_lookup({From}, G, []),
EdgesNext = lists:keystore(To, 1, Edges, {To, Weight, EdgeLabel}),
tree_update({From}, EdgesNext, G).
%% Find exising edge between two nodes, if any.
-spec find_edge(Node, Node, t(Node, Label)) -> {weight(), Label} | false.
find_edge(From, To, G) ->
Edges = tree_lookup({From}, G, []),
case lists:keyfind(To, 1, Edges) of
{To, Weight, Label} ->
{Weight, Label};
false ->
false
end.
%% Get all edges from the given node.
-spec get_edges(Node, t(Node, Label)) -> [{Node, weight(), Label}].
get_edges(Node, G) ->
tree_lookup({Node}, G, []).
-spec fold(FoldFun, Acc, t(Node, Label)) -> Acc when
FoldFun :: fun((Node, _Edge :: {Node, weight(), Label}, Acc) -> Acc).
fold(FoldFun, Acc, G) ->
fold_iterator(FoldFun, Acc, gb_trees:iterator(G)).
fold_iterator(FoldFun, AccIn, It) ->
case gb_trees:next(It) of
{{Node}, Edges = [_ | _], ItNext} ->
AccNext = lists:foldl(
fun(Edge = {_To, _Weight, _Label}, Acc) ->
FoldFun(Node, Edge, Acc)
end,
AccIn,
Edges
),
fold_iterator(FoldFun, AccNext, ItNext);
none ->
AccIn
end.
% Find the shortest path between two nodes, if any. If the path exists, return list
% of edge labels along that path.
% This is a Dijkstra shortest path algorithm. It is one-way right now, for
% simplicity sake.
-spec find_shortest_path(Node, Node, t(Node, Label)) -> [Label] | {false, _StoppedAt :: Node}.
find_shortest_path(From, To, G1) ->
% NOTE
% If `From` and `To` are the same node, then path is `[]` even if this
% node does not exist in the graph.
G2 = set_cost(From, 0, [], G1),
case find_shortest_path(From, 0, To, G2) of
{true, G3} ->
construct_path(From, To, [], G3);
{false, Last} ->
{false, Last}
end.
find_shortest_path(Node, Cost, Target, G1) ->
Edges = get_edges(Node, G1),
G2 = update_neighbours(Node, Cost, Edges, G1),
case take_queued(G2) of
{Target, _NextCost, G3} ->
{true, G3};
{Next, NextCost, G3} ->
find_shortest_path(Next, NextCost, Target, G3);
none ->
{false, Node}
end.
construct_path(From, From, Acc, _) ->
Acc;
construct_path(From, To, Acc, G) ->
{Prev, Label} = get_label(To, G),
construct_path(From, Prev, [Label | Acc], G).
update_neighbours(Node, NodeCost, Edges, G1) ->
lists:foldl(
fun(Edge, GAcc) -> update_neighbour(Node, NodeCost, Edge, GAcc) end,
G1,
Edges
).
update_neighbour(Node, NodeCost, {Neighbour, Weight, Label}, G) ->
case is_visited(G, Neighbour) of
false ->
CurrentCost = get_cost(Neighbour, G),
case NodeCost + Weight of
NeighCost when NeighCost < CurrentCost ->
set_cost(Neighbour, NeighCost, {Node, Label}, G);
_ ->
G
end;
true ->
G
end.
get_cost(Node, G) ->
case tree_lookup({Node, cost}, G, inf) of
{Cost, _Label} ->
Cost;
inf ->
inf
end.
get_label(Node, G) ->
{_Cost, Label} = gb_trees:get({Node, cost}, G),
Label.
set_cost(Node, Cost, Label, G1) ->
G3 =
case tree_lookup({Node, cost}, G1, inf) of
{CostWas, _Label} ->
{true, G2} = gb_trees:take({queued, CostWas, Node}, G1),
gb_trees:insert({queued, Cost, Node}, true, G2);
inf ->
gb_trees:insert({queued, Cost, Node}, true, G1)
end,
G4 = tree_update({Node, cost}, {Cost, Label}, G3),
G4.
take_queued(G1) ->
It = gb_trees:iterator_from({queued, 0, 0}, G1),
case gb_trees:next(It) of
{{queued, Cost, Node} = Index, true, _It} ->
{Node, Cost, gb_trees:delete(Index, G1)};
_ ->
none
end.
is_visited(G, Node) ->
case tree_lookup({Node, cost}, G, inf) of
inf ->
false;
{Cost, _Label} ->
not tree_lookup({queued, Cost, Node}, G, false)
end.
tree_lookup(Index, Tree, Default) ->
case gb_trees:lookup(Index, Tree) of
{value, V} ->
V;
none ->
Default
end.
tree_update(Index, Value, Tree) ->
case gb_trees:is_defined(Index, Tree) of
true ->
gb_trees:update(Index, Value, Tree);
false ->
gb_trees:insert(Index, Value, Tree)
end.

88
apps/emqx/src/proto/emqx_cm_proto_v2.erl Normal file
View File

@ -0,0 +1,88 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_cm_proto_v2).
-behaviour(emqx_bpapi).
-export([
introduced_in/0,
lookup_client/2,
kickout_client/2,
get_chan_stats/2,
get_chan_info/2,
get_chann_conn_mod/2,
takeover_session/2,
takeover_finish/2,
kick_session/3
]).
-include("bpapi.hrl").
-include("src/emqx_cm.hrl").
introduced_in() ->
"5.0.0".
-spec kickout_client(node(), emqx_types:clientid()) -> ok | {badrpc, _}.
kickout_client(Node, ClientId) ->
rpc:call(Node, emqx_cm, kick_session, [ClientId]).
-spec lookup_client(node(), {clientid, emqx_types:clientid()} | {username, emqx_types:username()}) ->
[emqx_cm:channel_info()] | {badrpc, _}.
lookup_client(Node, Key) ->
rpc:call(Node, emqx_cm, lookup_client, [Key]).
-spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:stats() | {badrpc, _}.
get_chan_stats(ClientId, ChanPid) ->
rpc:call(node(ChanPid), emqx_cm, do_get_chan_stats, [ClientId, ChanPid], ?T_GET_INFO * 2).
-spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:infos() | {badrpc, _}.
get_chan_info(ClientId, ChanPid) ->
rpc:call(node(ChanPid), emqx_cm, do_get_chan_info, [ClientId, ChanPid], ?T_GET_INFO * 2).
-spec get_chann_conn_mod(emqx_types:clientid(), emqx_cm:chan_pid()) ->
module() | undefined | {badrpc, _}.
get_chann_conn_mod(ClientId, ChanPid) ->
rpc:call(node(ChanPid), emqx_cm, do_get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO * 2).
-spec takeover_session(emqx_types:clientid(), emqx_cm:chan_pid()) ->
none
| {expired | persistent, emqx_session:session()}
| {living, _ConnMod :: atom(), emqx_cm:chan_pid(), emqx_session:session()}
| {badrpc, _}.
takeover_session(ClientId, ChanPid) ->
rpc:call(node(ChanPid), emqx_cm, takeover_session, [ClientId, ChanPid], ?T_TAKEOVER * 2).
-spec takeover_finish(module(), emqx_cm:chan_pid()) ->
{ok, emqx_type:takeover_data()}
| {ok, list(emqx_type:deliver()), emqx_type:takeover_data()}
| {error, term()}
| {badrpc, _}.
takeover_finish(ConnMod, ChanPid) ->
erpc:call(
node(ChanPid),
emqx_cm,
takeover_finish,
[ConnMod, ChanPid],
?T_TAKEOVER * 2
).
-spec kick_session(kick | discard, emqx_types:clientid(), emqx_cm:chan_pid()) -> ok | {badrpc, _}.
kick_session(Action, ClientId, ChanPid) ->
rpc:call(node(ChanPid), emqx_cm, do_kick_session, [Action, ClientId, ChanPid], ?T_KICK * 2).

2
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -1133,7 +1133,7 @@ t_ws_cookie_init(_) ->
?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)).
%%--------------------------------------------------------------------
%% Test cases for other mechnisms
%% Test cases for other mechanisms
%%--------------------------------------------------------------------
t_flapping_detect(_) ->

70
apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl Normal file
View File

@ -0,0 +1,70 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_channel_delayed_puback_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
Config.
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(Case, Config) ->
?MODULE:Case({init, Config}).
end_per_testcase(Case, Config) ->
?MODULE:Case({'end', Config}).
%%--------------------------------------------------------------------
%% Test cases
%%--------------------------------------------------------------------
t_delayed_puback({init, Config}) ->
emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST),
Config;
t_delayed_puback({'end', _Config}) ->
emqx_hooks:del('message.puback', {?MODULE, on_message_puback});
t_delayed_puback(_Config) ->
{ok, ConnPid} = emqtt:start_link([{clientid, <<"clientid">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(ConnPid),
{ok, #{reason_code := ?RC_UNSPECIFIED_ERROR}} = emqtt:publish(
ConnPid, <<"topic">>, <<"hello">>, 1
),
emqtt:disconnect(ConnPid).
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
on_message_puback(PacketId, _Msg, PubRes, _RC) ->
erlang:send(self(), {puback, PacketId, PubRes, ?RC_UNSPECIFIED_ERROR}),
{stop, undefined}.

19
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -30,6 +30,7 @@
start_apps/1,
start_apps/2,
start_apps/3,
start_app/2,
stop_apps/1,
stop_apps/2,
reload/2,
@ -246,6 +247,9 @@ do_render_app_config(App, Schema, ConfigFile, Opts) ->
copy_certs(App, RenderedConfigFile),
ok.
start_app(App, SpecAppConfig) ->
start_app(App, SpecAppConfig, #{}).
start_app(App, SpecAppConfig, Opts) ->
render_and_load_app_config(App, Opts),
SpecAppConfig(App),
@ -326,12 +330,7 @@ read_schema_configs(no_schema, _ConfigFile) ->
ok;
read_schema_configs(Schema, ConfigFile) ->
NewConfig = generate_config(Schema, ConfigFile),
lists:foreach(
fun({App, Configs}) ->
[application:set_env(App, Par, Value) || {Par, Value} <- Configs]
end,
NewConfig
).
application:set_env(NewConfig).
generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) ->
{ok, Conf0} = hocon:load(ConfigFile, #{format => richmap}),
@ -687,11 +686,17 @@ emqx_cluster(Specs0, CommonOpts) ->
]),
%% Set the default node of the cluster:
CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs],
JoinTo =
JoinTo0 =
case CoreNodes of
[First | _] -> First;
_ -> undefined
end,
JoinTo =
case maps:find(join_to, CommonOpts) of
{ok, true} -> JoinTo0;
{ok, JT} -> JT;
error -> JoinTo0
end,
[
{Name,
merge_opts(Opts, #{

33
apps/emqx/test/emqx_proper_types.erl
View File

@ -43,12 +43,21 @@
ip/0,
port/0,
limited_atom/0,
limited_latin_atom/0
limited_latin_atom/0,
printable_utf8/0,
printable_codepoint/0
]).
%% Generic Types
-export([
scaled/2
]).
%% Iterators
-export([nof/1]).
-type proptype() :: proper_types:raw_type().
%%--------------------------------------------------------------------
%% Types High level
%%--------------------------------------------------------------------
@ -606,6 +615,20 @@ limited_atom() ->
limited_any_term() ->
oneof([binary(), number(), string()]).
printable_utf8() ->
?SUCHTHAT(
String,
?LET(L, list(printable_codepoint()), unicode:characters_to_binary(L)),
is_binary(String)
).
printable_codepoint() ->
frequency([
{7, range(16#20, 16#7E)},
{2, range(16#00A0, 16#D7FF)},
{1, range(16#E000, 16#FFFD)}
]).
%%--------------------------------------------------------------------
%% Iterators
%%--------------------------------------------------------------------
@ -632,6 +655,14 @@ limited_list(N, T) ->
end
).
%%--------------------------------------------------------------------
%% Generic Types
%%--------------------------------------------------------------------
-spec scaled(number(), proptype()) -> proptype().
scaled(F, T) when F > 0 ->
?SIZED(S, resize(round(S * F), T)).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------

104
apps/emqx/test/emqx_wdgraph_tests.erl Normal file
View File

@ -0,0 +1,104 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_wdgraph_tests).
-include_lib("eunit/include/eunit.hrl").
empty_test_() ->
G = emqx_wdgraph:new(),
[
?_assertEqual([], emqx_wdgraph:get_edges(foo, G)),
?_assertEqual(false, emqx_wdgraph:find_edge(foo, bar, G))
].
edges_nodes_test_() ->
G1 = emqx_wdgraph:new(),
G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1),
G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2),
G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3),
G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4),
[
?_assertEqual({42, "fancy"}, emqx_wdgraph:find_edge(foo, bar, G2)),
?_assertEqual({100, "luxury"}, emqx_wdgraph:find_edge(foo, bar, G5)),
?_assertEqual([{bar, 100, "luxury"}], emqx_wdgraph:get_edges(foo, G5)),
?_assertEqual({1, "cheapest"}, emqx_wdgraph:find_edge(bar, baz, G5)),
?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5))
].
fold_test_() ->
G1 = emqx_wdgraph:new(),
G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1),
G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2),
G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3),
G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4),
[
?_assertEqual(
% 100 + 0 + 1
101,
emqx_wdgraph:fold(fun(_From, {_, Weight, _}, Acc) -> Weight + Acc end, 0, G5)
),
?_assertEqual(
[bar, baz, foo],
lists:usort(
emqx_wdgraph:fold(fun(From, {To, _, _}, Acc) -> [From, To | Acc] end, [], G5)
)
)
].
nonexistent_nodes_path_test_() ->
G1 = emqx_wdgraph:new(),
G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1),
G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2),
[
?_assertEqual(
{false, nosuchnode},
emqx_wdgraph:find_shortest_path(nosuchnode, baz, G3)
),
?_assertEqual(
[],
emqx_wdgraph:find_shortest_path(nosuchnode, nosuchnode, G3)
)
].
nonexistent_path_test_() ->
G1 = emqx_wdgraph:new(),
G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1),
G3 = emqx_wdgraph:insert_edge(baz, boo, 1, "cheapest", G2),
G4 = emqx_wdgraph:insert_edge(boo, last, 3.5, "change", G3),
[
?_assertEqual(
{false, last},
emqx_wdgraph:find_shortest_path(baz, foo, G4)
),
?_assertEqual(
{false, bar},
emqx_wdgraph:find_shortest_path(foo, last, G4)
)
].
shortest_path_test() ->
G1 = emqx_wdgraph:new(),
G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1),
G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2),
G4 = emqx_wdgraph:insert_edge(baz, last, 0, "free", G3),
G5 = emqx_wdgraph:insert_edge(bar, last, 100, "luxury", G4),
G6 = emqx_wdgraph:insert_edge(bar, foo, 0, "comeback", G5),
?assertEqual(
["fancy", "cheapest", "free"],
emqx_wdgraph:find_shortest_path(foo, last, G6)
).

23
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -886,6 +886,29 @@ format_metrics(#{
Rate5m,
RateMax,
Rcvd
);
format_metrics(_Metrics) ->
%% Empty metrics: can happen when a node joins another and a
%% bridge is not yet replicated to it, so the counters map is
%% empty.
?METRICS(
_Dropped = 0,
_DroppedOther = 0,
_DroppedExpired = 0,
_DroppedQueueFull = 0,
_DroppedResourceNotFound = 0,
_DroppedResourceStopped = 0,
_Matched = 0,
_Queued = 0,
_Retried = 0,
_LateReply = 0,
_SentFailed = 0,
_SentInflight = 0,
_SentSucc = 0,
_Rate = 0,
_Rate5m = 0,
_RateMax = 0,
_Rcvd = 0
).
fill_defaults(Type, RawConf) ->

34
apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
View File

@ -51,11 +51,39 @@ post_request() ->
api_schema(Method) ->
Broker = [
ref(Mod, Method)
|| Mod <- [emqx_bridge_webhook_schema, emqx_bridge_mqtt_schema]
{Type, ref(Mod, Method)}
|| {Type, Mod} <- [
{<<"webhook">>, emqx_bridge_webhook_schema},
{<<"mqtt">>, emqx_bridge_mqtt_schema}
]
],
EE = ee_api_schemas(Method),
hoconsc:union(Broker ++ EE).
hoconsc:union(bridge_api_union(Broker ++ EE)).
bridge_api_union(Refs) ->
Index = maps:from_list(Refs),
fun
(all_union_members) ->
maps:values(Index);
({value, V}) ->
case V of
#{<<"type">> := T} ->
case maps:get(T, Index, undefined) of
undefined ->
throw(#{
field_name => type,
reason => <<"unknown bridge type">>
});
Ref ->
[Ref]
end;
_ ->
throw(#{
field_name => type,
reason => <<"unknown bridge type">>
})
end
end.
-if(?EMQX_RELEASE_EDITION == ee).
ee_api_schemas(Method) ->

68
apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
View File

@ -70,18 +70,22 @@
all() ->
[
{group, single},
{group, cluster_later_join},
{group, cluster}
].
groups() ->
AllTCs = emqx_common_test_helpers:all(?MODULE),
SingleOnlyTests = [
t_broken_bpapi_vsn,
t_old_bpapi_vsn,
t_bridges_probe
],
ClusterLaterJoinOnlyTCs = [t_cluster_later_join_metrics],
[
{single, [], emqx_common_test_helpers:all(?MODULE)},
{cluster, [], emqx_common_test_helpers:all(?MODULE) -- SingleOnlyTests}
{single, [], AllTCs -- ClusterLaterJoinOnlyTCs},
{cluster_later_join, [], ClusterLaterJoinOnlyTCs},
{cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs}
].
suite() ->
@ -104,6 +108,17 @@ init_per_group(cluster, Config) ->
ok = erpc:call(NodePrimary, fun() -> init_node(primary) end),
_ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest],
[{group, cluster}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config];
init_per_group(cluster_later_join, Config) ->
Cluster = mk_cluster_specs(Config, #{join_to => undefined}),
ct:pal("Starting ~p", [Cluster]),
Nodes = [
emqx_common_test_helpers:start_slave(Name, Opts)
|| {Name, Opts} <- Cluster
],
[NodePrimary | NodesRest] = Nodes,
ok = erpc:call(NodePrimary, fun() -> init_node(primary) end),
_ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest],
[{group, cluster_later_join}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config];
init_per_group(_, Config) ->
ok = emqx_mgmt_api_test_util:init_suite(?SUITE_APPS),
ok = load_suite_config(emqx_rule_engine),
@ -111,6 +126,9 @@ init_per_group(_, Config) ->
[{group, single}, {api_node, node()} | Config].
mk_cluster_specs(Config) ->
mk_cluster_specs(Config, #{}).
mk_cluster_specs(Config, Opts) ->
Specs = [
{core, emqx_bridge_api_SUITE1, #{}},
{core, emqx_bridge_api_SUITE2, #{}}
@ -132,6 +150,7 @@ mk_cluster_specs(Config) ->
load_apps => ?SUITE_APPS ++ [emqx_dashboard],
env_handler => fun load_suite_config/1,
load_schema => false,
join_to => maps:get(join_to, Opts, true),
priv_data_dir => ?config(priv_dir, Config)
},
emqx_common_test_helpers:emqx_cluster(Specs, CommonOpts).
@ -142,7 +161,7 @@ init_node(Type) ->
primary ->
ok = emqx_config:put(
[dashboard, listeners],
#{http => #{enable => true, bind => 18083}}
#{http => #{enable => true, bind => 18083}, proxy_header => false}
),
ok = emqx_dashboard:start_listeners(),
ready = emqx_dashboard_listener:regenerate_minirest_dispatch(),
@ -164,7 +183,10 @@ load_suite_config(emqx_bridge) ->
load_suite_config(_) ->
ok.
end_per_group(cluster, Config) ->
end_per_group(Group, Config) when
Group =:= cluster;
Group =:= cluster_later_join
->
ok = lists:foreach(
fun(Node) ->
_ = erpc:call(Node, emqx_common_test_helpers, stop_apps, [?SUITE_APPS]),
@ -1298,6 +1320,44 @@ t_inconsistent_webhook_request_timeouts(Config) ->
validate_resource_request_timeout(proplists:get_value(group, Config), 1000, Name),
ok.
t_cluster_later_join_metrics(Config) ->
Port = ?config(port, Config),
APINode = ?config(api_node, Config),
ClusterNodes = ?config(cluster_nodes, Config),
[OtherNode | _] = ClusterNodes -- [APINode],
URL1 = ?URL(Port, "path1"),
Name = ?BRIDGE_NAME,
BridgeParams = ?HTTP_BRIDGE(URL1, Name),
BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
?check_trace(
begin
%% Create a bridge on only one of the nodes.
?assertMatch({ok, 201, _}, request_json(post, uri(["bridges"]), BridgeParams, Config)),
%% Pre-condition.
?assertMatch(
{ok, 200, #{
<<"metrics">> := #{<<"success">> := _},
<<"node_metrics">> := [_ | _]
}},
request_json(get, uri(["bridges", BridgeID, "metrics"]), Config)
),
%% Now join the other node join with the api node.
ok = erpc:call(OtherNode, ekka, join, [APINode]),
%% Check metrics; shouldn't crash even if the bridge is not
%% ready on the node that just joined the cluster.
?assertMatch(
{ok, 200, #{
<<"metrics">> := #{<<"success">> := _},
<<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _]
}},
request_json(get, uri(["bridges", BridgeID, "metrics"]), Config)
),
ok
end,
[]
),
ok.
validate_resource_request_timeout(single, Timeout, Name) ->
SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000},
BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),

2
apps/emqx_bridge_dynamo/rebar.config
View File

@ -1,6 +1,6 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}}
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}
, {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}

34
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl
View File

@ -45,9 +45,19 @@ fields(config) ->
}
)},
{authentication,
mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{
default => none, desc => ?DESC("authentication")
})}
mk(
hoconsc:union(fun auth_union_member_selector/1),
#{
default => none,
%% must mark this whole union as sensitive because
%% hocon ignores the `sensitive' metadata in struct
%% fields... Also, when trying to type check a struct
%% that doesn't match the intended type, it won't have
%% sensitivity information from sibling types.
sensitive => true,
desc => ?DESC("authentication")
}
)}
] ++ emqx_connector_schema_lib:ssl_fields();
fields(producer_opts) ->
[
@ -226,3 +236,21 @@ override_default(OriginalFn, NewDefault) ->
(default) -> NewDefault;
(Field) -> OriginalFn(Field)
end.
auth_union_member_selector(all_union_members) ->
[none, ref(auth_basic), ref(auth_token)];
auth_union_member_selector({value, V}) ->
case V of
#{<<"password">> := _} ->
[ref(auth_basic)];
#{<<"jwt">> := _} ->
[ref(auth_token)];
<<"none">> ->
[none];
_ ->
Expected = "none | basic | token",
throw(#{
field_name => authentication,
expected => Expected
})
end.

7
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
View File

@ -100,7 +100,7 @@ on_start(InstanceId, Config) ->
msg => "failed_to_start_pulsar_client",
instance_id => InstanceId,
pulsar_hosts => Servers,
reason => Reason
reason => emqx_utils:redact(Reason, fun is_sensitive_key/1)
}),
throw(failed_to_start_pulsar_client)
end,
@ -332,7 +332,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) ->
#{
instance_id => InstanceId,
kind => Kind,
reason => Error,
reason => emqx_utils:redact(Error, fun is_sensitive_key/1),
stacktrace => Stacktrace
}
),
@ -419,3 +419,6 @@ get_producer_status(Producers) ->
partition_strategy(key_dispatch) -> first_key_dispatch;
partition_strategy(Strategy) -> Strategy.
is_sensitive_key(auth_data) -> true;
is_sensitive_key(_) -> false.

2
apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_tdengine, [
{description, "EMQX Enterprise TDEngine Bridge"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [kernel, stdlib, tdengine]},
{env, []},

139
apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
View File

@ -25,7 +25,7 @@
on_get_status/2
]).
-export([connect/1, do_get_status/1, execute/3]).
-export([connect/1, do_get_status/1, execute/3, do_batch_insert/4]).
-import(hoconsc, [mk/2, enum/1, ref/2]).
@ -124,32 +124,36 @@ on_stop(InstanceId, #{pool_name := PoolName}) ->
on_query(InstanceId, {query, SQL}, State) ->
do_query(InstanceId, SQL, State);
on_query(InstanceId, Request, State) ->
%% because the `emqx-tdengine` client only supports a single SQL cmd
%% so the `on_query` and `on_batch_query` have the same process, that is:
%% we need to collect all data into one SQL cmd and then call the insert API
on_batch_query(InstanceId, [Request], State).
on_batch_query(
InstanceId,
BatchReq,
#{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State
) ->
case hd(BatchReq) of
{Key, _} ->
case maps:get(Key, Inserts, undefined) of
undefined ->
{error, {unrecoverable_error, batch_prepare_not_implemented}};
InsertSQL ->
Tokens = maps:get(Key, ParamsTokens),
do_batch_insert(InstanceId, BatchReq, InsertSQL, Tokens, State)
end;
Request ->
LogMeta = #{connector => InstanceId, first_request => Request, state => State},
?SLOG(error, LogMeta#{msg => "invalid request"}),
on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) ->
case maps:find(Key, InsertTksMap) of
{ok, Tokens} ->
SQL = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Data),
do_query(InstanceId, SQL, State);
_ ->
{error, {unrecoverable_error, invalid_request}}
end.
%% aggregate the batch queries to one SQL is a heavy job, we should put it in the worker process
on_batch_query(
InstanceId,
[{Key, _} | _] = BatchReq,
#{batch_tokens := BatchTksMap, query_opts := Opts} = State
) ->
case maps:find(Key, BatchTksMap) of
{ok, Tokens} ->
do_query_job(
InstanceId,
{?MODULE, do_batch_insert, [Tokens, BatchReq, Opts]},
State
);
_ ->
{error, {unrecoverable_error, batch_prepare_not_implemented}}
end;
on_batch_query(InstanceId, BatchReq, State) ->
LogMeta = #{connector => InstanceId, request => BatchReq, state => State},
?SLOG(error, LogMeta#{msg => "invalid request"}),
{error, {unrecoverable_error, invalid_request}}.
on_get_status(_InstanceId, #{pool_name := PoolName}) ->
Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1),
status_result(Health).
@ -167,17 +171,16 @@ status_result(_Status = false) -> connecting.
%% Helper fns
%%========================================================================================
do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) ->
SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens),
do_query(InstanceId, SQL, State).
do_query(InstanceId, Query, #{query_opts := Opts} = State) ->
do_query_job(InstanceId, {?MODULE, execute, [Query, Opts]}, State).
do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State) ->
do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) ->
?TRACE(
"QUERY",
"tdengine_connector_received",
#{connector => InstanceId, query => Query, state => State}
#{connector => InstanceId, job => Job, state => State}
),
Result = ecpool:pick_and_do(PoolName, {?MODULE, execute, [Query, Opts]}, no_handover),
Result = ecpool:pick_and_do(PoolName, Job, no_handover),
case Result of
{error, Reason} ->
@ -188,7 +191,7 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State
?SLOG(error, #{
msg => "tdengine_connector_do_query_failed",
connector => InstanceId,
query => Query,
job => Job,
reason => Reason
}),
Result;
@ -203,6 +206,35 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State
execute(Conn, Query, Opts) ->
tdengine:insert(Conn, Query, Opts).
do_batch_insert(Conn, Tokens, BatchReqs, Opts) ->
Queries = aggregate_query(Tokens, BatchReqs),
SQL = maps:fold(
fun(InsertPart, Values, Acc) ->
lists:foldl(
fun(ValuePart, IAcc) ->
<<IAcc/binary, " ", ValuePart/binary>>
end,
<<Acc/binary, " ", InsertPart/binary, " VALUES">>,
Values
)
end,
<<"INSERT INTO">>,
Queries
),
execute(Conn, SQL, Opts).
aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) ->
lists:foldl(
fun({_, Data}, Acc) ->
InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data),
ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data),
Values = maps:get(InsertPart, Acc, []),
maps:put(InsertPart, [ParamsPart | Values], Acc)
end,
#{},
BatchReqs
).
connect(Opts) ->
tdengine:start_link(Opts).
@ -218,32 +250,49 @@ parse_prepare_sql(Config) ->
parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}).
parse_batch_prepare_sql([{Key, H} | T], BatchInserts, BatchTks) ->
parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) ->
case emqx_plugin_libs_rule:detect_sql_type(H) of
{ok, select} ->
parse_batch_prepare_sql(T, BatchInserts, BatchTks);
parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap);
{ok, insert} ->
case emqx_plugin_libs_rule:split_insert_sql(H) of
{ok, {InsertSQL, Params}} ->
ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params),
InsertTks = emqx_plugin_libs_rule:preproc_tmpl(H),
H1 = string:trim(H, trailing, ";"),
case split_insert_sql(H1) of
[_InsertStr, InsertPart, _ValuesStr, ParamsPart] ->
InsertPartTks = emqx_plugin_libs_rule:preproc_tmpl(InsertPart),
ParamsPartTks = emqx_plugin_libs_rule:preproc_tmpl(ParamsPart),
parse_batch_prepare_sql(
T,
BatchInserts#{Key => InsertSQL},
BatchTks#{Key => ParamsTks}
InsertTksMap#{Key => InsertTks},
BatchTksMap#{Key => {InsertPartTks, ParamsPartTks}}
);
{error, Reason} ->
?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}),
parse_batch_prepare_sql(T, BatchInserts, BatchTks)
Result ->
?SLOG(error, #{msg => "split sql failed", sql => H, result => Result}),
parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap)
end;
{error, Reason} ->
?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
parse_batch_prepare_sql(T, BatchInserts, BatchTks)
parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap)
end;
parse_batch_prepare_sql([], BatchInserts, BatchTks) ->
parse_batch_prepare_sql([], InsertTksMap, BatchTksMap) ->
#{
batch_inserts => BatchInserts,
batch_params_tokens => BatchTks
insert_tokens => InsertTksMap,
batch_tokens => BatchTksMap
}.
to_bin(List) when is_list(List) ->
unicode:characters_to_binary(List, utf8).
split_insert_sql(SQL0) ->
SQL = emqx_plugin_libs_rule:formalize_sql(SQL0),
lists:filtermap(
fun(E) ->
case string:trim(E) of
<<>> ->
false;
E1 ->
{true, E1}
end
end,
re:split(SQL, "(?i)(insert into)|(?i)(values)")
).

218
apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl
View File

@ -24,9 +24,21 @@
");"
).
-define(SQL_DROP_TABLE, "DROP TABLE t_mqtt_msg").
-define(SQL_DELETE, "DELETE from t_mqtt_msg").
-define(SQL_DROP_STABLE, "DROP STABLE s_tab").
-define(SQL_DELETE, "DELETE FROM t_mqtt_msg").
-define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg").
-define(AUTO_CREATE_BRIDGE,
"insert into ${clientid} USING s_tab TAGS (${clientid}) values (${timestamp}, ${payload})"
).
-define(SQL_CREATE_STABLE,
"CREATE STABLE s_tab (\n"
" ts timestamp,\n"
" payload BINARY(1024)\n"
") TAGS (clientid BINARY(128));"
).
% DB defaults
-define(TD_DATABASE, "mqtt").
-define(TD_USERNAME, "root").
@ -53,12 +65,13 @@ all() ->
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
NonBatchCases = [t_write_timeout],
MustBatchCases = [t_batch_insert, t_auto_create_batch_insert],
BatchingGroups = [{group, with_batch}, {group, without_batch}],
[
{async, BatchingGroups},
{sync, BatchingGroups},
{with_batch, TCs -- NonBatchCases},
{without_batch, TCs}
{without_batch, TCs -- MustBatchCases}
].
init_per_group(async, Config) ->
@ -117,7 +130,8 @@ common_init(ConfigT) ->
Config0 = [
{td_host, Host},
{td_port, Port},
{proxy_name, "tdengine_restful"}
{proxy_name, "tdengine_restful"},
{template, ?SQL_BRIDGE}
| ConfigT
],
@ -165,6 +179,7 @@ tdengine_config(BridgeType, Config) ->
false -> 1
end,
QueryMode = ?config(query_mode, Config),
Template = ?config(template, Config),
ConfigString =
io_lib:format(
"bridges.~s.~s {\n"
@ -187,7 +202,7 @@ tdengine_config(BridgeType, Config) ->
?TD_DATABASE,
?TD_USERNAME,
?TD_PASSWORD,
?SQL_BRIDGE,
Template,
BatchSize,
QueryMode
]
@ -272,11 +287,15 @@ connect_direct_tdengine(Config) ->
connect_and_create_table(Config) ->
?WITH_CON(begin
{ok, _} = directly_query(Con, ?SQL_CREATE_DATABASE, []),
{ok, _} = directly_query(Con, ?SQL_CREATE_TABLE)
{ok, _} = directly_query(Con, ?SQL_CREATE_TABLE),
{ok, _} = directly_query(Con, ?SQL_CREATE_STABLE)
end).
connect_and_drop_table(Config) ->
?WITH_CON({ok, _} = directly_query(Con, ?SQL_DROP_TABLE)).
?WITH_CON(begin
{ok, _} = directly_query(Con, ?SQL_DROP_TABLE),
{ok, _} = directly_query(Con, ?SQL_DROP_STABLE)
end).
connect_and_clear_table(Config) ->
?WITH_CON({ok, _} = directly_query(Con, ?SQL_DELETE)).
@ -287,6 +306,15 @@ connect_and_get_payload(Config) ->
),
Result.
connect_and_exec(Config, SQL) ->
?WITH_CON({ok, _} = directly_query(Con, SQL)).
connect_and_query(Config, SQL) ->
?WITH_CON(
{ok, #{<<"code">> := 0, <<"data">> := Data}} = directly_query(Con, SQL)
),
Data.
directly_query(Con, Query) ->
directly_query(Con, Query, [{db_name, ?TD_DATABASE}]).
@ -407,7 +435,12 @@ t_write_failure(Config) ->
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?assertMatch({error, econnrefused}, Result),
case Result of
{error, Reason} when Reason =:= econnrefused; Reason =:= closed ->
ok;
_ ->
throw({unexpected, Result})
end,
ok
end),
ok.
@ -490,26 +523,19 @@ t_missing_data(Config) ->
ok.
t_bad_sql_parameter(Config) ->
EnableBatch = ?config(enable_batch, Config),
?assertMatch(
{ok, _},
create_bridge(Config)
),
Request = {sql, <<"">>, [bad_parameter]},
Request = {send_message, <<"">>},
{_, {ok, #{result := Result}}} =
?wait_async_action(
query_resource(Config, Request),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
case EnableBatch of
true ->
?assertEqual({error, {unrecoverable_error, invalid_request}}, Result);
false ->
?assertMatch(
{error, {unrecoverable_error, _}}, Result
)
end,
?assertMatch({error, #{<<"code">> := _}}, Result),
ok.
t_nasty_sql_string(Config) ->
@ -544,7 +570,165 @@ t_nasty_sql_string(Config) ->
connect_and_get_payload(Config)
).
t_simple_insert(Config) ->
connect_and_clear_table(Config),
?assertMatch(
{ok, _},
create_bridge(Config)
),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
Request = {send_message, SentData},
{_, {ok, #{result := _Result}}} =
?wait_async_action(
query_resource(Config, Request),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?assertMatch(
?PAYLOAD,
connect_and_get_payload(Config)
).
t_batch_insert(Config) ->
connect_and_clear_table(Config),
?assertMatch(
{ok, _},
create_bridge(Config)
),
Size = 5,
Ts = erlang:system_time(millisecond),
{_, {ok, #{result := _Result}}} =
?wait_async_action(
lists:foreach(
fun(Idx) ->
SentData = #{payload => ?PAYLOAD, timestamp => Ts + Idx},
Request = {send_message, SentData},
query_resource(Config, Request)
end,
lists:seq(1, Size)
),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?retry(
_Sleep = 50,
_Attempts = 30,
?assertMatch(
[[Size]],
connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg")
)
).
t_auto_create_simple_insert(Config0) ->
ClientId = to_str(?FUNCTION_NAME),
Config = get_auto_create_config(Config0),
?assertMatch(
{ok, _},
create_bridge(Config)
),
SentData = #{
payload => ?PAYLOAD,
timestamp => 1668602148000,
clientid => ClientId
},
Request = {send_message, SentData},
{_, {ok, #{result := _Result}}} =
?wait_async_action(
query_resource(Config, Request),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?assertMatch(
[[?PAYLOAD]],
connect_and_query(Config, "SELECT payload FROM " ++ ClientId)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId)
).
t_auto_create_batch_insert(Config0) ->
ClientId1 = "client1",
ClientId2 = "client2",
Config = get_auto_create_config(Config0),
?assertMatch(
{ok, _},
create_bridge(Config)
),
Size1 = 2,
Size2 = 3,
Ts = erlang:system_time(millisecond),
{_, {ok, #{result := _Result}}} =
?wait_async_action(
lists:foreach(
fun({Offset, ClientId, Size}) ->
lists:foreach(
fun(Idx) ->
SentData = #{
payload => ?PAYLOAD,
timestamp => Ts + Idx + Offset,
clientid => ClientId
},
Request = {send_message, SentData},
query_resource(Config, Request)
end,
lists:seq(1, Size)
)
end,
[{0, ClientId1, Size1}, {100, ClientId2, Size2}]
),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?retry(
_Sleep = 50,
_Attempts = 30,
?assertMatch(
[[Size1]],
connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1)
)
),
?retry(
50,
30,
?assertMatch(
[[Size2]],
connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2)
)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId1)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId2)
).
to_bin(List) when is_list(List) ->
unicode:characters_to_binary(List, utf8);
to_bin(Bin) when is_binary(Bin) ->
Bin.
to_str(Atom) when is_atom(Atom) ->
erlang:atom_to_list(Atom).
get_auto_create_config(Config0) ->
Config = lists:keyreplace(template, 1, Config0, {template, ?AUTO_CREATE_BRIDGE}),
BridgeType = proplists:get_value(bridge_type, Config, <<"tdengine">>),
{_Name, TDConf} = tdengine_config(BridgeType, Config),
lists:keyreplace(tdengine_config, 1, Config, {tdengine_config, TDConf}).

4
apps/emqx_connector/src/emqx_connector_mqtt.erl
View File

@ -112,7 +112,7 @@ bridge_spec(Config) ->
id => Name,
start => {emqx_connector_mqtt_worker, start_link, [Name, NConfig]},
restart => temporary,
shutdown => 5000
shutdown => 1000
}.
-spec bridges() -> [{_Name, _Status}].
@ -181,7 +181,7 @@ on_stop(_InstId, #{name := InstanceId}) ->
ok;
{error, Reason} ->
?SLOG(error, #{
msg => "stop_mqtt_connector",
msg => "stop_mqtt_connector_error",
connector => InstanceId,
reason => Reason
})

6
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
View File

@ -202,13 +202,13 @@ connect(Name) ->
Error
end;
{error, Reason} = Error ->
?SLOG(error, #{
?SLOG(warning, #{
msg => "client_connect_failed",
reason => Reason
reason => Reason,
name => Name
}),
Error
end.
subscribe_remote_topics(Ref, #{remote := #{topic := FromTopic, qos := QoS}}) ->
emqtt:subscribe(ref(Ref), FromTopic, QoS);
subscribe_remote_topics(_Ref, undefined) ->

2
apps/emqx_dashboard/src/emqx_dashboard.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_dashboard, [
{description, "EMQX Web Dashboard"},
% strict semver, bump manually!
{vsn, "5.0.20"},
{vsn, "5.0.21"},
{modules, []},
{registered, [emqx_dashboard_sup]},
{applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]},

46
apps/emqx_dashboard/src/emqx_dashboard.erl
View File

@ -32,8 +32,6 @@
-include_lib("emqx/include/http_api.hrl").
-include_lib("emqx/include/emqx_release.hrl").
-define(BASE_PATH, "/api/v5").
-define(EMQX_MIDDLE, emqx_dashboard_middleware).
%%--------------------------------------------------------------------
@ -52,7 +50,7 @@ start_listeners(Listeners) ->
GlobalSpec = #{
openapi => "3.0.0",
info => #{title => "EMQX API", version => ?EMQX_API_VERSION},
servers => [#{url => ?BASE_PATH}],
servers => [#{url => emqx_dashboard_swagger:base_path()}],
components => #{
schemas => #{},
'securitySchemes' => #{
@ -69,11 +67,11 @@ start_listeners(Listeners) ->
{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}},
{"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}},
{emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []},
{?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []},
{emqx_dashboard_swagger:relative_uri("/[...]"), emqx_dashboard_bad_api, []},
{'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}
],
BaseMinirest = #{
base_path => ?BASE_PATH,
base_path => emqx_dashboard_swagger:base_path(),
modules => minirest_api:find_api_modules(apps()),
authorization => Authorization,
security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
@ -97,7 +95,7 @@ start_listeners(Listeners) ->
end
end,
{[], []},
listeners(Listeners)
listeners(ensure_ssl_cert(Listeners))
),
case ErrListeners of
[] ->
@ -142,18 +140,18 @@ apps() ->
listeners(Listeners) ->
lists:filtermap(
fun({Protocol, Conf}) ->
maps:get(enable, Conf) andalso
begin
{Conf1, Bind} = ip_port(Conf),
{true, {
listener_name(Protocol),
Protocol,
Bind,
ranch_opts(Conf1),
proto_opts(Conf1)
}}
end
fun
({Protocol, Conf = #{enable := true}}) ->
{Conf1, Bind} = ip_port(Conf),
{true, {
listener_name(Protocol),
Protocol,
Bind,
ranch_opts(Conf1),
proto_opts(Conf1)
}};
({_Protocol, #{enable := false}}) ->
false
end,
maps:to_list(Listeners)
).
@ -193,8 +191,8 @@ ranch_opts(Options) ->
end,
RanchOpts#{socket_opts => InetOpts ++ SocketOpts}.
proto_opts(Options) ->
maps:with([proxy_header], Options).
proto_opts(#{proxy_header := ProxyHeader}) ->
#{proxy_header => ProxyHeader}.
filter_false(_K, false, S) -> S;
filter_false(K, V, S) -> [{K, V} | S].
@ -226,7 +224,7 @@ return_unauthorized(Code, Message) ->
{401,
#{
<<"WWW-Authenticate">> =>
<<"Basic Realm=\"minirest-server\"">>
<<"Basic Realm=\"emqx-dashboard\"">>
},
#{code => Code, message => Message}}.
@ -249,3 +247,9 @@ api_key_authorize(Req, Key, Secret) ->
<<"Check api_key/api_secret">>
)
end.
ensure_ssl_cert(Listeners = #{https := Https0}) ->
Https1 = emqx_tls_lib:to_server_opts(tls, Https0),
Listeners#{https => maps:from_list(Https1)};
ensure_ssl_cert(Listeners) ->
Listeners.

5
apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
View File

@ -20,6 +20,7 @@
-include_lib("emqx/include/http_api.hrl").
-include("emqx_dashboard.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([
api_spec/0,
@ -50,7 +51,7 @@ schema("/error_codes") ->
'operationId' => error_codes,
get => #{
security => [],
description => <<"API Error Codes">>,
description => ?DESC(error_codes),
tags => [<<"Error Codes">>],
responses => #{
200 => hoconsc:array(hoconsc:ref(?MODULE, error_code))
@ -62,7 +63,7 @@ schema("/error_codes/:code") ->
'operationId' => error_code,
get => #{
security => [],
description => <<"API Error Codes">>,
description => ?DESC(error_codes_u),
tags => [<<"Error Codes">>],
parameters => [
{code,

9
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -6,6 +6,7 @@
-include("emqx_dashboard.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hocon_types.hrl").
-behaviour(minirest_api).
@ -38,7 +39,7 @@ schema("/monitor") ->
'operationId' => monitor,
get => #{
tags => [<<"Metrics">>],
desc => <<"List monitor data.">>,
description => ?DESC(list_monitor),
parameters => [parameter_latest()],
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}),
@ -51,7 +52,7 @@ schema("/monitor/nodes/:node") ->
'operationId' => monitor,
get => #{
tags => [<<"Metrics">>],
desc => <<"List the monitor data on the node.">>,
description => ?DESC(list_monitor_node),
parameters => [parameter_node(), parameter_latest()],
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}),
@ -64,7 +65,7 @@ schema("/monitor_current") ->
'operationId' => monitor_current,
get => #{
tags => [<<"Metrics">>],
desc => <<"Current status. Gauge and rate.">>,
description => ?DESC(current_status),
responses => #{
200 => hoconsc:mk(hoconsc:ref(sampler_current), #{})
}
@ -75,7 +76,7 @@ schema("/monitor_current/nodes/:node") ->
'operationId' => monitor_current,
get => #{
tags => [<<"Metrics">>],
desc => <<"Node current status. Gauge and rate.">>,
description => ?DESC(current_status_node),
parameters => [parameter_node()],
responses => #{
200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}),

78
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -19,12 +19,17 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-define(BASE_PATH, "/api/v5").
%% API
-export([spec/1, spec/2]).
-export([namespace/0, namespace/1, fields/1]).
-export([schema_with_example/2, schema_with_examples/2]).
-export([error_codes/1, error_codes/2]).
-export([file_schema/1]).
-export([base_path/0]).
-export([relative_uri/1]).
-export([compose_filters/2]).
-export([
filter_check_request/2,
@ -84,14 +89,30 @@
-type request() :: #{bindings => map(), query_string => map(), body => map()}.
-type request_meta() :: #{module => module(), path => string(), method => atom()}.
-type filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}.
-type filter() :: fun((request(), request_meta()) -> filter_result()).
%% More exact types are defined in minirest.hrl, but we don't want to include it
%% because it defines a lot of types and they may clash with the types declared locally.
-type status_code() :: pos_integer().
-type error_code() :: atom() | binary().
-type error_message() :: binary().
-type response_body() :: term().
-type headers() :: map().
-type response() ::
status_code()
| {status_code()}
| {status_code(), response_body()}
| {status_code(), headers(), response_body()}
| {status_code(), error_code(), error_message()}.
-type filter_result() :: {ok, request()} | response().
-type filter() :: emqx_maybe:t(fun((request(), request_meta()) -> filter_result())).
-type spec_opts() :: #{
check_schema => boolean() | filter(),
translate_body => boolean(),
schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()),
i18n_lang => atom() | string() | binary()
i18n_lang => atom() | string() | binary(),
filter => filter()
}.
-type route_path() :: string() | binary().
@ -117,9 +138,9 @@ spec(Module, Options) ->
lists:foldl(
fun(Path, {AllAcc, AllRefsAcc}) ->
{OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options),
CheckSchema = support_check_schema(Options),
Opts = #{filter => filter(Options)},
{
[{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc],
[{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc],
Refs ++ AllRefsAcc
}
end,
@ -184,6 +205,14 @@ error_codes(Codes = [_ | _], MsgDesc) ->
})}
].
-spec base_path() -> uri_string:uri_string().
base_path() ->
?BASE_PATH.
-spec relative_uri(uri_string:uri_string()) -> uri_string:uri_string().
relative_uri(Uri) ->
base_path() ++ Uri.
file_schema(FileName) ->
#{
content => #{
@ -242,6 +271,21 @@ gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Converter) ->
[pretty, force_utf8]
).
-spec compose_filters(filter(), filter()) -> filter().
compose_filters(undefined, Filter2) ->
Filter2;
compose_filters(Filter1, undefined) ->
Filter1;
compose_filters(Filter1, Filter2) ->
fun(Request, RequestMeta) ->
case Filter1(Request, RequestMeta) of
{ok, Request1} ->
Filter2(Request1, RequestMeta);
Response ->
Response
end
end.
%%------------------------------------------------------------------------------
%% Private functions
%%------------------------------------------------------------------------------
@ -273,14 +317,22 @@ check_only(Schema, Map, Opts) ->
_ = hocon_tconf:check_plain(Schema, Map, Opts),
Map.
support_check_schema(#{check_schema := true, translate_body := true}) ->
#{filter => fun ?MODULE:filter_check_request_and_translate_body/2};
support_check_schema(#{check_schema := true}) ->
#{filter => fun ?MODULE:filter_check_request/2};
support_check_schema(#{check_schema := Filter}) when is_function(Filter, 2) ->
#{filter => Filter};
support_check_schema(_) ->
#{filter => undefined}.
filter(Options) ->
CheckSchemaFilter = check_schema_filter(Options),
CustomFilter = custom_filter(Options),
compose_filters(CheckSchemaFilter, CustomFilter).
custom_filter(Options) ->
maps:get(filter, Options, undefined).
check_schema_filter(#{check_schema := true, translate_body := true}) ->
fun ?MODULE:filter_check_request_and_translate_body/2;
check_schema_filter(#{check_schema := true}) ->
fun ?MODULE:filter_check_request/2;
check_schema_filter(#{check_schema := Filter}) when is_function(Filter, 2) ->
Filter;
check_schema_filter(_) ->
undefined.
parse_spec_ref(Module, Path, Options) ->
Schema =

6
apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
View File

@ -36,8 +36,6 @@
-define(HOST, "http://127.0.0.1:18083").
%% -define(API_VERSION, "v5").
-define(BASE_PATH, "/api/v5").
-define(APP_DASHBOARD, emqx_dashboard).
@ -57,6 +55,10 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
%% Load all applications to ensure swagger.json is fully generated.
Apps = emqx_machine_boot:reboot_apps(),
ct:pal("load apps:~p~n", [Apps]),
lists:foreach(fun(App) -> application:load(App) end, Apps),
emqx_mgmt_api_test_util:init_suite([emqx_management]),
Config.

8
apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl
View File

@ -26,11 +26,12 @@
request/4,
multipart_formdata_request/3,
multipart_formdata_request/4,
host/0,
uri/0,
uri/1
]).
-define(HOST, "http://127.0.0.1:18083/").
-define(HOST, "http://127.0.0.1:18083").
-define(API_VERSION, "v5").
-define(BASE_PATH, "api").
@ -98,10 +99,13 @@ request(Username, Method, Url, Body) ->
{error, Reason}
end.
host() ->
?HOST.
uri() -> uri([]).
uri(Parts) when is_list(Parts) ->
NParts = [E || E <- Parts],
?HOST ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])).
host() ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])).
auth_header(Username) ->
Password = <<"public">>,

214
apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl Normal file
View File

@ -0,0 +1,214 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_dashboard_https_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include("emqx_dashboard.hrl").
-define(NAME, 'https:dashboard').
-define(HOST, "https://127.0.0.1:18084").
-define(BASE_PATH, "/api/v5").
-define(OVERVIEWS, [
"alarms",
"banned",
"stats",
"metrics",
"listeners",
"clients",
"subscriptions"
]).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) -> Config.
end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]).
init_per_testcase(_TestCase, Config) -> Config.
end_per_testcase(_TestCase, _Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]).
t_default_ssl_cert(_Config) ->
Conf = #{dashboard => #{listeners => #{https => #{bind => 18084, enable => true}}}},
validate_https(Conf, 512, default_ssl_cert(), verify_none),
ok.
t_normal_ssl_cert(_Config) ->
MaxConnection = 1000,
Conf = #{
dashboard => #{
listeners => #{
https => #{
bind => 18084,
enable => true,
cacertfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>),
certfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cert.pem">>),
keyfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/key.pem">>),
max_connections => MaxConnection
}
}
}
},
validate_https(Conf, MaxConnection, default_ssl_cert(), verify_none),
ok.
t_verify_cacertfile(_Config) ->
MaxConnection = 1024,
DefaultSSLCert = default_ssl_cert(),
SSLCert = DefaultSSLCert#{cacertfile => <<"">>},
%% default #{verify => verify_none}
Conf = #{
dashboard => #{
listeners => #{
https => #{
bind => 18084,
enable => true,
cacertfile => <<"">>,
max_connections => MaxConnection
}
}
}
},
validate_https(Conf, MaxConnection, SSLCert, verify_none),
%% verify_peer but cacertfile is empty
VerifyPeerConf1 = emqx_utils_maps:deep_put(
[dashboard, listeners, https, verify],
Conf,
verify_peer
),
emqx_common_test_helpers:load_config(emqx_dashboard_schema, VerifyPeerConf1),
?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()),
%% verify_peer and cacertfile is ok.
VerifyPeerConf2 = emqx_utils_maps:deep_put(
[dashboard, listeners, https, cacertfile],
VerifyPeerConf1,
naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>)
),
validate_https(VerifyPeerConf2, MaxConnection, DefaultSSLCert, verify_peer),
ok.
t_bad_certfile(_Config) ->
Conf = #{
dashboard => #{
listeners => #{
https => #{
bind => 18084,
enable => true,
certfile => <<"${EMQX_ETC_DIR}/certs/not_found_cert.pem">>
}
}
}
},
emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf),
?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()),
ok.
validate_https(Conf, MaxConnection, SSLCert, Verify) ->
emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf),
emqx_mgmt_api_test_util:init_suite([emqx_management], fun(X) -> X end),
assert_ranch_options(MaxConnection, SSLCert, Verify),
assert_https_request(),
emqx_mgmt_api_test_util:end_suite([emqx_management]).
assert_ranch_options(MaxConnections0, SSLCert, Verify) ->
Middlewares = [emqx_dashboard_middleware, cowboy_router, cowboy_handler],
[
?NAME,
ranch_ssl,
#{
max_connections := MaxConnections,
num_acceptors := _,
socket_opts := SocketOpts
},
cowboy_tls,
#{
env := #{
dispatch := {persistent_term, ?NAME},
options := #{
name := ?NAME,
protocol := https,
protocol_options := #{proxy_header := false},
security := [#{basicAuth := []}, #{bearerAuth := []}],
swagger_global_spec := _
}
},
middlewares := Middlewares,
proxy_header := false
}
] = ranch_server:get_listener_start_args(?NAME),
?assertEqual(MaxConnections0, MaxConnections),
?assert(lists:member(inet, SocketOpts), SocketOpts),
#{
backlog := 1024,
ciphers := Ciphers,
port := 18084,
send_timeout := 10000,
verify := Verify,
versions := Versions
} = SocketMaps = maps:from_list(SocketOpts -- [inet]),
%% without tlsv1.1 tlsv1
?assertMatch(['tlsv1.3', 'tlsv1.2'], Versions),
?assert(Ciphers =/= []),
maps:foreach(
fun(K, ConfVal) ->
case maps:find(K, SocketMaps) of
{ok, File} -> ?assertEqual(naive_env_interpolation(ConfVal), File);
error -> ?assertEqual(<<"">>, ConfVal)
end
end,
SSLCert
),
?assertMatch(
#{
env := #{dispatch := {persistent_term, ?NAME}},
middlewares := Middlewares,
proxy_header := false
},
ranch:get_protocol_options(?NAME)
),
ok.
assert_https_request() ->
Headers = emqx_dashboard_SUITE:auth_header_(),
lists:foreach(
fun(Path) ->
ApiPath = api_path([Path]),
?assertMatch(
{ok, _},
emqx_dashboard_SUITE:request_dashboard(get, ApiPath, Headers)
)
end,
?OVERVIEWS
).
api_path(Parts) ->
?HOST ++ filename:join([?BASE_PATH | Parts]).
naive_env_interpolation(Str0) ->
Str1 = emqx_schema:naive_env_interpolation(Str0),
%% assert all envs are replaced
?assertNot(lists:member($$, Str1)),
Str1.
default_ssl_cert() ->
#{
cacertfile => <<"${EMQX_ETC_DIR}/certs/cacert.pem">>,
certfile => <<"${EMQX_ETC_DIR}/certs/cert.pem">>,
keyfile => <<"${EMQX_ETC_DIR}/certs/key.pem">>
}.

54
apps/emqx_enterprise/src/emqx_enterprise_schema.erl
View File

@ -6,28 +6,24 @@
-behaviour(hocon_schema).
-export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]).
-export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]).
-define(EE_SCHEMA_MODULES, [emqx_license_schema, emqx_ee_schema_registry_schema]).
-define(EE_SCHEMA_MODULES, [
emqx_license_schema,
emqx_ee_schema_registry_schema,
emqx_ft_schema
]).
namespace() ->
emqx_conf_schema:namespace().
roots() ->
redefine_roots(
lists:foldl(
fun(Module, Roots) ->
Roots ++ apply(Module, roots, [])
end,
emqx_conf_schema:roots(),
?EE_SCHEMA_MODULES
)
).
redefine_roots(emqx_conf_schema:roots()) ++ ee_roots().
fields("node") ->
redefine_node(emqx_conf_schema:fields("node"));
fields(Name) ->
emqx_conf_schema:fields(Name).
ee_delegate(fields, ?EE_SCHEMA_MODULES, Name).
translations() ->
emqx_conf_schema:translations().
@ -35,17 +31,42 @@ translations() ->
translation(Name) ->
emqx_conf_schema:translation(Name).
desc(Name) ->
ee_delegate(desc, ?EE_SCHEMA_MODULES, Name).
validations() ->
emqx_conf_schema:validations().
redefine_node(Fields) ->
Overrides = [{"applications", #{default => <<"emqx_license">>}}],
override(Fields, Overrides).
%%------------------------------------------------------------------------------
%% helpers
%%------------------------------------------------------------------------------
ee_roots() ->
lists:flatmap(
fun(Module) ->
apply(Module, roots, [])
end,
?EE_SCHEMA_MODULES
).
ee_delegate(Method, [EEMod | EEMods], Name) ->
case lists:member(Name, apply(EEMod, roots, [])) of
true ->
apply(EEMod, Method, [Name]);
false ->
ee_delegate(Method, EEMods, Name)
end;
ee_delegate(Method, [], Name) ->
apply(emqx_conf_schema, Method, [Name]).
redefine_roots(Roots) ->
Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}],
override(Roots, Overrides).
redefine_node(Fields) ->
Overrides = [{"applications", #{default => <<"emqx_license">>}}],
override(Fields, Overrides).
override(Fields, []) ->
Fields;
override(Fields, [{Name, Override} | More]) ->
@ -60,6 +81,3 @@ find_schema(Name, Fields) ->
replace_schema(Name, Schema, Fields) ->
lists:keyreplace(Name, 1, Fields, {Name, Schema}).
desc(Name) ->
emqx_conf_schema:desc(Name).

3
apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
View File

@ -1,3 +0,0 @@
##--------------------------------------------------------------------
## EMQX Eviction Agent Plugin
##--------------------------------------------------------------------

94
apps/emqx_ft/BSL.txt Normal file
View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2023
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2027-02-01
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

86
apps/emqx_ft/README.md Normal file
View File

@ -0,0 +1,86 @@
# EMQX File Transfer
EMQX File Transfer application enables the _File Transfer over MQTT_ feature described in [EIP-0021](https://github.com/emqx/eip), and provides support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage.
## Usage
As almost any other EMQX application, `emqx_ft` is configured via the EMQX configuration system. The following snippet is the minimal configuration that will enable File Transfer over MQTT.
```
file_transfer {
enable = true
}
```
The configuration above will make File Transfer available to all MQTT clients, and will use the default storage backend, which in turn uses node-local file system both for temporary storage and for the final destination of the transferred files.
## Configuration
Every configuration parameter is described in the `emqx_ft_schema` module.
The most important configuration parameter is `storage`, which defines the storage backend to use. Currently, only `local` storage backend is available, which stores all the temporary data accumulating during file transfers in the node-local file system. Those go into `${EMQX_DATA_DIR}/file_transfer` directory by default, but can be configured via `local.storage.segments.root` parameter. The final destination of the transferred files on the other hand is defined by `local.storage.exporter` parameter, and currently can be either `local` or `s3`.
### Local Exporter
The `local` exporter is the default one, and it stores the transferred files in the node-local file system. The final destination directory is defined by `local.storage.exporter.local.root` parameter, and defaults to `${EMQX_DATA_DIR}/file_transfer/exports` directory.
```
file_transfer {
enable = true
storage {
local {
exporter {
local { root = "/var/lib/emqx/transfers" }
}
}
}
}
```
Important to note that even though the transferred files go into the node-local file system, the File Transfer API provides a cluster-wide view of the transferred files, and any file can be downloaded from any node in the cluster.
### S3 Exporter
The `s3` exporter stores the transferred files in the S3 API compatible remote object storage. The destination bucket is defined by `local.storage.exporter.s3.bucket` parameter.
This snippet configures File Transfer to store the transferred files in the `my-bucket` bucket in the `us-east-1` region of the AWS S3 service.
```
file_transfer {
enable = true
storage {
local {
exporter {
s3 {
host = "s3.us-east-1.amazonaws.com"
port = "443"
access_key_id = "AKIA27EZDDM9XLINWXFE"
secret_access_key = "..."
bucket = "my-bucket"
}
}
}
}
}
```
## API
### MQTT
When enabled, File Transfer application reserves MQTT topics starting with `$file/` prefix for the purpose of serving the File Transfer protocol, as described in [EIP-0021](https://github.com/emqx/eip).
### REST
Application publishes a basic set of APIs, to:
* List all the transferred files available for download.
* Configure the application, including the storage backend.
* (When using `local` storage exporter) Download the transferred files.
Switching to the `s3` storage exporter is possible at any time, but the files transferred before the switch will not be
available for download anymore. Though, the files will still be available in the node-local file system.
## Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).

1
apps/emqx_ft/docker-ct Normal file
View File

@ -0,0 +1 @@
minio

0
apps/emqx_ft/etc/emqx_ft.conf Normal file
View File

29
apps/emqx_ft/include/emqx_ft_storage_fs.hrl Normal file
View File

@ -0,0 +1,29 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_FT_STORAGE_FS_HRL).
-define(EMQX_FT_STORAGE_FS_HRL, true).
-record(gcstats, {
started_at :: integer(),
finished_at :: integer() | undefined,
files = 0 :: non_neg_integer(),
directories = 0 :: non_neg_integer(),
space = 0 :: non_neg_integer(),
errors = #{} :: #{_GCSubject => {error, _}}
}).
-endif.

11
apps/emqx_ft/rebar.config Normal file
View File

@ -0,0 +1,11 @@
%% -*- mode: erlang -*-
{erl_opts, [debug_info]}.
{deps, [{emqx, {path, "../emqx"}}]}.
{shell, [
% {config, "config/sys.config"},
{apps, [emqx_ft]}
]}.
{project_plugins, [erlfmt]}.

14
apps/emqx_ft/src/emqx_ft.app.src Normal file
View File

@ -0,0 +1,14 @@
{application, emqx_ft, [
{description, "EMQX file transfer over MQTT"},
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_ft_app, []}},
{applications, [
kernel,
stdlib,
gproc,
emqx_s3
]},
{env, []},
{modules, []}
]}.

425
apps/emqx_ft/src/emqx_ft.erl Normal file
View File

@ -0,0 +1,425 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-export([
hook/0,
unhook/0
]).
-export([
on_message_publish/1,
on_message_puback/4
]).
-export([
decode_filemeta/1,
encode_filemeta/1
]).
-export([on_complete/4]).
-export_type([
clientid/0,
transfer/0,
bytes/0,
offset/0,
filemeta/0,
segment/0,
checksum/0
]).
%% Number of bytes
-type bytes() :: non_neg_integer().
%% MQTT Client ID
-type clientid() :: binary().
-type fileid() :: binary().
-type transfer() :: {clientid(), fileid()}.
-type offset() :: bytes().
-type checksum() :: {_Algo :: atom(), _Digest :: binary()}.
-type filemeta() :: #{
%% Display name
name := string(),
%% Size in bytes, as advertised by the client.
%% Client is free to specify here whatever it wants, which means we can end
%% up with a file of different size after assembly. It's not clear from
%% specification what that means (e.g. what are clients' expectations), we
%% currently do not condider that an error (or, specifically, a signal that
%% the resulting file is corrupted during transmission).
size => _Bytes :: non_neg_integer(),
checksum => checksum(),
expire_at := emqx_datetime:epoch_second(),
%% TTL of individual segments
%% Somewhat confusing that we won't know it on the nodes where the filemeta
%% is missing.
segments_ttl => _Seconds :: pos_integer(),
user_data => emqx_ft_schema:json_value()
}.
-type segment() :: {offset(), _Content :: binary()}.
%%--------------------------------------------------------------------
%% API for app
%%--------------------------------------------------------------------
hook() ->
ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST),
ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST).
unhook() ->
ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}),
ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
decode_filemeta(Payload) when is_binary(Payload) ->
case emqx_utils_json:safe_decode(Payload, [return_maps]) of
{ok, Map} ->
decode_filemeta(Map);
{error, Error} ->
{error, {invalid_filemeta_json, Error}}
end;
decode_filemeta(Map) when is_map(Map) ->
Schema = emqx_ft_schema:schema(filemeta),
try
Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}),
{ok, Meta}
catch
throw:{_Schema, Errors} ->
{error, {invalid_filemeta, Errors}}
end.
encode_filemeta(Meta = #{}) ->
Schema = emqx_ft_schema:schema(filemeta),
hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Meta), #{}).
%%--------------------------------------------------------------------
%% Hooks
%%--------------------------------------------------------------------
on_message_publish(
Msg = #message{
id = _Id,
topic = <<"$file/", _/binary>>
}
) ->
Headers = Msg#message.headers,
{stop, Msg#message{headers = Headers#{allow_publish => false}}};
on_message_publish(Msg) ->
{ok, Msg}.
on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) ->
case Topic of
<<"$file/", FileCommand/binary>> ->
{stop, on_file_command(PacketId, Msg, FileCommand)};
_ ->
ignore
end.
%%--------------------------------------------------------------------
%% Handlers for transfer messages
%%--------------------------------------------------------------------
%% TODO Move to emqx_ft_mqtt?
on_file_command(PacketId, Msg, FileCommand) ->
case emqx_topic:tokens(FileCommand) of
[FileIdIn | Rest] ->
validate([{fileid, FileIdIn}], fun([FileId]) ->
on_file_command(PacketId, FileId, Msg, Rest)
end);
[] ->
?RC_UNSPECIFIED_ERROR
end.
on_file_command(PacketId, FileId, Msg, FileCommand) ->
Transfer = transfer(Msg, FileId),
case FileCommand of
[<<"init">>] ->
validate(
[{filemeta, Msg#message.payload}],
fun([Meta]) ->
on_init(PacketId, Msg, Transfer, Meta)
end
);
[<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 ->
ChecksumBin = emqx_maybe:from_list(MaybeChecksum),
validate(
[{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}],
fun([FinalSize, Checksum]) ->
on_fin(PacketId, Msg, Transfer, FinalSize, Checksum)
end
);
[<<"abort">>] ->
on_abort(Msg, Transfer);
[OffsetBin] ->
validate([{offset, OffsetBin}], fun([Offset]) ->
on_segment(PacketId, Msg, Transfer, Offset, undefined)
end);
[OffsetBin, ChecksumBin] ->
validate(
[{offset, OffsetBin}, {checksum, ChecksumBin}],
fun([Offset, Checksum]) ->
validate(
[{integrity, Msg#message.payload, Checksum}],
fun(_) ->
on_segment(PacketId, Msg, Transfer, Offset, Checksum)
end
)
end
);
_ ->
?RC_UNSPECIFIED_ERROR
end.
on_init(PacketId, Msg, Transfer, Meta) ->
?tp(info, "file_transfer_init", #{
mqtt_msg => Msg,
packet_id => PacketId,
transfer => Transfer,
filemeta => Meta
}),
PacketKey = {self(), PacketId},
Callback = fun(Result) ->
?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result)
end,
with_responder(PacketKey, Callback, emqx_ft_conf:init_timeout(), fun() ->
case store_filemeta(Transfer, Meta) of
% Stored, ack through the responder right away
ok ->
emqx_ft_responder:ack(PacketKey, ok);
% Storage operation started, packet will be acked by the responder
% {async, Pid} ->
% ok = emqx_ft_responder:kickoff(PacketKey, Pid),
% ok;
%% Storage operation failed, ack through the responder
{error, _} = Error ->
emqx_ft_responder:ack(PacketKey, Error)
end
end).
on_abort(_Msg, _FileId) ->
%% TODO
?RC_SUCCESS.
on_segment(PacketId, Msg, Transfer, Offset, Checksum) ->
?tp(info, "file_transfer_segment", #{
mqtt_msg => Msg,
packet_id => PacketId,
transfer => Transfer,
offset => Offset,
checksum => Checksum
}),
Segment = {Offset, Msg#message.payload},
PacketKey = {self(), PacketId},
Callback = fun(Result) ->
?MODULE:on_complete("store_segment", PacketKey, Transfer, Result)
end,
with_responder(PacketKey, Callback, emqx_ft_conf:store_segment_timeout(), fun() ->
case store_segment(Transfer, Segment) of
ok ->
emqx_ft_responder:ack(PacketKey, ok);
% {async, Pid} ->
% ok = emqx_ft_responder:kickoff(PacketKey, Pid),
% ok;
{error, _} = Error ->
emqx_ft_responder:ack(PacketKey, Error)
end
end).
on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) ->
?tp(info, "file_transfer_fin", #{
mqtt_msg => Msg,
packet_id => PacketId,
transfer => Transfer,
final_size => FinalSize,
checksum => Checksum
}),
%% TODO: handle checksum? Do we need it?
FinPacketKey = {self(), PacketId},
Callback = fun(Result) ->
?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result)
end,
with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() ->
case assemble(Transfer, FinalSize) of
%% Assembling completed, ack through the responder right away
ok ->
emqx_ft_responder:ack(FinPacketKey, ok);
%% Assembling started, packet will be acked by the responder
{async, Pid} ->
ok = emqx_ft_responder:kickoff(FinPacketKey, Pid),
ok;
%% Assembling failed, ack through the responder
{error, _} = Error ->
emqx_ft_responder:ack(FinPacketKey, Error)
end
end).
with_responder(Key, Callback, Timeout, CriticalSection) ->
case emqx_ft_responder:start(Key, Callback, Timeout) of
%% We have new packet
{ok, _} ->
CriticalSection();
%% Packet already received.
%% Since we are still handling the previous one,
%% we probably have retransmit here
{error, {already_started, _}} ->
ok
end,
undefined.
store_filemeta(Transfer, Segment) ->
try
emqx_ft_storage:store_filemeta(Transfer, Segment)
catch
C:E:S ->
?tp(error, "start_store_filemeta_failed", #{
class => C, reason => E, stacktrace => S
}),
{error, {internal_error, E}}
end.
store_segment(Transfer, Segment) ->
try
emqx_ft_storage:store_segment(Transfer, Segment)
catch
C:E:S ->
?tp(error, "start_store_segment_failed", #{
class => C, reason => E, stacktrace => S
}),
{error, {internal_error, E}}
end.
assemble(Transfer, FinalSize) ->
try
emqx_ft_storage:assemble(Transfer, FinalSize)
catch
C:E:S ->
?tp(error, "start_assemble_failed", #{
class => C, reason => E, stacktrace => S
}),
{error, {internal_error, E}}
end.
transfer(Msg, FileId) ->
ClientId = Msg#message.from,
{clientid_to_binary(ClientId), FileId}.
on_complete(Op, {ChanPid, PacketId}, Transfer, Result) ->
?tp(debug, "on_complete", #{
operation => Op,
packet_id => PacketId,
transfer => Transfer
}),
case Result of
{Mode, ok} when Mode == ack orelse Mode == down ->
erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS});
{Mode, {error, _} = Reason} when Mode == ack orelse Mode == down ->
?tp(error, Op ++ "_failed", #{
transfer => Transfer,
reason => Reason
}),
erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR});
timeout ->
?tp(error, Op ++ "_timed_out", #{
transfer => Transfer
}),
erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR})
end.
validate(Validations, Fun) ->
case do_validate(Validations, []) of
{ok, Parsed} ->
Fun(Parsed);
{error, Reason} ->
?tp(info, "client_violated_protocol", #{reason => Reason}),
?RC_UNSPECIFIED_ERROR
end.
do_validate([], Parsed) ->
{ok, lists:reverse(Parsed)};
do_validate([{fileid, FileId} | Rest], Parsed) ->
case byte_size(FileId) of
S when S > 0 ->
do_validate(Rest, [FileId | Parsed]);
0 ->
{error, {invalid_fileid, FileId}}
end;
do_validate([{filemeta, Payload} | Rest], Parsed) ->
case decode_filemeta(Payload) of
{ok, Meta} ->
do_validate(Rest, [Meta | Parsed]);
{error, Reason} ->
{error, Reason}
end;
do_validate([{offset, Offset} | Rest], Parsed) ->
case string:to_integer(Offset) of
{Int, <<>>} ->
do_validate(Rest, [Int | Parsed]);
_ ->
{error, {invalid_offset, Offset}}
end;
do_validate([{size, Size} | Rest], Parsed) ->
case string:to_integer(Size) of
{Int, <<>>} ->
do_validate(Rest, [Int | Parsed]);
_ ->
{error, {invalid_size, Size}}
end;
do_validate([{checksum, Checksum} | Rest], Parsed) ->
case parse_checksum(Checksum) of
{ok, Bin} ->
do_validate(Rest, [Bin | Parsed]);
{error, _Reason} ->
{error, {invalid_checksum, Checksum}}
end;
do_validate([{integrity, Payload, Checksum} | Rest], Parsed) ->
case crypto:hash(sha256, Payload) of
Checksum ->
do_validate(Rest, [Payload | Parsed]);
Mismatch ->
{error, {checksum_mismatch, binary:encode_hex(Mismatch)}}
end;
do_validate([{{maybe, _}, undefined} | Rest], Parsed) ->
do_validate(Rest, [undefined | Parsed]);
do_validate([{{maybe, T}, Value} | Rest], Parsed) ->
do_validate([{T, Value} | Rest], Parsed).
parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 ->
try
{ok, binary:decode_hex(Checksum)}
catch
error:badarg ->
{error, invalid_checksum}
end;
parse_checksum(_Checksum) ->
{error, invalid_checksum}.
clientid_to_binary(A) when is_atom(A) ->
atom_to_binary(A);
clientid_to_binary(B) when is_binary(B) ->
B.

239
apps/emqx_ft/src/emqx_ft_api.erl Normal file
View File

@ -0,0 +1,239 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_api).
-behaviour(minirest_api).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
%% Swagger specs from hocon schema
-export([
api_spec/0,
paths/0,
schema/1,
namespace/0
]).
-export([
roots/0,
fields/1
]).
%% Minirest filter for checking if file transfer is enabled
-export([check_ft_enabled/2]).
%% API callbacks
-export([
'/file_transfer/files'/2,
'/file_transfer/files/:clientid/:fileid'/2
]).
-import(hoconsc, [mk/2, ref/1, ref/2]).
namespace() -> "file_transfer".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{
check_schema => true, filter => fun ?MODULE:check_ft_enabled/2
}).
paths() ->
[
"/file_transfer/files",
"/file_transfer/files/:clientid/:fileid"
].
schema("/file_transfer/files") ->
#{
'operationId' => '/file_transfer/files',
get => #{
tags => [<<"file_transfer">>],
summary => <<"List all uploaded files">>,
description => ?DESC("file_list"),
parameters => [
ref(following),
ref(emqx_dashboard_swagger, limit)
],
responses => #{
200 => <<"Operation success">>,
400 => emqx_dashboard_swagger:error_codes(
['BAD_REQUEST'], <<"Invalid cursor">>
),
503 => emqx_dashboard_swagger:error_codes(
['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE')
)
}
}
};
schema("/file_transfer/files/:clientid/:fileid") ->
#{
'operationId' => '/file_transfer/files/:clientid/:fileid',
get => #{
tags => [<<"file_transfer">>],
summary => <<"List files uploaded in a specific transfer">>,
description => ?DESC("file_list_transfer"),
parameters => [
ref(client_id),
ref(file_id)
],
responses => #{
200 => <<"Operation success">>,
404 => emqx_dashboard_swagger:error_codes(
['FILES_NOT_FOUND'], error_desc('FILES_NOT_FOUND')
),
503 => emqx_dashboard_swagger:error_codes(
['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE')
)
}
}
}.
check_ft_enabled(Params, _Meta) ->
case emqx_ft_conf:enabled() of
true ->
{ok, Params};
false ->
{503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)}
end.
'/file_transfer/files'(get, #{
query_string := QueryString
}) ->
try
Limit = limit(QueryString),
Query =
case maps:get(<<"following">>, QueryString, undefined) of
undefined ->
#{limit => Limit};
Cursor ->
#{limit => Limit, following => Cursor}
end,
case emqx_ft_storage:files(Query) of
{ok, Page} ->
{200, format_page(Page)};
{error, _} ->
{503, error_msg('SERVICE_UNAVAILABLE')}
end
catch
error:{badarg, cursor} ->
{400, error_msg('BAD_REQUEST', <<"Invalid cursor">>)}
end.
'/file_transfer/files/:clientid/:fileid'(get, #{
bindings := #{clientid := ClientId, fileid := FileId}
}) ->
Transfer = {ClientId, FileId},
case emqx_ft_storage:files(#{transfer => Transfer}) of
{ok, Page} ->
{200, format_page(Page)};
{error, [{_Node, enoent} | _]} ->
{404, error_msg('FILES_NOT_FOUND')};
{error, _} ->
{503, error_msg('SERVICE_UNAVAILABLE')}
end.
format_page(#{items := Files, cursor := Cursor}) ->
#{
<<"files">> => lists:map(fun format_file_info/1, Files),
<<"cursor">> => Cursor
};
format_page(#{items := Files}) ->
#{
<<"files">> => lists:map(fun format_file_info/1, Files)
}.
error_msg(Code) ->
#{code => Code, message => error_desc(Code)}.
error_msg(Code, Msg) ->
#{code => Code, message => emqx_utils:readable_error_msg(Msg)}.
error_desc('FILES_NOT_FOUND') ->
<<"Files requested for this transfer could not be found">>;
error_desc('SERVICE_UNAVAILABLE') ->
<<"Service unavailable">>.
roots() ->
[].
-spec fields(hocon_schema:name()) -> [hoconsc:field()].
fields(client_id) ->
[
{clientid,
mk(binary(), #{
in => path,
desc => <<"MQTT Client ID">>,
required => true
})}
];
fields(file_id) ->
[
{fileid,
mk(binary(), #{
in => path,
desc => <<"File ID">>,
required => true
})}
];
fields(following) ->
[
{following,
mk(binary(), #{
in => query,
desc => <<"Cursor to start listing files from">>,
required => false
})}
].
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
format_file_info(
Info = #{
name := Name,
size := Size,
uri := URI,
timestamp := Timestamp,
transfer := {ClientId, FileId}
}
) ->
Res = #{
name => format_name(Name),
size => Size,
timestamp => format_timestamp(Timestamp),
clientid => ClientId,
fileid => FileId,
uri => iolist_to_binary(URI)
},
case Info of
#{meta := Meta} ->
Res#{metadata => emqx_ft:encode_filemeta(Meta)};
#{} ->
Res
end.
format_timestamp(Timestamp) ->
iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
format_name(NameBin) when is_binary(NameBin) ->
NameBin;
format_name(Name) when is_list(Name) ->
iolist_to_binary(Name).
limit(QueryString) ->
maps:get(<<"limit">>, QueryString, emqx_mgmt:default_row_limit()).

30
apps/emqx_ft/src/emqx_ft_app.erl Normal file
View File

@ -0,0 +1,30 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_app).
-behaviour(application).
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_ft_sup:start_link(),
ok = emqx_ft_conf:load(),
{ok, Sup}.
stop(_State) ->
ok = emqx_ft_conf:unload(),
ok.

192
apps/emqx_ft/src/emqx_ft_assembler.erl Normal file
View File

@ -0,0 +1,192 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_assembler).
-export([start_link/3]).
-behaviour(gen_statem).
-export([callback_mode/0]).
-export([init/1]).
-export([handle_event/4]).
-export([terminate/3]).
-export([where/1]).
-type stdata() :: #{
storage := emqx_ft_storage_fs:storage(),
transfer := emqx_ft:transfer(),
assembly := emqx_ft_assembly:t(),
export => emqx_ft_storage_exporter:export()
}.
-define(NAME(Transfer), {n, l, {?MODULE, Transfer}}).
-define(REF(Transfer), {via, gproc, ?NAME(Transfer)}).
%%
start_link(Storage, Transfer, Size) ->
gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []).
where(Transfer) ->
gproc:where(?NAME(Transfer)).
%%
-type state() ::
idle
| list_local_fragments
| {list_remote_fragments, [node()]}
| start_assembling
| {assemble, [{node(), emqx_ft_storage_fs:filefrag()}]}
| complete.
-define(internal(C), {next_event, internal, C}).
callback_mode() ->
handle_event_function.
-spec init(_Args) -> {ok, state(), stdata()}.
init({Storage, Transfer, Size}) ->
_ = erlang:process_flag(trap_exit, true),
St = #{
storage => Storage,
transfer => Transfer,
assembly => emqx_ft_assembly:new(Size)
},
{ok, idle, St}.
-spec handle_event(info | internal, _, state(), stdata()) ->
{next_state, state(), stdata(), {next_event, internal, _}}
| {stop, {shutdown, ok | {error, _}}, stdata()}.
handle_event(info, kickoff, idle, St) ->
% NOTE
% Someone's told us to start the work, which usually means that it has set up a monitor.
% We could wait for this message and handle it at the end of the assembling rather than at
% the beginning, however it would make error handling much more messier.
{next_state, list_local_fragments, St, ?internal([])};
handle_event(info, kickoff, _, _St) ->
keep_state_and_data;
handle_event(
internal,
_,
list_local_fragments,
St = #{storage := Storage, transfer := Transfer, assembly := Asm}
) ->
% TODO: what we do with non-transients errors here (e.g. `eacces`)?
{ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment),
NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)),
NSt = St#{assembly := NAsm},
case emqx_ft_assembly:status(NAsm) of
complete ->
{next_state, start_assembling, NSt, ?internal([])};
{incomplete, _} ->
Nodes = mria_mnesia:running_nodes() -- [node()],
{next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])};
% TODO: recovery?
{error, _} = Error ->
{stop, {shutdown, Error}}
end;
handle_event(
internal,
_,
{list_remote_fragments, Nodes},
St = #{transfer := Transfer, assembly := Asm}
) ->
% TODO
% Async would better because we would not need to wait for some lagging nodes if
% the coverage is already complete.
% TODO: portable "storage" ref
Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, Transfer, fragment),
NodeResults = lists:zip(Nodes, Results),
NAsm = emqx_ft_assembly:update(
lists:foldl(
fun
({Node, {ok, {ok, Fragments}}}, Acc) ->
emqx_ft_assembly:append(Acc, Node, Fragments);
({_Node, _Result}, Acc) ->
% TODO: log?
Acc
end,
Asm,
NodeResults
)
),
NSt = St#{assembly := NAsm},
case emqx_ft_assembly:status(NAsm) of
complete ->
{next_state, start_assembling, NSt, ?internal([])};
% TODO: retries / recovery?
{incomplete, _} = Status ->
{stop, {shutdown, {error, Status}}};
{error, _} = Error ->
{stop, {shutdown, Error}}
end;
handle_event(
internal,
_,
start_assembling,
St = #{storage := Storage, transfer := Transfer, assembly := Asm}
) ->
Filemeta = emqx_ft_assembly:filemeta(Asm),
Coverage = emqx_ft_assembly:coverage(Asm),
case emqx_ft_storage_exporter:start_export(Storage, Transfer, Filemeta) of
{ok, Export} ->
{next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])};
{error, _} = Error ->
{stop, {shutdown, Error}}
end;
handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) ->
% TODO
% Currently, race is possible between getting segment info from the remote node and
% this node garbage collecting the segment itself.
% TODO: pipelining
% TODO: better error handling
{ok, Content} = pread(Node, Segment, St),
case emqx_ft_storage_exporter:write(Export, Content) of
{ok, NExport} ->
{next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])};
{error, _} = Error ->
{stop, {shutdown, Error}, maps:remove(export, St)}
end;
handle_event(internal, _, {assemble, []}, St = #{}) ->
{next_state, complete, St, ?internal([])};
handle_event(internal, _, complete, St = #{export := Export}) ->
Result = emqx_ft_storage_exporter:complete(Export),
_ = maybe_garbage_collect(Result, St),
{stop, {shutdown, Result}, maps:remove(export, St)}.
-spec terminate(_Reason, state(), stdata()) -> _.
terminate(_Reason, _StateName, #{export := Export}) ->
emqx_ft_storage_exporter:discard(Export);
terminate(_Reason, _StateName, #{}) ->
ok.
pread(Node, Segment, #{storage := Storage, transfer := Transfer}) when Node =:= node() ->
emqx_ft_storage_fs:pread(Storage, Transfer, Segment, 0, segsize(Segment));
pread(Node, Segment, #{transfer := Transfer}) ->
emqx_ft_storage_fs_proto_v1:pread(Node, Transfer, Segment, 0, segsize(Segment)).
%%
maybe_garbage_collect(ok, #{storage := Storage, transfer := Transfer, assembly := Asm}) ->
Nodes = emqx_ft_assembly:nodes(Asm),
emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes);
maybe_garbage_collect({error, _}, _St) ->
ok.
segsize(#{fragment := {segment, Info}}) ->
maps:get(size, Info).

47
apps/emqx_ft/src/emqx_ft_assembler_sup.erl Normal file
View File

@ -0,0 +1,47 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_assembler_sup).
-export([start_link/0]).
-export([ensure_child/3]).
-behaviour(supervisor).
-export([init/1]).
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
ensure_child(Storage, Transfer, Size) ->
Childspec = #{
id => Transfer,
start => {emqx_ft_assembler, start_link, [Storage, Transfer, Size]},
restart => temporary
},
case supervisor:start_child(?MODULE, Childspec) of
{ok, Pid} ->
{ok, Pid};
{error, {already_started, Pid}} ->
{ok, Pid}
end.
init(_) ->
SupFlags = #{
strategy => one_for_one,
intensity => 10,
period => 1000
},
{ok, {SupFlags, []}}.

416
apps/emqx_ft/src/emqx_ft_assembly.erl Normal file
View File

@ -0,0 +1,416 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_assembly).
-export([new/1]).
-export([append/3]).
-export([update/1]).
-export([status/1]).
-export([filemeta/1]).
-export([nodes/1]).
-export([coverage/1]).
-export([properties/1]).
-export_type([t/0]).
-type filemeta() :: emqx_ft:filemeta().
-type filefrag() :: emqx_ft_storage_fs:filefrag().
-type filefrag(T) :: emqx_ft_storage_fs:filefrag(T).
-type segmentinfo() :: emqx_ft_storage_fs:segmentinfo().
-record(asm, {
status :: status(),
coverage :: coverage() | undefined,
properties :: properties() | undefined,
meta :: #{filemeta() => {node(), filefrag({filemeta, filemeta()})}},
segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}),
size :: emqx_ft:bytes()
}).
-type status() ::
{incomplete, {missing, _}}
| complete
| {error, {inconsistent, _}}.
-type coverage() :: [{node(), filefrag({segment, segmentinfo()})}].
-type properties() :: #{
%% Node where "most" of the segments are located.
dominant => node()
}.
-opaque t() :: #asm{}.
-spec new(emqx_ft:bytes()) -> t().
new(Size) ->
#asm{
status = {incomplete, {missing, filemeta}},
meta = #{},
segs = emqx_wdgraph:new(),
size = Size
}.
-spec append(t(), node(), filefrag() | [filefrag()]) -> t().
append(Asm, Node, Fragments) when is_list(Fragments) ->
lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments);
append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) ->
append_filemeta(Asm, Node, Fragment);
append(Asm, Node, Segment = #{fragment := {segment, _}}) ->
append_segmentinfo(Asm, Node, Segment).
-spec update(t()) -> t().
update(Asm) ->
case status(meta, Asm) of
{complete, _Meta} ->
case status(coverage, Asm) of
{complete, Coverage, Props} ->
Asm#asm{
status = complete,
coverage = Coverage,
properties = Props
};
Status ->
Asm#asm{status = Status}
end;
Status ->
Asm#asm{status = Status}
end.
-spec status(t()) -> status().
status(#asm{status = Status}) ->
Status.
-spec filemeta(t()) -> filemeta().
filemeta(Asm) ->
case status(meta, Asm) of
{complete, Meta} -> Meta;
_Other -> undefined
end.
-spec coverage(t()) -> coverage() | undefined.
coverage(#asm{coverage = Coverage}) ->
Coverage.
-spec nodes(t()) -> [node()].
nodes(#asm{meta = Meta, segs = Segs}) ->
S1 = maps:fold(
fun(_Meta, {Node, _Fragment}, Acc) ->
ordsets:add_element(Node, Acc)
end,
ordsets:new(),
Meta
),
S2 = emqx_wdgraph:fold(
fun(_Offset, {_End, _, {Node, _Fragment}}, Acc) ->
ordsets:add_element(Node, Acc)
end,
ordsets:new(),
Segs
),
ordsets:to_list(ordsets:union(S1, S2)).
properties(#asm{properties = Properties}) ->
Properties.
status(meta, #asm{meta = Meta}) ->
status(meta, maps:to_list(Meta));
status(meta, [{Meta, {_Node, _Frag}}]) ->
{complete, Meta};
status(meta, []) ->
{incomplete, {missing, filemeta}};
status(meta, [_M1, _M2 | _] = Metas) ->
{error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}};
status(coverage, #asm{segs = Segments, size = Size}) ->
case coverage(Segments, Size) of
Coverage when is_list(Coverage) ->
{complete, Coverage, #{
dominant => dominant(Coverage)
}};
Missing = {missing, _} ->
{incomplete, Missing}
end.
append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) ->
Asm#asm{
meta = maps:put(Meta, {Node, Fragment}, Asm#asm.meta)
}.
append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) ->
% NOTE
% Empty segments are valid but meaningless for coverage.
Asm;
append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) ->
Offset = maps:get(offset, Info),
Size = maps:get(size, Info),
End = Offset + Size,
Segs = add_edge(Asm#asm.segs, Offset, End, locality(Node) * Size, {Node, Fragment}),
Asm#asm{
% TODO
% In theory it's possible to have two segments with same offset + size on
% different nodes but with differing content. We'd need a checksum to
% be able to disambiguate them though.
segs = Segs
}.
add_edge(Segs, Offset, End, Weight, Label) ->
% NOTE
% We are expressing coverage problem as a shortest path problem on weighted directed
% graph, where nodes are segments offsets, two nodes are connected with edge if
% there is a segment which "covers" these offsets (i.e. it starts at first node's
% offset and ends at second node's offst) and weights are segments sizes adjusted
% for locality (i.e. weight are always 0 for any local segment).
case emqx_wdgraph:find_edge(Offset, End, Segs) of
{WeightWas, _Label} when WeightWas =< Weight ->
% NOTE
% Discarding any edges with higher weight here. This is fine as long as we
% optimize for locality.
Segs;
_ ->
emqx_wdgraph:insert_edge(Offset, End, Weight, Label, Segs)
end.
coverage(Segs, Size) ->
case emqx_wdgraph:find_shortest_path(0, Size, Segs) of
Path when is_list(Path) ->
Path;
{false, LastOffset} ->
% NOTE
% This is far from being accurate, but needs no hairy specifics in the
% `emqx_wdgraph` interface.
{missing, {segment, LastOffset, Size}}
end.
dominant(Coverage) ->
% TODO: needs improvement, better defined _dominance_, maybe some score
Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage),
maxfreq(Freqs, node()).
frequencies(Fun, List) ->
lists:foldl(
fun(E, Acc) ->
{K, N} = Fun(E),
maps:update_with(K, fun(M) -> M + N end, N, Acc)
end,
#{},
List
).
maxfreq(Freqs, Init) ->
{_, Max} = maps:fold(
fun
(F, N, {M, _MF}) when N > M -> {N, F};
(_F, _N, {M, MF}) -> {M, MF}
end,
{0, Init},
Freqs
),
Max.
locality(Node) when Node =:= node() ->
% NOTE
% This should prioritize locally available segments over those on remote nodes.
0;
locality(_RemoteNode) ->
1.
segsize(#{fragment := {segment, Info}}) ->
maps:get(size, Info).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
incomplete_new_test() ->
?assertEqual(
{incomplete, {missing, filemeta}},
status(update(new(42)))
).
incomplete_test() ->
?assertEqual(
{incomplete, {missing, filemeta}},
status(
update(
append(new(142), node(), [
segment(p1, 0, 42),
segment(p1, 42, 100)
])
)
)
).
consistent_test() ->
Asm1 = append(new(42), n1, [filemeta(m1, "blarg")]),
Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]),
Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]),
?assertMatch({complete, _}, status(meta, Asm3)).
inconsistent_test() ->
Asm1 = append(new(42), node(), [segment(s1, 0, 42)]),
Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]),
Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]),
Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]),
?assertMatch(
{error,
{inconsistent, [
% blarg < blorg
#{node := n3, path := m3, fragment := {filemeta, #{name := "blarg"}}},
#{node := n2, path := m1, fragment := {filemeta, #{name := "blorg"}}}
]}},
status(meta, Asm4)
).
simple_coverage_test() ->
Node = node(),
Segs = [
{node42, segment(n1, 20, 30)},
{Node, segment(n2, 0, 10)},
{Node, segment(n3, 50, 50)},
{Node, segment(n4, 10, 10)}
],
Asm = append_many(new(100), Segs),
?assertMatch(
{complete,
[
{Node, #{path := n2}},
{Node, #{path := n4}},
{node42, #{path := n1}},
{Node, #{path := n3}}
],
#{dominant := Node}},
status(coverage, Asm)
).
redundant_coverage_test() ->
Node = node(),
Segs = [
{Node, segment(n1, 0, 20)},
{node1, segment(n2, 0, 10)},
{Node, segment(n3, 20, 40)},
{node2, segment(n4, 10, 10)},
{node2, segment(n5, 50, 20)},
{node3, segment(n6, 20, 20)},
{Node, segment(n7, 50, 10)},
{node1, segment(n8, 40, 10)}
],
Asm = append_many(new(70), Segs),
?assertMatch(
{complete,
[
{Node, #{path := n1}},
{node3, #{path := n6}},
{node1, #{path := n8}},
{node2, #{path := n5}}
],
#{dominant := _}},
status(coverage, Asm)
).
redundant_coverage_prefer_local_test() ->
Node = node(),
Segs = [
{node1, segment(n1, 0, 20)},
{Node, segment(n2, 0, 10)},
{Node, segment(n3, 10, 10)},
{node2, segment(n4, 20, 20)},
{Node, segment(n5, 30, 10)},
{Node, segment(n6, 20, 10)}
],
Asm = append_many(new(40), Segs),
?assertMatch(
{complete,
[
{Node, #{path := n2}},
{Node, #{path := n3}},
{Node, #{path := n6}},
{Node, #{path := n5}}
],
#{dominant := Node}},
status(coverage, Asm)
).
missing_coverage_test() ->
Node = node(),
Segs = [
{Node, segment(n1, 0, 10)},
{node1, segment(n3, 10, 20)},
{Node, segment(n2, 0, 20)},
{node2, segment(n4, 50, 50)},
{Node, segment(n5, 40, 60)}
],
Asm = append_many(new(100), Segs),
?assertEqual(
% {incomplete, {missing, {segment, 30, 40}}} would be more accurate
{incomplete, {missing, {segment, 30, 100}}},
status(coverage, Asm)
).
missing_end_coverage_test() ->
Node = node(),
Segs = [
{Node, segment(n1, 0, 15)},
{node1, segment(n3, 10, 10)}
],
Asm = append_many(new(20), Segs),
?assertEqual(
{incomplete, {missing, {segment, 15, 20}}},
status(coverage, Asm)
).
missing_coverage_with_redudancy_test() ->
Segs = [
{node(), segment(n1, 0, 10)},
{node(), segment(n2, 0, 20)},
{node42, segment(n3, 10, 20)},
{node43, segment(n4, 10, 50)},
{node(), segment(n5, 40, 60)}
],
Asm = append_many(new(100), Segs),
?assertEqual(
% {incomplete, {missing, {segment, 50, 60}}}, ???
{incomplete, {missing, {segment, 60, 100}}},
status(coverage, Asm)
).
append_many(Asm, List) ->
lists:foldl(
fun({Node, Frag}, Acc) -> append(Acc, Node, Frag) end,
Asm,
List
).
filemeta(Path, Name) ->
#{
path => Path,
fragment =>
{filemeta, #{
name => Name
}}
}.
segment(Path, Offset, Size) ->
#{
path => Path,
fragment =>
{segment, #{
offset => Offset,
size => Size
}}
}.
-endif.

143
apps/emqx_ft/src/emqx_ft_conf.erl Normal file
View File

@ -0,0 +1,143 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc File Transfer configuration management module
-module(emqx_ft_conf).
-behaviour(emqx_config_handler).
-include_lib("emqx/include/logger.hrl").
%% Accessors
-export([enabled/0]).
-export([storage/0]).
-export([gc_interval/1]).
-export([segments_ttl/1]).
-export([init_timeout/0]).
-export([store_segment_timeout/0]).
-export([assemble_timeout/0]).
%% Load/Unload
-export([
load/0,
unload/0
]).
%% callbacks for emqx_config_handler
-export([
pre_config_update/3,
post_config_update/5
]).
-type milliseconds() :: non_neg_integer().
-type seconds() :: non_neg_integer().
%%--------------------------------------------------------------------
%% Accessors
%%--------------------------------------------------------------------
-spec enabled() -> boolean().
enabled() ->
emqx_config:get([file_transfer, enable], false).
-spec storage() -> emqx_config:config().
storage() ->
emqx_config:get([file_transfer, storage]).
-spec gc_interval(emqx_ft_storage_fs:storage()) ->
emqx_maybe:t(milliseconds()).
gc_interval(Storage) ->
emqx_utils_maps:deep_get([segments, gc, interval], Storage, undefined).
-spec segments_ttl(emqx_ft_storage_fs:storage()) ->
emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}).
segments_ttl(Storage) ->
Min = emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Storage, undefined),
Max = emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Storage, undefined),
case is_integer(Min) andalso is_integer(Max) of
true ->
{Min, Max};
false ->
undefined
end.
init_timeout() ->
emqx_config:get([file_transfer, init_timeout]).
assemble_timeout() ->
emqx_config:get([file_transfer, assemble_timeout]).
store_segment_timeout() ->
emqx_config:get([file_transfer, store_segment_timeout]).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
-spec load() -> ok.
load() ->
ok = maybe_start(),
emqx_conf:add_handler([file_transfer], ?MODULE).
-spec unload() -> ok.
unload() ->
ok = stop(),
emqx_conf:remove_handler([file_transfer]).
%%--------------------------------------------------------------------
%% emqx_config_handler callbacks
%%--------------------------------------------------------------------
-spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) ->
{ok, emqx_config:update_request()} | {error, term()}.
pre_config_update(_, Req, _Config) ->
{ok, Req}.
-spec post_config_update(
list(atom()),
emqx_config:update_request(),
emqx_config:config(),
emqx_config:config(),
emqx_config:app_envs()
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) ->
on_config_update(OldConfig, NewConfig).
on_config_update(#{enable := false}, #{enable := false}) ->
ok;
on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) ->
ok = emqx_ft_storage:on_config_update(OldStorage, undefined),
ok = emqx_ft:unhook();
on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) ->
ok = emqx_ft_storage:on_config_update(undefined, NewStorage),
ok = emqx_ft:hook();
on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) ->
ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage).
maybe_start() ->
case emqx_config:get([file_transfer]) of
#{enable := true, storage := Storage} ->
ok = emqx_ft_storage:on_config_update(undefined, Storage),
ok = emqx_ft:hook();
_ ->
ok
end.
stop() ->
ok = emqx_ft:unhook(),
ok = emqx_ft_storage:on_config_update(storage(), undefined).

235
apps/emqx_ft/src/emqx_ft_fs_iterator.erl Normal file
View File

@ -0,0 +1,235 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_fs_iterator).
-export([new/2]).
-export([next/1]).
-export([next_leaf/1]).
-export([seek/3]).
-export([fold/3]).
-export([fold_n/4]).
-export_type([t/0]).
-export_type([glob/0]).
-export_type([pathstack/0]).
-type root() :: file:name().
-type glob() :: ['*' | globfun()].
-type globfun() ::
fun((_Filename :: file:name()) -> boolean())
| fun((_Filename :: file:name(), pathstack()) -> boolean()).
% A path stack is a list of path components, in reverse order.
-type pathstack() :: [file:name(), ...].
-opaque t() :: #{
root := root(),
queue := [_PathStack :: [file:name()]],
head := glob(),
stack := [{[pathstack()], glob()}]
}.
-type entry() :: entry_leaf() | entry_node().
-type entry_leaf() ::
{leaf, file:name(), file:file_info() | {error, file:posix()}, pathstack()}.
-type entry_node() ::
{node, file:name(), {error, file:posix()}, pathstack()}.
-spec new(root(), glob()) ->
t().
new(Root, Glob) ->
#{
root => Root,
queue => [[]],
head => Glob,
stack => []
}.
-spec next(t()) ->
{entry(), t()} | none.
next(It = #{queue := [PathStack | Rest], head := []}) ->
{emit(PathStack, It), It#{queue => Rest}};
next(It = #{queue := [PathStack | Rest], head := [Pat | _], root := Root}) ->
Filepath = mk_filepath(PathStack),
case emqx_ft_fs_util:list_dir(filename:join(Root, Filepath)) of
{ok, Filenames} ->
Sorted = lists:sort(Filenames),
Matches = [[Fn | PathStack] || Fn <- Sorted, matches_glob(Pat, Fn, [Fn | PathStack])],
ItNext = windup(It),
next(ItNext#{queue => Matches});
{error, _} = Error ->
{{node, Filepath, Error, PathStack}, It#{queue => Rest}}
end;
next(It = #{queue := []}) ->
unwind(It).
windup(It = #{queue := [_ | Rest], head := [Pat | Glob], stack := Stack}) ->
% NOTE
% Preserve unfinished paths and glob in the stack, so that we can resume traversal
% when the lower levels of the tree are exhausted.
It#{
head => Glob,
stack => [{Rest, [Pat | Glob]} | Stack]
}.
unwind(It = #{stack := [{Queue, Glob} | StackRest]}) ->
% NOTE
% Resume traversal of unfinished paths from the upper levels of the tree.
next(It#{
queue => Queue,
head => Glob,
stack => StackRest
});
unwind(#{stack := []}) ->
none.
emit(PathStack, #{root := Root}) ->
Filepath = mk_filepath(PathStack),
case emqx_ft_fs_util:read_info(filename:join(Root, Filepath)) of
{ok, Fileinfo} ->
{leaf, Filepath, Fileinfo, PathStack};
{error, _} = Error ->
{leaf, Filepath, Error, PathStack}
end.
mk_filepath([]) ->
"";
mk_filepath(PathStack) ->
filename:join(lists:reverse(PathStack)).
matches_glob('*', _, _) ->
true;
matches_glob(FilterFun, Filename, _PathStack) when is_function(FilterFun, 1) ->
FilterFun(Filename);
matches_glob(FilterFun, Filename, PathStack) when is_function(FilterFun, 2) ->
FilterFun(Filename, PathStack).
%%
-spec next_leaf(t()) ->
{entry_leaf(), t()} | none.
next_leaf(It) ->
case next(It) of
{{leaf, _, _, _} = Leaf, ItNext} ->
{Leaf, ItNext};
{{node, _Filename, _Error, _PathStack}, ItNext} ->
% NOTE
% Intentionally skipping intermediate traversal errors here, for simplicity.
next_leaf(ItNext);
none ->
none
end.
%%
-spec seek([file:name()], root(), glob()) ->
t().
seek(PathSeek, Root, Glob) ->
SeekGlob = mk_seek_glob(PathSeek, Glob),
SeekStack = lists:reverse(PathSeek),
case next_leaf(new(Root, SeekGlob)) of
{{leaf, _Filepath, _Info, SeekStack}, It} ->
fixup_glob(Glob, It);
{{leaf, _Filepath, _Info, Successor}, It = #{queue := Queue}} ->
fixup_glob(Glob, It#{queue => [Successor | Queue]});
none ->
none(Root)
end.
mk_seek_glob(PathSeek, Glob) ->
% NOTE
% The seek glob is a glob that skips all the nodes / leaves that are lexicographically
% smaller than the seek path. For example, if the seek path is ["a", "b", "c"], and
% the glob is ['*', '*', '*', '*'], then the seek glob is:
% [ fun(Path) -> Path >= ["a"] end,
% fun(Path) -> Path >= ["a", "b"] end,
% fun(Path) -> Path >= ["a", "b", "c"] end,
% '*'
% ]
L = min(length(PathSeek), length(Glob)),
merge_glob([mk_seek_pat(lists:sublist(PathSeek, N)) || N <- lists:seq(1, L)], Glob).
mk_seek_pat(PathSeek) ->
% NOTE
% The `PathStack` and `PathSeek` are of the same length here.
fun(_Filename, PathStack) -> lists:reverse(PathStack) >= PathSeek end.
merge_glob([Pat | SeekRest], [PatOrig | Rest]) ->
[merge_pat(Pat, PatOrig) | merge_glob(SeekRest, Rest)];
merge_glob([], [PatOrig | Rest]) ->
[PatOrig | merge_glob([], Rest)];
merge_glob([], []) ->
[].
merge_pat(Pat, PatOrig) ->
fun(Filename, PathStack) ->
Pat(Filename, PathStack) andalso matches_glob(PatOrig, Filename, PathStack)
end.
fixup_glob(Glob, It = #{head := [], stack := Stack}) ->
% NOTE
% Restoring original glob through the stack. Strictly speaking, this is not usually
% necessary, it's a kind of optimization.
fixup_glob(Glob, lists:reverse(Stack), It#{stack => []}).
fixup_glob(Glob = [_ | Rest], [{Queue, _} | StackRest], It = #{stack := Stack}) ->
fixup_glob(Rest, StackRest, It#{stack => [{Queue, Glob} | Stack]});
fixup_glob(Rest, [], It) ->
It#{head => Rest}.
%%
-spec fold(fun((entry(), Acc) -> Acc), Acc, t()) ->
Acc.
fold(FoldFun, Acc, It) ->
case next(It) of
{Entry, ItNext} ->
fold(FoldFun, FoldFun(Entry, Acc), ItNext);
none ->
Acc
end.
%% NOTE
%% Passing negative `N` is allowed, in which case the iterator will be exhausted
%% completely, like in `fold/3`.
-spec fold_n(fun((entry(), Acc) -> Acc), Acc, t(), _N :: integer()) ->
{Acc, {more, t()} | none}.
fold_n(_FoldFun, Acc, It, 0) ->
{Acc, {more, It}};
fold_n(FoldFun, Acc, It, N) ->
case next(It) of
{Entry, ItNext} ->
fold_n(FoldFun, FoldFun(Entry, Acc), ItNext, N - 1);
none ->
{Acc, none}
end.
%%
-spec none(root()) ->
t().
none(Root) ->
% NOTE
% The _none_ iterator is a valid iterator, but it will never yield any entries.
#{
root => Root,
queue => [],
head => [],
stack => []
}.

180
apps/emqx_ft/src/emqx_ft_fs_util.erl Normal file
View File

@ -0,0 +1,180 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_fs_util).
-include_lib("snabbkaffe/include/trace.hrl").
-include_lib("kernel/include/file.hrl").
-export([is_filename_safe/1]).
-export([escape_filename/1]).
-export([unescape_filename/1]).
-export([read_decode_file/2]).
-export([read_info/1]).
-export([list_dir/1]).
-export([fold/4]).
-type foldfun(Acc) ::
fun(
(
_Filepath :: file:name(),
_Info :: file:file_info() | {error, file:posix()},
_Stack :: emqx_ft_fs_iterator:pathstack(),
Acc
) -> Acc
).
-define(IS_UNSAFE(C),
((C) =:= $% orelse
(C) =:= $: orelse
(C) =:= $\\ orelse
(C) =:= $/)
).
-define(IS_PRINTABLE(C),
% NOTE: See `io_lib:printable_unicode_list/1`
(((C) >= 32 andalso (C) =< 126) orelse
((C) >= 16#A0 andalso (C) < 16#D800) orelse
((C) > 16#DFFF andalso (C) < 16#FFFE) orelse
((C) > 16#FFFF andalso (C) =< 16#10FFFF))
).
%%
-spec is_filename_safe(file:filename_all()) -> ok | {error, atom()}.
is_filename_safe(FN) when is_binary(FN) ->
is_filename_safe(unicode:characters_to_list(FN));
is_filename_safe("") ->
{error, empty};
is_filename_safe(FN) when FN == "." orelse FN == ".." ->
{error, special};
is_filename_safe(FN) ->
verify_filename_safe(FN).
verify_filename_safe([$% | Rest]) ->
verify_filename_safe(Rest);
verify_filename_safe([C | _]) when ?IS_UNSAFE(C) ->
{error, unsafe};
verify_filename_safe([C | _]) when not ?IS_PRINTABLE(C) ->
{error, nonprintable};
verify_filename_safe([_ | Rest]) ->
verify_filename_safe(Rest);
verify_filename_safe([]) ->
ok.
-spec escape_filename(binary()) -> file:name().
escape_filename(Name) when Name == <<".">> orelse Name == <<"..">> ->
lists:reverse(percent_encode(Name, ""));
escape_filename(Name) ->
escape(Name, "").
escape(<<C/utf8, Rest/binary>>, Acc) when ?IS_UNSAFE(C) ->
escape(Rest, percent_encode(<<C/utf8>>, Acc));
escape(<<C/utf8, Rest/binary>>, Acc) when not ?IS_PRINTABLE(C) ->
escape(Rest, percent_encode(<<C/utf8>>, Acc));
escape(<<C/utf8, Rest/binary>>, Acc) ->
escape(Rest, [C | Acc]);
escape(<<>>, Acc) ->
lists:reverse(Acc).
-spec unescape_filename(file:name()) -> binary().
unescape_filename(Name) ->
unescape(Name, <<>>).
unescape([$%, A, B | Rest], Acc) ->
unescape(Rest, percent_decode(A, B, Acc));
unescape([C | Rest], Acc) ->
unescape(Rest, <<Acc/binary, C/utf8>>);
unescape([], Acc) ->
Acc.
percent_encode(<<A:4, B:4, Rest/binary>>, Acc) ->
percent_encode(Rest, [dec2hex(B), dec2hex(A), $% | Acc]);
percent_encode(<<>>, Acc) ->
Acc.
percent_decode(A, B, Acc) ->
<<Acc/binary, (hex2dec(A) * 16 + hex2dec(B))>>.
dec2hex(X) when (X >= 0) andalso (X =< 9) -> X + $0;
dec2hex(X) when (X >= 10) andalso (X =< 15) -> X + $A - 10.
hex2dec(X) when (X >= $0) andalso (X =< $9) -> X - $0;
hex2dec(X) when (X >= $A) andalso (X =< $F) -> X - $A + 10;
hex2dec(X) when (X >= $a) andalso (X =< $f) -> X - $a + 10;
hex2dec(_) -> error(badarg).
%%
-spec read_decode_file(file:name(), fun((binary()) -> Value)) ->
{ok, Value} | {error, _IoError}.
read_decode_file(Filepath, DecodeFun) ->
case file:read_file(Filepath) of
{ok, Content} ->
safe_decode(Content, DecodeFun);
{error, _} = Error ->
Error
end.
safe_decode(Content, DecodeFun) ->
try
{ok, DecodeFun(Content)}
catch
C:E:Stacktrace ->
?tp(warning, "safe_decode_failed", #{
class => C,
exception => E,
stacktrace => Stacktrace
}),
{error, corrupted}
end.
-spec read_info(file:name_all()) ->
{ok, file:file_info()} | {error, file:posix() | badarg}.
read_info(AbsPath) ->
% NOTE
% Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`.
file:read_link_info(AbsPath, [{time, posix}, raw]).
-spec list_dir(file:name_all()) ->
{ok, [file:name()]} | {error, file:posix() | badarg}.
list_dir(AbsPath) ->
case ?MODULE:read_info(AbsPath) of
{ok, #file_info{type = directory}} ->
file:list_dir(AbsPath);
{ok, #file_info{}} ->
{error, enotdir};
{error, Reason} ->
{error, Reason}
end.
-spec fold(foldfun(Acc), Acc, _Root :: file:name(), emqx_ft_fs_iterator:glob()) ->
Acc.
fold(FoldFun, Acc, Root, Glob) ->
fold(FoldFun, Acc, emqx_ft_fs_iterator:new(Root, Glob)).
fold(FoldFun, Acc, It) ->
case emqx_ft_fs_iterator:next(It) of
{{node, _Path, {error, enotdir}, _PathStack}, ItNext} ->
fold(FoldFun, Acc, ItNext);
{{_Type, Path, Info, PathStack}, ItNext} ->
AccNext = FoldFun(Path, Info, PathStack, Acc),
fold(FoldFun, AccNext, ItNext);
none ->
Acc
end.

116
apps/emqx_ft/src/emqx_ft_responder.erl Normal file
View File

@ -0,0 +1,116 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_responder).
-behaviour(gen_server).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% API
-export([start/3]).
-export([kickoff/2]).
-export([ack/2]).
%% Supervisor API
-export([start_link/3]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
-define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}).
-type key() :: term().
-type respfun() :: fun(({ack, _Result} | {down, _Result} | timeout) -> _SideEffect).
%%--------------------------------------------------------------------
%% API
%% -------------------------------------------------------------------
-spec start(key(), respfun(), timeout()) -> startlink_ret().
start(Key, RespFun, Timeout) ->
emqx_ft_responder_sup:start_child(Key, RespFun, Timeout).
-spec kickoff(key(), pid()) -> ok.
kickoff(Key, Pid) ->
gen_server:call(?REF(Key), {kickoff, Pid}).
-spec ack(key(), _Result) -> _Return.
ack(Key, Result) ->
% TODO: it's possible to avoid term copy
gen_server:call(?REF(Key), {ack, Result}, infinity).
-spec start_link(key(), timeout(), respfun()) -> startlink_ret().
start_link(Key, RespFun, Timeout) ->
gen_server:start_link(?REF(Key), ?MODULE, {Key, RespFun, Timeout}, []).
%%--------------------------------------------------------------------
%% gen_server callbacks
%% -------------------------------------------------------------------
init({Key, RespFun, Timeout}) ->
_ = erlang:process_flag(trap_exit, true),
_TRef = erlang:send_after(Timeout, self(), timeout),
{ok, {Key, RespFun}}.
handle_call({kickoff, Pid}, _From, St) ->
% TODO: more state?
_MRef = erlang:monitor(process, Pid),
_ = Pid ! kickoff,
{reply, ok, St};
handle_call({ack, Result}, _From, {Key, RespFun}) ->
Ret = apply(RespFun, [{ack, Result}]),
?tp(debug, ft_responder_ack, #{key => Key, result => Result, return => Ret}),
{stop, {shutdown, Ret}, Ret, undefined};
handle_call(Msg, _From, State) ->
?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}),
{reply, {error, unknown_call}, State}.
handle_cast(Msg, State) ->
?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}),
{noreply, State}.
handle_info(timeout, {Key, RespFun}) ->
Ret = apply(RespFun, [timeout]),
?tp(debug, ft_responder_timeout, #{key => Key, return => Ret}),
{stop, {shutdown, Ret}, undefined};
handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) ->
Ret = apply(RespFun, [{down, map_down_reason(Reason)}]),
?tp(debug, ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}),
{stop, {shutdown, Ret}, undefined};
handle_info(Msg, State) ->
?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}),
{noreply, State}.
terminate(_Reason, undefined) ->
ok;
terminate(Reason, {Key, RespFun}) ->
Ret = apply(RespFun, [timeout]),
?tp(debug, ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}),
ok.
map_down_reason(normal) ->
ok;
map_down_reason(shutdown) ->
ok;
map_down_reason({shutdown, Result}) ->
Result;
map_down_reason(noproc) ->
{error, noproc};
map_down_reason(Error) ->
{error, {internal_error, Error}}.

48
apps/emqx_ft/src/emqx_ft_responder_sup.erl Normal file
View File

@ -0,0 +1,48 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_responder_sup).
-export([start_link/0]).
-export([start_child/3]).
-behaviour(supervisor).
-export([init/1]).
-define(SUPERVISOR, ?MODULE).
%%
-spec start_link() -> {ok, pid()}.
start_link() ->
supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []).
start_child(Key, RespFun, Timeout) ->
supervisor:start_child(?SUPERVISOR, [Key, RespFun, Timeout]).
-spec init(_) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
init(_) ->
Flags = #{
strategy => simple_one_for_one,
intensity => 100,
period => 100
},
ChildSpec = #{
id => responder,
start => {emqx_ft_responder, start_link, []},
restart => temporary
},
{ok, {Flags, [ChildSpec]}}.

317
apps/emqx_ft/src/emqx_ft_schema.erl Normal file
View File

@ -0,0 +1,317 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_schema).
-behaviour(hocon_schema).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-export([namespace/0, roots/0, fields/1, tags/0, desc/1]).
-export([schema/1]).
-export([translate/1]).
-type json_value() ::
null
| boolean()
| binary()
| number()
| [json_value()]
| #{binary() => json_value()}.
-reflect_type([json_value/0]).
%% NOTE
%% This is rather conservative limit, mostly dictated by the filename limitations
%% on most filesystems. Even though, say, S3 does not have such limitations, it's
%% still useful to have a limit on the filename length, to avoid having to deal with
%% limits in the storage backends.
-define(MAX_FILENAME_BYTELEN, 255).
-import(hoconsc, [ref/2, mk/2]).
namespace() -> file_transfer.
tags() ->
[<<"File Transfer">>].
roots() -> [file_transfer].
fields(file_transfer) ->
[
{enable,
mk(
boolean(),
#{
desc => ?DESC("enable"),
required => false,
default => false
}
)},
{init_timeout,
mk(
emqx_schema:duration_ms(),
#{
desc => ?DESC("init_timeout"),
required => false,
default => "10s"
}
)},
{store_segment_timeout,
mk(
emqx_schema:duration_ms(),
#{
desc => ?DESC("store_segment_timeout"),
required => false,
default => "5m"
}
)},
{assemble_timeout,
mk(
emqx_schema:duration_ms(),
#{
desc => ?DESC("assemble_timeout"),
required => false,
default => "5m"
}
)},
{storage,
mk(
ref(storage_backend),
#{
desc => ?DESC("storage_backend"),
required => false,
validator => validator(backend),
default => #{
<<"local">> => #{}
}
}
)}
];
fields(storage_backend) ->
[
{local,
mk(
ref(local_storage),
#{
desc => ?DESC("local_storage"),
required => {false, recursively}
}
)}
];
fields(local_storage) ->
[
{segments,
mk(
ref(local_storage_segments),
#{
desc => ?DESC("local_storage_segments"),
required => false,
default => #{
<<"gc">> => #{}
}
}
)},
{exporter,
mk(
ref(local_storage_exporter_backend),
#{
desc => ?DESC("local_storage_exporter_backend"),
required => false,
validator => validator(backend),
default => #{
<<"local">> => #{}
}
}
)}
];
fields(local_storage_segments) ->
[
{root,
mk(
binary(),
#{
desc => ?DESC("local_storage_segments_root"),
required => false
}
)},
{gc,
mk(
ref(local_storage_segments_gc), #{
desc => ?DESC("local_storage_segments_gc"),
required => false
}
)}
];
fields(local_storage_exporter_backend) ->
[
{local,
mk(
ref(local_storage_exporter),
#{
desc => ?DESC("local_storage_exporter"),
required => {false, recursively}
}
)},
{s3,
mk(
ref(s3_exporter),
#{
desc => ?DESC("s3_exporter"),
required => {false, recursively}
}
)}
];
fields(local_storage_exporter) ->
[
{root,
mk(
binary(),
#{
desc => ?DESC("local_storage_exporter_root"),
required => false
}
)}
];
fields(s3_exporter) ->
emqx_s3_schema:fields(s3);
fields(local_storage_segments_gc) ->
[
{interval,
mk(
emqx_schema:duration_ms(),
#{
desc => ?DESC("storage_gc_interval"),
required => false,
default => "1h"
}
)},
{maximum_segments_ttl,
mk(
emqx_schema:duration_s(),
#{
desc => ?DESC("storage_gc_max_segments_ttl"),
required => false,
default => "24h"
}
)},
{minimum_segments_ttl,
mk(
emqx_schema:duration_s(),
#{
desc => ?DESC("storage_gc_min_segments_ttl"),
required => false,
default => "5m",
% NOTE
% This setting does not seem to be useful to an end-user.
hidden => true
}
)}
].
desc(file_transfer) ->
"File transfer settings";
desc(local_storage) ->
"File transfer local storage settings";
desc(local_storage_segments) ->
"File transfer local segments storage settings";
desc(local_storage_exporter) ->
"Local Exporter settings for the File transfer local storage backend";
desc(s3_exporter) ->
"S3 Exporter settings for the File transfer local storage backend";
desc(local_storage_segments_gc) ->
"Garbage collection settings for the File transfer local segments storage";
desc(local_storage_exporter_backend) ->
"Exporter for the local file system storage backend";
desc(storage_backend) ->
"Storage backend settings for file transfer";
desc(_) ->
undefined.
schema(filemeta) ->
#{
roots => [
{name,
hoconsc:mk(string(), #{
required => true,
validator => validator(filename),
converter => converter(unicode_string)
})},
{size, hoconsc:mk(non_neg_integer())},
{expire_at, hoconsc:mk(non_neg_integer())},
{checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})},
{segments_ttl, hoconsc:mk(pos_integer())},
{user_data, hoconsc:mk(json_value())}
]
}.
validator(filename) ->
[
fun(Value) ->
Bin = unicode:characters_to_binary(Value),
byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded}
end,
fun emqx_ft_fs_util:is_filename_safe/1
];
validator(backend) ->
fun(Config) ->
case maps:keys(Config) of
[_Type] ->
ok;
_Conflicts = [_ | _] ->
{error, multiple_conflicting_backends}
end
end.
converter(checksum) ->
fun
(undefined, #{}) ->
undefined;
({sha256, Bin}, #{make_serializable := true}) ->
_ = is_binary(Bin) orelse throw({expected_type, string}),
_ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}),
binary:encode_hex(Bin);
(Hex, #{}) ->
_ = is_binary(Hex) orelse throw({expected_type, string}),
_ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}),
{sha256, binary:decode_hex(Hex)}
end;
converter(unicode_string) ->
fun
(undefined, #{}) ->
undefined;
(Str, #{make_serializable := true}) ->
_ = is_list(Str) orelse throw({expected_type, string}),
unicode:characters_to_binary(Str);
(Str, #{}) ->
_ = is_binary(Str) orelse throw({expected_type, string}),
unicode:characters_to_list(Str)
end.
ref(Ref) ->
ref(?MODULE, Ref).
translate(Conf) ->
[Root] = roots(),
maps:get(
Root,
hocon_tconf:check_plain(
?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root]
)
).

195
apps/emqx_ft/src/emqx_ft_storage.erl Normal file
View File

@ -0,0 +1,195 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage).
-export(
[
store_filemeta/2,
store_segment/2,
assemble/2,
files/0,
files/1,
with_storage_type/2,
with_storage_type/3,
backend/0,
on_config_update/2
]
).
-type type() :: local.
-type backend() :: {type(), storage()}.
-type storage() :: config().
-type config() :: emqx_config:config().
-export_type([backend/0]).
-export_type([assemble_callback/0]).
-export_type([query/1]).
-export_type([page/2]).
-export_type([file_info/0]).
-export_type([export_data/0]).
-export_type([reader/0]).
-type assemble_callback() :: fun((ok | {error, term()}) -> any()).
-type query(Cursor) ::
#{transfer => emqx_ft:transfer()}
| #{
limit => non_neg_integer(),
following => Cursor
}.
-type page(Item, Cursor) :: #{
items := [Item],
cursor => Cursor
}.
-type file_info() :: #{
transfer := emqx_ft:transfer(),
name := file:name(),
size := _Bytes :: non_neg_integer(),
timestamp := emqx_datetime:epoch_second(),
uri => uri_string:uri_string(),
meta => emqx_ft:filemeta()
}.
-type export_data() :: binary() | qlc:query_handle().
-type reader() :: pid().
%%--------------------------------------------------------------------
%% Behaviour
%%--------------------------------------------------------------------
%% NOTE
%% An async task will wait for a `kickoff` message to start processing, to give some time
%% to set up monitors, etc. Async task will not explicitly report the processing result,
%% you are expected to receive and handle exit reason of the process, which is
%% -type result() :: `{shutdown, ok | {error, _}}`.
-callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) ->
ok | {async, pid()} | {error, term()}.
-callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) ->
ok | {async, pid()} | {error, term()}.
-callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) ->
ok | {async, pid()} | {error, term()}.
-callback files(storage(), query(Cursor)) ->
{ok, page(file_info(), Cursor)} | {error, term()}.
-callback start(emqx_config:config()) -> any().
-callback stop(emqx_config:config()) -> any().
-callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) ->
any().
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
-spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) ->
ok | {async, pid()} | {error, term()}.
store_filemeta(Transfer, FileMeta) ->
dispatch(store_filemeta, [Transfer, FileMeta]).
-spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) ->
ok | {async, pid()} | {error, term()}.
store_segment(Transfer, Segment) ->
dispatch(store_segment, [Transfer, Segment]).
-spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) ->
ok | {async, pid()} | {error, term()}.
assemble(Transfer, Size) ->
dispatch(assemble, [Transfer, Size]).
-spec files() ->
{ok, page(file_info(), _)} | {error, term()}.
files() ->
files(#{}).
-spec files(query(Cursor)) ->
{ok, page(file_info(), Cursor)} | {error, term()}.
files(Query) ->
dispatch(files, [Query]).
-spec dispatch(atom(), list(term())) -> any().
dispatch(Fun, Args) when is_atom(Fun) ->
{Type, Storage} = backend(),
apply(mod(Type), Fun, [Storage | Args]).
%%
-spec with_storage_type(atom(), atom() | function()) -> any().
with_storage_type(Type, Fun) ->
with_storage_type(Type, Fun, []).
-spec with_storage_type(atom(), atom() | function(), list(term())) -> any().
with_storage_type(Type, Fun, Args) ->
case backend() of
{Type, Storage} when is_atom(Fun) ->
apply(mod(Type), Fun, [Storage | Args]);
{Type, Storage} when is_function(Fun) ->
apply(Fun, [Storage | Args]);
{_, _} = Backend ->
{error, {invalid_storage_backend, Backend}}
end.
%%
-spec backend() -> backend().
backend() ->
backend(emqx_ft_conf:storage()).
-spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) ->
ok.
on_config_update(ConfigOld, ConfigNew) ->
on_backend_update(
emqx_maybe:apply(fun backend/1, ConfigOld),
emqx_maybe:apply(fun backend/1, ConfigNew)
).
on_backend_update({Type, _} = Backend, {Type, _} = Backend) ->
ok;
on_backend_update({Type, StorageOld}, {Type, StorageNew}) ->
ok = (mod(Type)):on_config_update(StorageOld, StorageNew);
on_backend_update(BackendOld, BackendNew) when
(BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso
(BackendNew =:= undefined orelse is_tuple(BackendNew))
->
_ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld),
_ = emqx_maybe:apply(fun on_storage_start/1, BackendNew),
ok.
%%--------------------------------------------------------------------
%% Local API
%%--------------------------------------------------------------------
-spec backend(config()) -> backend().
backend(#{local := Storage}) ->
{local, Storage}.
on_storage_start({Type, Storage}) ->
(mod(Type)):start(Storage).
on_storage_stop({Type, Storage}) ->
(mod(Type)):stop(Storage).
mod(local) ->
emqx_ft_storage_fs.

195
apps/emqx_ft/src/emqx_ft_storage_exporter.erl Normal file
View File

@ -0,0 +1,195 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Filesystem storage exporter
%%
%% This is conceptually a part of the Filesystem storage backend that defines
%% how and where complete transfers are assembled into files and stored.
-module(emqx_ft_storage_exporter).
%% Export API
-export([start_export/3]).
-export([write/2]).
-export([complete/1]).
-export([discard/1]).
%% Listing API
-export([list/2]).
%% Lifecycle API
-export([on_config_update/2]).
%% Internal API
-export([exporter/1]).
-export_type([export/0]).
-type storage() :: emxt_ft_storage_fs:storage().
-type transfer() :: emqx_ft:transfer().
-type filemeta() :: emqx_ft:filemeta().
-type checksum() :: emqx_ft:checksum().
-type exporter_conf() :: map().
-type export_st() :: term().
-type hash_state() :: term().
-opaque export() :: #{
mod := module(),
st := export_st(),
hash := hash_state(),
filemeta := filemeta()
}.
%%------------------------------------------------------------------------------
%% Behaviour
%%------------------------------------------------------------------------------
-callback start_export(exporter_conf(), transfer(), filemeta()) ->
{ok, export_st()} | {error, _Reason}.
%% Exprter must discard the export itself in case of error
-callback write(ExportSt :: export_st(), iodata()) ->
{ok, ExportSt :: export_st()} | {error, _Reason}.
-callback complete(_ExportSt :: export_st(), _Checksum :: checksum()) ->
ok | {error, _Reason}.
-callback discard(ExportSt :: export_st()) ->
ok | {error, _Reason}.
-callback list(exporter_conf(), emqx_ft_storage:query(Cursor)) ->
{ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}.
%% Lifecycle callbacks
-callback start(exporter_conf()) ->
ok | {error, _Reason}.
-callback stop(exporter_conf()) ->
ok.
-callback update(exporter_conf(), exporter_conf()) ->
ok | {error, _Reason}.
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec start_export(storage(), transfer(), filemeta()) ->
{ok, export()} | {error, _Reason}.
start_export(Storage, Transfer, Filemeta) ->
{ExporterMod, ExporterConf} = exporter(Storage),
case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of
{ok, ExportSt} ->
{ok, #{
mod => ExporterMod,
st => ExportSt,
hash => init_checksum(Filemeta),
filemeta => Filemeta
}};
{error, _} = Error ->
Error
end.
-spec write(export(), iodata()) ->
{ok, export()} | {error, _Reason}.
write(#{mod := ExporterMod, st := ExportSt, hash := Hash} = Export, Content) ->
case ExporterMod:write(ExportSt, Content) of
{ok, ExportStNext} ->
{ok, Export#{
st := ExportStNext,
hash := update_checksum(Hash, Content)
}};
{error, _} = Error ->
Error
end.
-spec complete(export()) ->
ok | {error, _Reason}.
complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemeta}) ->
case verify_checksum(Hash, Filemeta) of
{ok, Checksum} ->
ExporterMod:complete(ExportSt, Checksum);
{error, _} = Error ->
_ = ExporterMod:discard(ExportSt),
Error
end.
-spec discard(export()) ->
ok | {error, _Reason}.
discard(#{mod := ExporterMod, st := ExportSt}) ->
ExporterMod:discard(ExportSt).
-spec list(storage(), emqx_ft_storage:query(Cursor)) ->
{ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}.
list(Storage, Query) ->
{ExporterMod, ExporterOpts} = exporter(Storage),
ExporterMod:list(ExporterOpts, Query).
%% Lifecycle
-spec on_config_update(storage(), storage()) -> ok | {error, term()}.
on_config_update(StorageOld, StorageNew) ->
on_exporter_update(
emqx_maybe:apply(fun exporter/1, StorageOld),
emqx_maybe:apply(fun exporter/1, StorageNew)
).
on_exporter_update(Config, Config) ->
ok;
on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) ->
ExporterMod:update(ConfigOld, ConfigNew);
on_exporter_update(ExporterOld, ExporterNew) ->
_ = emqx_maybe:apply(fun stop/1, ExporterOld),
_ = emqx_maybe:apply(fun start/1, ExporterNew),
ok.
start({ExporterMod, ExporterOpts}) ->
ok = ExporterMod:start(ExporterOpts).
stop({ExporterMod, ExporterOpts}) ->
ok = ExporterMod:stop(ExporterOpts).
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------
exporter(Storage) ->
case maps:get(exporter, Storage) of
#{local := Options} ->
{emqx_ft_storage_exporter_fs, Options};
#{s3 := Options} ->
{emqx_ft_storage_exporter_s3, Options}
end.
init_checksum(#{checksum := {Algo, _}}) ->
crypto:hash_init(Algo);
init_checksum(#{}) ->
crypto:hash_init(sha256).
update_checksum(Ctx, IoData) ->
crypto:hash_update(Ctx, IoData).
verify_checksum(Ctx, #{checksum := {Algo, Digest} = Checksum}) ->
case crypto:hash_final(Ctx) of
Digest ->
{ok, Checksum};
Mismatch ->
{error, {checksum, Algo, binary:encode_hex(Mismatch)}}
end;
verify_checksum(Ctx, #{}) ->
Digest = crypto:hash_final(Ctx),
{ok, {sha256, Digest}}.

489
apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl Normal file
View File

@ -0,0 +1,489 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_exporter_fs).
-include_lib("kernel/include/file.hrl").
-include_lib("emqx/include/logger.hrl").
%% Exporter API
-behaviour(emqx_ft_storage_exporter).
-export([start_export/3]).
-export([write/2]).
-export([complete/2]).
-export([discard/1]).
-export([list/1]).
-export([
start/1,
stop/1,
update/2
]).
%% Internal API for RPC
-export([list_local/1]).
-export([list_local/2]).
-export([list_local_transfer/2]).
-export([start_reader/3]).
-export([list/2]).
-export_type([export_st/0]).
-export_type([options/0]).
-type options() :: #{
root => file:name(),
_ => _
}.
-type query() :: emqx_ft_storage:query(cursor()).
-type page(T) :: emqx_ft_storage:page(T, cursor()).
-type cursor() :: iodata().
-type transfer() :: emqx_ft:transfer().
-type filemeta() :: emqx_ft:filemeta().
-type exportinfo() :: emqx_ft_storage:file_info().
-type file_error() :: emqx_ft_storage_fs:file_error().
-type export_st() :: #{
path := file:name(),
handle := io:device(),
result := file:name(),
meta := filemeta()
}.
-type reader() :: pid().
-define(TEMPDIR, "tmp").
-define(MANIFEST, ".MANIFEST.json").
%% NOTE
%% Bucketing of resulting files to accomodate the storage backend for considerably
%% large (e.g. > 10s of millions) amount of files.
-define(BUCKET_HASH, sha).
%% 2 symbols = at most 256 directories on the upper level
-define(BUCKET1_LEN, 2).
%% 2 symbols = at most 256 directories on the second level
-define(BUCKET2_LEN, 2).
%%--------------------------------------------------------------------
%% Exporter behaviour
%%--------------------------------------------------------------------
-spec start_export(options(), transfer(), filemeta()) ->
{ok, export_st()} | {error, file_error()}.
start_export(Options, Transfer, Filemeta = #{name := Filename}) ->
TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename),
ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename),
_ = filelib:ensure_dir(TempFilepath),
case file:open(TempFilepath, [write, raw, binary]) of
{ok, Handle} ->
{ok, #{
path => TempFilepath,
handle => Handle,
result => ResultFilepath,
meta => Filemeta
}};
{error, _} = Error ->
Error
end.
-spec write(export_st(), iodata()) ->
{ok, export_st()} | {error, file_error()}.
write(ExportSt = #{handle := Handle}, IoData) ->
case file:write(Handle, IoData) of
ok ->
{ok, ExportSt};
{error, _} = Error ->
_ = discard(ExportSt),
Error
end.
-spec complete(export_st(), emqx_ft:checksum()) ->
ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}.
complete(
#{
path := Filepath,
handle := Handle,
result := ResultFilepath,
meta := FilemetaIn
},
Checksum
) ->
Filemeta = FilemetaIn#{checksum => Checksum},
ok = file:close(Handle),
_ = filelib:ensure_dir(ResultFilepath),
_ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)),
file:rename(Filepath, ResultFilepath).
-spec discard(export_st()) ->
ok.
discard(#{path := Filepath, handle := Handle}) ->
ok = file:close(Handle),
file:delete(Filepath).
%%--------------------------------------------------------------------
%% Exporter behaviour (lifecycle)
%%--------------------------------------------------------------------
%% FS Exporter does not have require any stateful entities,
%% so lifecycle callbacks are no-op.
-spec start(options()) -> ok.
start(_Options) -> ok.
-spec stop(options()) -> ok.
stop(_Options) -> ok.
-spec update(options(), options()) -> ok.
update(_OldOptions, _NewOptions) -> ok.
%%--------------------------------------------------------------------
%% Internal API
%%--------------------------------------------------------------------
-type local_query() :: emqx_ft_storage:query({transfer(), file:name()}).
-spec list_local_transfer(options(), transfer()) ->
{ok, [exportinfo()]} | {error, file_error()}.
list_local_transfer(Options, Transfer) ->
It = emqx_ft_fs_iterator:new(
mk_absdir(Options, Transfer, result),
[fun filter_manifest/1]
),
Result = emqx_ft_fs_iterator:fold(
fun
({leaf, _Path, Fileinfo = #file_info{type = regular}, [Filename | _]}, Acc) ->
RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]),
Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo),
[Info | Acc];
({node, _Path, {error, Reason}, []}, []) ->
{error, Reason};
(Entry, Acc) ->
ok = log_invalid_entry(Options, Entry),
Acc
end,
[],
It
),
case Result of
Infos = [_ | _] ->
{ok, lists:reverse(Infos)};
[] ->
{error, enoent};
{error, Reason} ->
{error, Reason}
end.
-spec list_local(options()) ->
{ok, [exportinfo()]} | {error, file_error()}.
list_local(Options) ->
list_local(Options, #{}).
-spec list_local(options(), local_query()) ->
{ok, [exportinfo()]} | {error, file_error()}.
list_local(Options, #{transfer := Transfer}) ->
list_local_transfer(Options, Transfer);
list_local(Options, #{} = Query) ->
Root = get_storage_root(Options),
Glob = [
_Bucket1 = '*',
_Bucket2 = '*',
_Rest = '*',
_ClientId = '*',
_FileId = '*',
fun filter_manifest/1
],
It =
case Query of
#{following := Cursor} ->
emqx_ft_fs_iterator:seek(mk_path_seek(Cursor), Root, Glob);
#{} ->
emqx_ft_fs_iterator:new(Root, Glob)
end,
% NOTE
% In the rare case when some transfer contain more than one file, the paging mechanic
% here may skip over some files, when the cursor is transfer-only.
Limit = maps:get(limit, Query, -1),
{Exports, _} = emqx_ft_fs_iterator:fold_n(
fun(Entry, Acc) -> read_exportinfo(Options, Entry, Acc) end,
[],
It,
Limit
),
{ok, Exports}.
mk_path_seek(#{transfer := Transfer, name := Filename}) ->
mk_result_reldir(Transfer) ++ [Filename];
mk_path_seek(#{transfer := Transfer}) ->
% NOTE: Any bitstring is greater than any list.
mk_result_reldir(Transfer) ++ [<<>>].
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
filter_manifest(?MANIFEST) ->
% Filename equals `?MANIFEST`, there should also be a manifest for it.
false;
filter_manifest(Filename) ->
?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing).
read_exportinfo(
Options,
{leaf, RelFilepath, Fileinfo = #file_info{type = regular}, [Filename, FileId, ClientId | _]},
Acc
) ->
% NOTE
% There might be more than one file for a single transfer (though
% extremely bad luck is needed for that, e.g. concurrent assemblers with
% different filemetas from different nodes). This might be unexpected for a
% client given the current protocol, yet might be helpful in the future.
Transfer = dirnames_to_transfer(ClientId, FileId),
Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo),
[Info | Acc];
read_exportinfo(_Options, {node, _Root = "", {error, enoent}, []}, Acc) ->
% NOTE: Root directory does not exist, this is not an error.
Acc;
read_exportinfo(Options, Entry, Acc) ->
ok = log_invalid_entry(Options, Entry),
Acc.
mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) ->
Root = get_storage_root(Options),
try_read_filemeta(
filename:join(Root, mk_manifest_filename(RelFilepath)),
#{
transfer => Transfer,
name => Filename,
uri => mk_export_uri(RelFilepath),
timestamp => Fileinfo#file_info.mtime,
size => Fileinfo#file_info.size,
path => filename:join(Root, RelFilepath)
}
).
try_read_filemeta(Filepath, Info) ->
case emqx_ft_fs_util:read_decode_file(Filepath, fun decode_filemeta/1) of
{ok, Filemeta} ->
Info#{meta => Filemeta};
{error, Reason} ->
?SLOG(warning, "filemeta_inaccessible", #{
path => Filepath,
reason => Reason
}),
Info
end.
mk_export_uri(RelFilepath) ->
emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath).
log_invalid_entry(Options, {_Type, RelFilepath, Fileinfo = #file_info{}, _Stack}) ->
?SLOG(notice, "filesystem_object_unexpected", #{
relpath => RelFilepath,
fileinfo => Fileinfo,
options => Options
});
log_invalid_entry(Options, {_Type, RelFilepath, {error, Reason}, _Stack}) ->
?SLOG(warning, "filesystem_object_inaccessible", #{
relpath => RelFilepath,
reason => Reason,
options => Options
}).
-spec start_reader(options(), file:name(), _Caller :: pid()) ->
{ok, reader()} | {error, enoent}.
start_reader(Options, RelFilepath, CallerPid) ->
Root = get_storage_root(Options),
case filelib:safe_relative_path(RelFilepath, Root) of
SafeFilepath when SafeFilepath /= unsafe ->
AbsFilepath = filename:join(Root, SafeFilepath),
emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath);
unsafe ->
{error, enoent}
end.
%%
-spec list(options(), query()) ->
{ok, page(exportinfo())} | {error, [{node(), _Reason}]}.
list(_Options, Query = #{transfer := _Transfer}) ->
case list(Query) of
#{items := Exports = [_ | _]} ->
{ok, #{items => Exports}};
#{items := [], errors := NodeErrors} ->
{error, NodeErrors};
#{items := []} ->
{ok, #{items => []}}
end;
list(_Options, Query) ->
Result = list(Query),
case Result of
#{errors := NodeErrors} ->
?SLOG(warning, "list_exports_errors", #{
query => Query,
errors => NodeErrors
});
#{} ->
ok
end,
case Result of
#{items := Exports, cursor := Cursor} ->
{ok, #{items => lists:reverse(Exports), cursor => encode_cursor(Cursor)}};
#{items := Exports} ->
{ok, #{items => lists:reverse(Exports)}}
end.
list(QueryIn) ->
{Nodes, NodeQuery} = decode_query(QueryIn, lists:sort(mria_mnesia:running_nodes())),
list_nodes(NodeQuery, Nodes, #{items => []}).
list_nodes(Query, Nodes = [Node | Rest], Acc) ->
case emqx_ft_storage_exporter_fs_proto_v1:list_exports([Node], Query) of
[{ok, Result}] ->
list_accumulate(Result, Query, Nodes, Acc);
[Failure] ->
?SLOG(warning, #{
msg => "list_remote_exports_failed",
node => Node,
query => Query,
failure => Failure
}),
list_next(Query, Rest, Acc)
end;
list_nodes(_Query, [], Acc) ->
Acc.
list_accumulate({ok, Exports}, Query, [Node | Rest], Acc = #{items := EAcc}) ->
NExports = length(Exports),
AccNext = Acc#{items := Exports ++ EAcc},
case Query of
#{limit := Limit} when NExports < Limit ->
list_next(Query#{limit => Limit - NExports}, Rest, AccNext);
#{limit := _} ->
AccNext#{cursor => mk_cursor(Node, Exports)};
#{} ->
list_next(Query, Rest, AccNext)
end;
list_accumulate({error, Reason}, Query, [Node | Rest], Acc) ->
EAcc = maps:get(errors, Acc, []),
list_next(Query, Rest, Acc#{errors => [{Node, Reason} | EAcc]}).
list_next(Query, Nodes, Acc) ->
list_nodes(maps:remove(following, Query), Nodes, Acc).
decode_query(Query = #{following := Cursor}, Nodes) ->
{Node, NodeCursor} = decode_cursor(Cursor),
{skip_query_nodes(Node, Nodes), Query#{following => NodeCursor}};
decode_query(Query = #{}, Nodes) ->
{Nodes, Query}.
skip_query_nodes(CNode, Nodes) ->
lists:dropwhile(fun(N) -> N < CNode end, Nodes).
mk_cursor(Node, [_Last = #{transfer := Transfer, name := Name} | _]) ->
{Node, #{transfer => Transfer, name => Name}}.
encode_cursor({Node, #{transfer := {ClientId, FileId}, name := Name}}) ->
emqx_utils_json:encode(#{
<<"n">> => Node,
<<"cid">> => ClientId,
<<"fid">> => FileId,
<<"fn">> => unicode:characters_to_binary(Name)
}).
decode_cursor(Cursor) ->
try
#{
<<"n">> := NodeIn,
<<"cid">> := ClientId,
<<"fid">> := FileId,
<<"fn">> := NameIn
} = emqx_utils_json:decode(Cursor),
true = is_binary(ClientId),
true = is_binary(FileId),
Node = binary_to_existing_atom(NodeIn),
Name = unicode:characters_to_list(NameIn),
true = is_list(Name),
{Node, #{transfer => {ClientId, FileId}, name => Name}}
catch
error:{_, invalid_json} ->
error({badarg, cursor});
error:{badmatch, _} ->
error({badarg, cursor});
error:badarg ->
error({badarg, cursor})
end.
%%
-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]).
encode_filemeta(Meta) ->
emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))).
decode_filemeta(Binary) when is_binary(Binary) ->
?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]),
case emqx_ft:decode_filemeta(Map) of
{ok, Meta} ->
Meta;
{error, Reason} ->
error(Reason)
end.
mk_manifest_filename(Filename) when is_list(Filename) ->
Filename ++ ?MANIFEST;
mk_manifest_filename(Filename) when is_binary(Filename) ->
<<Filename/binary, ?MANIFEST>>.
mk_temp_absfilepath(Options, Transfer, Filename) ->
Unique = erlang:unique_integer([positive]),
TempFilename = integer_to_list(Unique) ++ "." ++ Filename,
filename:join(mk_absdir(Options, Transfer, temporary), TempFilename).
mk_absdir(Options, _Transfer, temporary) ->
filename:join([get_storage_root(Options), ?TEMPDIR]);
mk_absdir(Options, Transfer, result) ->
filename:join([get_storage_root(Options) | mk_result_reldir(Transfer)]).
mk_absfilepath(Options, Transfer, What, Filename) ->
filename:join(mk_absdir(Options, Transfer, What), Filename).
mk_result_reldir(Transfer = {ClientId, FileId}) ->
Hash = mk_transfer_hash(Transfer),
<<
Bucket1:?BUCKET1_LEN/binary,
Bucket2:?BUCKET2_LEN/binary,
BucketRest/binary
>> = binary:encode_hex(Hash),
[
binary_to_list(Bucket1),
binary_to_list(Bucket2),
binary_to_list(BucketRest),
emqx_ft_fs_util:escape_filename(ClientId),
emqx_ft_fs_util:escape_filename(FileId)
].
dirnames_to_transfer(ClientId, FileId) ->
{emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}.
mk_transfer_hash(Transfer) ->
crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)).
get_storage_root(Options) ->
maps:get(root, Options, filename:join([emqx:data_dir(), file_transfer, exports])).

182
apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl Normal file
View File

@ -0,0 +1,182 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_exporter_fs_api).
-behaviour(minirest_api).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
%% Swagger specs from hocon schema
-export([
api_spec/0,
paths/0,
schema/1,
namespace/0
]).
-export([
fields/1,
roots/0
]).
%% API callbacks
-export([
'/file_transfer/file'/2
]).
-export([mk_export_uri/2]).
%%
namespace() -> "file_transfer".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{
check_schema => true, filter => fun emqx_ft_api:check_ft_enabled/2
}).
paths() ->
[
"/file_transfer/file"
].
schema("/file_transfer/file") ->
#{
'operationId' => '/file_transfer/file',
get => #{
tags => [<<"file_transfer">>],
summary => <<"Download a particular file">>,
description => ?DESC("file_get"),
parameters => [
hoconsc:ref(file_node),
hoconsc:ref(file_ref)
],
responses => #{
200 => <<"Operation success">>,
404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>),
503 => emqx_dashboard_swagger:error_codes(
['SERVICE_UNAVAILABLE'], <<"Service unavailable">>
)
}
}
}.
roots() ->
[
file_node,
file_ref
].
-spec fields(hocon_schema:name()) -> hocon_schema:fields().
fields(file_ref) ->
[
{fileref,
hoconsc:mk(binary(), #{
in => query,
desc => <<"File reference">>,
example => <<"file1">>,
required => true
})}
];
fields(file_node) ->
[
{node,
hoconsc:mk(binary(), #{
in => query,
desc => <<"Node under which the file is located">>,
example => atom_to_list(node()),
required => true
})}
].
'/file_transfer/file'(get, #{query_string := Query}) ->
try
Node = parse_node(maps:get(<<"node">>, Query)),
Filepath = parse_filepath(maps:get(<<"fileref">>, Query)),
case emqx_ft_storage_exporter_fs_proto_v1:read_export_file(Node, Filepath, self()) of
{ok, ReaderPid} ->
FileData = emqx_ft_storage_fs_reader:table(ReaderPid),
{200,
#{
<<"content-type">> => <<"application/data">>,
<<"content-disposition">> => <<"attachment">>
},
FileData};
{error, enoent} ->
{404, error_msg('NOT_FOUND', <<"Not found">>)};
{error, Error} ->
?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}),
{503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)}
end
catch
throw:{invalid, Param} ->
{404,
error_msg(
'NOT_FOUND',
iolist_to_binary(["Invalid query parameter: ", Param])
)};
error:{erpc, noconnection} ->
{503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)}
end.
error_msg(Code, Msg) ->
#{code => Code, message => emqx_utils:readable_error_msg(Msg)}.
-spec mk_export_uri(node(), file:name()) ->
uri_string:uri_string().
mk_export_uri(Node, Filepath) ->
emqx_dashboard_swagger:relative_uri([
"/file_transfer/file?",
uri_string:compose_query([
{"node", atom_to_list(Node)},
{"fileref", Filepath}
])
]).
%%
parse_node(NodeBin) ->
case emqx_utils:safe_to_existing_atom(NodeBin) of
{ok, Node} ->
Node;
{error, _} ->
throw({invalid, NodeBin})
end.
parse_filepath(PathBin) ->
case filename:pathtype(PathBin) of
relative ->
ok;
absolute ->
throw({invalid, PathBin})
end,
PathComponents = filename:split(PathBin),
case lists:any(fun is_special_component/1, PathComponents) of
false ->
filename:join(PathComponents);
true ->
throw({invalid, PathBin})
end.
is_special_component(<<".", _/binary>>) ->
true;
is_special_component([$. | _]) ->
true;
is_special_component(_) ->
false.

50
apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl Normal file
View File

@ -0,0 +1,50 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% This methods are called via rpc by `emqx_ft_storage_exporter_fs`
%% They populate the call with actual storage which may be configured differently
%% on a concrete node.
-module(emqx_ft_storage_exporter_fs_proxy).
-export([
list_exports_local/1,
read_export_file_local/2
]).
list_exports_local(Query) ->
emqx_ft_storage:with_storage_type(local, fun(Storage) ->
case emqx_ft_storage_exporter:exporter(Storage) of
{emqx_ft_storage_exporter_fs, Options} ->
emqx_ft_storage_exporter_fs:list_local(Options, Query)
% NOTE
% This case clause is currently deemed unreachable by dialyzer.
% InvalidExporter ->
% {error, {invalid_exporter, InvalidExporter}}
end
end).
read_export_file_local(Filepath, CallerPid) ->
emqx_ft_storage:with_storage_type(local, fun(Storage) ->
case emqx_ft_storage_exporter:exporter(Storage) of
{emqx_ft_storage_exporter_fs, Options} ->
emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid)
% NOTE
% This case clause is currently deemed unreachable by dialyzer.
% InvalidExporter ->
% {error, {invalid_exporter, InvalidExporter}}
end
end).

251
apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl Normal file
View File

@ -0,0 +1,251 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_exporter_s3).
-include_lib("emqx/include/logger.hrl").
%% Exporter API
-export([start_export/3]).
-export([write/2]).
-export([complete/2]).
-export([discard/1]).
-export([list/2]).
-export([
start/1,
stop/1,
update/2
]).
-type options() :: emqx_s3:profile_config().
-type transfer() :: emqx_ft:transfer().
-type filemeta() :: emqx_ft:filemeta().
-type exportinfo() :: #{
transfer := transfer(),
name := file:name(),
uri := uri_string:uri_string(),
timestamp := emqx_datetime:epoch_second(),
size := _Bytes :: non_neg_integer(),
filemeta => filemeta()
}.
-type query() :: emqx_ft_storage:query(cursor()).
-type page(T) :: emqx_ft_storage:page(T, cursor()).
-type cursor() :: iodata().
-type export_st() :: #{
pid := pid(),
filemeta := filemeta(),
transfer := transfer()
}.
-define(S3_PROFILE_ID, ?MODULE).
-define(FILEMETA_VSN, <<"1">>).
-define(S3_LIST_LIMIT, 500).
%%--------------------------------------------------------------------
%% Exporter behaviour
%%--------------------------------------------------------------------
-spec start_export(options(), transfer(), filemeta()) ->
{ok, export_st()} | {error, term()}.
start_export(_Options, Transfer, Filemeta) ->
Options = #{
key => s3_key(Transfer, Filemeta),
headers => s3_headers(Transfer, Filemeta)
},
case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of
{ok, Pid} ->
true = erlang:link(Pid),
{ok, #{filemeta => Filemeta, pid => Pid}};
{error, _Reason} = Error ->
Error
end.
-spec write(export_st(), iodata()) ->
{ok, export_st()} | {error, term()}.
write(#{pid := Pid} = ExportSt, IoData) ->
case emqx_s3_uploader:write(Pid, IoData) of
ok ->
{ok, ExportSt};
{error, _Reason} = Error ->
Error
end.
-spec complete(export_st(), emqx_ft:checksum()) ->
ok | {error, term()}.
complete(#{pid := Pid} = _ExportSt, _Checksum) ->
emqx_s3_uploader:complete(Pid).
-spec discard(export_st()) ->
ok.
discard(#{pid := Pid} = _ExportSt) ->
emqx_s3_uploader:abort(Pid).
-spec list(options(), query()) ->
{ok, page(exportinfo())} | {error, term()}.
list(Options, Query) ->
emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options, Query) end).
%%--------------------------------------------------------------------
%% Exporter behaviour (lifecycle)
%%--------------------------------------------------------------------
-spec start(options()) -> ok | {error, term()}.
start(Options) ->
emqx_s3:start_profile(?S3_PROFILE_ID, Options).
-spec stop(options()) -> ok.
stop(_Options) ->
ok = emqx_s3:stop_profile(?S3_PROFILE_ID).
-spec update(options(), options()) -> ok.
update(_OldOptions, NewOptions) ->
emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions).
%%--------------------------------------------------------------------
%% Internal functions
%% -------------------------------------------------------------------
s3_key(Transfer, #{name := Filename}) ->
s3_prefix(Transfer) ++ "/" ++ Filename.
s3_prefix({ClientId, FileId} = _Transfer) ->
emqx_ft_fs_util:escape_filename(ClientId) ++ "/" ++ emqx_ft_fs_util:escape_filename(FileId).
s3_headers({ClientId, FileId}, Filemeta) ->
#{
%% The ClientID MUST be a UTF-8 Encoded String
<<"x-amz-meta-clientid">> => ClientId,
%% It [Topic Name] MUST be a UTF-8 Encoded String
<<"x-amz-meta-fileid">> => FileId,
<<"x-amz-meta-filemeta">> => s3_header_filemeta(Filemeta),
<<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN
}.
s3_header_filemeta(Filemeta) ->
emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]).
list(Client, _Options, #{transfer := Transfer}) ->
case list_key_info(Client, [{prefix, s3_prefix(Transfer)}, {max_keys, ?S3_LIST_LIMIT}]) of
{ok, {Exports, _Marker}} ->
{ok, #{items => Exports}};
{error, _Reason} = Error ->
Error
end;
list(Client, _Options, Query) ->
Limit = maps:get(limit, Query, undefined),
Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(following, Query, undefined)),
case list_pages(Client, Marker, Limit, []) of
{ok, {Exports, undefined}} ->
{ok, #{items => Exports}};
{ok, {Exports, NextMarker}} ->
{ok, #{items => Exports, cursor => encode_cursor(NextMarker)}};
{error, _Reason} = Error ->
Error
end.
list_pages(Client, Marker, Limit, Acc) ->
MaxKeys = min(?S3_LIST_LIMIT, Limit),
ListOptions = [{marker, Marker} || Marker =/= undefined],
case list_key_info(Client, [{max_keys, MaxKeys} | ListOptions]) of
{ok, {Exports, NextMarker}} ->
list_accumulate(Client, Limit, NextMarker, [Exports | Acc]);
{error, _Reason} = Error ->
Error
end.
list_accumulate(_Client, _Limit, undefined, Acc) ->
{ok, {flatten_pages(Acc), undefined}};
list_accumulate(Client, undefined, Marker, Acc) ->
list_pages(Client, Marker, undefined, Acc);
list_accumulate(Client, Limit, Marker, Acc = [Exports | _]) ->
case Limit - length(Exports) of
0 ->
{ok, {flatten_pages(Acc), Marker}};
Left ->
list_pages(Client, Marker, Left, Acc)
end.
flatten_pages(Pages) ->
lists:append(lists:reverse(Pages)).
list_key_info(Client, ListOptions) ->
case emqx_s3_client:list(Client, ListOptions) of
{ok, Result} ->
?SLOG(debug, #{msg => "list_key_info", result => Result}),
KeyInfos = proplists:get_value(contents, Result, []),
Exports = lists:filtermap(
fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo) end, KeyInfos
),
Marker =
case proplists:get_value(is_truncated, Result, false) of
true ->
next_marker(KeyInfos);
false ->
undefined
end,
{ok, {Exports, Marker}};
{error, _Reason} = Error ->
Error
end.
encode_cursor(Key) ->
unicode:characters_to_binary(Key).
decode_cursor(Cursor) ->
case unicode:characters_to_list(Cursor) of
Key when is_list(Key) ->
Key;
_ ->
error({badarg, cursor})
end.
next_marker(KeyInfos) ->
proplists:get_value(key, lists:last(KeyInfos)).
key_info_to_exportinfo(Client, KeyInfo) ->
Key = proplists:get_value(key, KeyInfo),
case parse_transfer_and_name(Key) of
{ok, {Transfer, Name}} ->
{true, #{
transfer => Transfer,
name => unicode:characters_to_binary(Name),
uri => emqx_s3_client:uri(Client, Key),
timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)),
size => proplists:get_value(size, KeyInfo)
}};
{error, _Reason} ->
false
end.
-define(EPOCH_START, 62167219200).
datetime_to_epoch_second(DateTime) ->
calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START.
parse_transfer_and_name(Key) ->
case string:split(Key, "/", all) of
[ClientId, FileId, Name] ->
Transfer = {
emqx_ft_fs_util:unescape_filename(ClientId),
emqx_ft_fs_util:unescape_filename(FileId)
},
{ok, {Transfer, Name}};
_ ->
{error, invalid_key}
end.

506
apps/emqx_ft/src/emqx_ft_storage_fs.erl Normal file
View File

@ -0,0 +1,506 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Filesystem storage backend
%%
%% NOTE
%% If you plan to change storage layout please consult `emqx_ft_storage_fs_gc`
%% to see how much it would break or impair GC.
-module(emqx_ft_storage_fs).
-behaviour(emqx_ft_storage).
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-export([child_spec/1]).
% Segments-related API
-export([store_filemeta/3]).
-export([store_segment/3]).
-export([read_filemeta/2]).
-export([list/3]).
-export([pread/5]).
-export([lookup_local_assembler/1]).
-export([assemble/3]).
-export([transfers/1]).
% GC API
% TODO: This is quickly becomes hairy.
-export([get_root/1]).
-export([get_subdir/2]).
-export([get_subdir/3]).
-export([files/2]).
-export([on_config_update/2]).
-export([start/1]).
-export([stop/1]).
-export_type([storage/0]).
-export_type([filefrag/1]).
-export_type([filefrag/0]).
-export_type([transferinfo/0]).
-export_type([file_error/0]).
-type transfer() :: emqx_ft:transfer().
-type offset() :: emqx_ft:offset().
-type filemeta() :: emqx_ft:filemeta().
-type segment() :: emqx_ft:segment().
-type segmentinfo() :: #{
offset := offset(),
size := _Bytes :: non_neg_integer()
}.
-type transferinfo() :: #{
filemeta => filemeta()
}.
% TODO naming
-type filefrag(T) :: #{
path := file:name(),
timestamp := emqx_datetime:epoch_second(),
size := _Bytes :: non_neg_integer(),
fragment := T
}.
-type filefrag() :: filefrag(
{filemeta, filemeta()}
| {segment, segmentinfo()}
).
-define(FRAGDIR, frags).
-define(TEMPDIR, tmp).
-define(MANIFEST, "MANIFEST.json").
-define(SEGMENT, "SEG").
-type segments() :: #{
root := file:name(),
gc := #{
interval := non_neg_integer(),
maximum_segments_ttl := non_neg_integer(),
minimum_segments_ttl := non_neg_integer()
}
}.
-type storage() :: #{
type := 'local',
segments := segments(),
exporter := emqx_ft_storage_exporter:exporter()
}.
-type file_error() ::
file:posix()
%% Filename is incompatible with the backing filesystem.
| badarg
%% System limit (e.g. number of ports) reached.
| system_limit.
%% Related resources childspecs
-spec child_spec(storage()) ->
[supervisor:child_spec()].
child_spec(Storage) ->
[
#{
id => emqx_ft_storage_fs_gc,
start => {emqx_ft_storage_fs_gc, start_link, [Storage]},
restart => permanent
}
].
%% Store manifest in the backing filesystem.
%% Atomic operation.
-spec store_filemeta(storage(), transfer(), filemeta()) ->
% Quota? Some lower level errors?
ok | {error, conflict} | {error, file_error()}.
store_filemeta(Storage, Transfer, Meta) ->
Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST),
case read_file(Filepath, fun decode_filemeta/1) of
{ok, Meta} ->
_ = touch_file(Filepath),
ok;
{ok, Conflict} ->
?SLOG(warning, #{
msg => "filemeta_conflict", transfer => Transfer, new => Meta, old => Conflict
}),
% TODO
% We won't see conflicts in case of concurrent `store_filemeta`
% requests. It's rather odd scenario so it's fine not to worry
% about it too much now.
{error, conflict};
{error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent ->
write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta));
{error, _} = Error ->
Error
end.
%% Store a segment in the backing filesystem.
%% Atomic operation.
-spec store_segment(storage(), transfer(), segment()) ->
% Where is the checksum gets verified? Upper level probably.
% Quota? Some lower level errors?
ok | {error, file_error()}.
store_segment(Storage, Transfer, Segment = {_Offset, Content}) ->
Filename = mk_segment_filename(Segment),
Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), Filename),
write_file_atomic(Storage, Transfer, Filepath, Content).
-spec read_filemeta(storage(), transfer()) ->
{ok, filemeta()} | {error, corrupted} | {error, file_error()}.
read_filemeta(Storage, Transfer) ->
Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST),
read_file(Filepath, fun decode_filemeta/1).
-spec list(storage(), transfer(), _What :: fragment) ->
% Some lower level errors? {error, notfound}?
% Result will contain zero or only one filemeta.
{ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]}
| {error, file_error()}.
list(Storage, Transfer, What = fragment) ->
Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)),
case file:list_dir(Dirname) of
{ok, Filenames} ->
% TODO
% In case of `What = result` there might be more than one file (though
% extremely bad luck is needed for that, e.g. concurrent assemblers with
% different filemetas from different nodes). This might be unexpected for a
% client given the current protocol, yet might be helpful in the future.
{ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)};
{error, enoent} ->
{ok, []};
{error, _} = Error ->
Error
end.
-spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) ->
{ok, _Content :: iodata()} | {error, eof} | {error, file_error()}.
pread(_Storage, _Transfer, Frag, Offset, Size) ->
Filepath = maps:get(path, Frag),
case file:open(Filepath, [read, raw, binary]) of
{ok, IoDevice} ->
% NOTE
% Reading empty file is always `eof`.
Read = file:pread(IoDevice, Offset, Size),
ok = file:close(IoDevice),
case Read of
{ok, Content} ->
{ok, Content};
eof ->
{error, eof};
{error, Reason} ->
{error, Reason}
end;
{error, Reason} ->
{error, Reason}
end.
-spec assemble(storage(), transfer(), emqx_ft:bytes()) ->
{async, _Assembler :: pid()} | ok | {error, _TODO}.
assemble(Storage, Transfer, Size) ->
LookupSources = [
fun() -> lookup_local_assembler(Transfer) end,
fun() -> lookup_remote_assembler(Transfer) end,
fun() -> check_if_already_exported(Storage, Transfer) end,
fun() -> ensure_local_assembler(Storage, Transfer, Size) end
],
lookup_assembler(LookupSources).
%%
files(Storage, Query) ->
emqx_ft_storage_exporter:list(Storage, Query).
%%
on_config_update(StorageOld, StorageNew) ->
% NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely
ok = emqx_ft_storage_fs_gc:reset(StorageNew),
emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew).
start(Storage) ->
ok = lists:foreach(
fun(ChildSpec) ->
{ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec)
end,
child_spec(Storage)
),
ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage),
ok.
stop(Storage) ->
ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined),
ok = lists:foreach(
fun(#{id := ChildId}) ->
_ = supervisor:terminate_child(emqx_ft_sup, ChildId),
ok = supervisor:delete_child(emqx_ft_sup, ChildId)
end,
child_spec(Storage)
),
ok.
%%
lookup_assembler([LastSource]) ->
LastSource();
lookup_assembler([Source | Sources]) ->
case Source() of
{error, not_found} -> lookup_assembler(Sources);
Result -> Result
end.
check_if_already_exported(Storage, Transfer) ->
case files(Storage, #{transfer => Transfer}) of
{ok, #{items := [_ | _]}} -> ok;
_ -> {error, not_found}
end.
lookup_local_assembler(Transfer) ->
case emqx_ft_assembler:where(Transfer) of
Pid when is_pid(Pid) -> {async, Pid};
_ -> {error, not_found}
end.
lookup_remote_assembler(Transfer) ->
Nodes = emqx:running_nodes() -- [node()],
Assemblers = lists:flatmap(
fun
({ok, {async, Pid}}) -> [Pid];
(_) -> []
end,
emqx_ft_storage_fs_proto_v1:list_assemblers(Nodes, Transfer)
),
case Assemblers of
[Pid | _] -> {async, Pid};
_ -> {error, not_found}
end.
ensure_local_assembler(Storage, Transfer, Size) ->
{ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size),
{async, Pid}.
-spec transfers(storage()) ->
{ok, #{transfer() => transferinfo()}}.
transfers(Storage) ->
% TODO `Continuation`
% There might be millions of transfers on the node, we need a protocol and
% storage schema to iterate through them effectively.
ClientIds = try_list_dir(get_root(Storage)),
{ok,
lists:foldl(
fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end,
#{},
ClientIds
)}.
transfers(Storage, ClientId, AccIn) ->
Dirname = filename:join(get_root(Storage), ClientId),
case file:list_dir(Dirname) of
{ok, FileIds} ->
lists:foldl(
fun(FileId, Acc) ->
Transfer = dirnames_to_transfer(ClientId, FileId),
read_transferinfo(Storage, Transfer, Acc)
end,
AccIn,
FileIds
);
{error, _Reason} ->
?tp(warning, "list_dir_failed", #{
storage => Storage,
directory => Dirname
}),
AccIn
end.
read_transferinfo(Storage, Transfer, Acc) ->
case read_filemeta(Storage, Transfer) of
{ok, Filemeta} ->
Acc#{Transfer => #{filemeta => Filemeta}};
{error, enoent} ->
Acc#{Transfer => #{}};
{error, Reason} ->
?tp(warning, "read_transferinfo_failed", #{
storage => Storage,
transfer => Transfer,
reason => Reason
}),
Acc
end.
-spec get_root(storage()) ->
file:name().
get_root(Storage) ->
case emqx_utils_maps:deep_find([segments, root], Storage) of
{ok, Root} ->
Root;
{not_found, _, _} ->
filename:join([emqx:data_dir(), file_transfer, segments])
end.
-spec get_subdir(storage(), transfer()) ->
file:name().
get_subdir(Storage, Transfer) ->
mk_filedir(Storage, Transfer, []).
-spec get_subdir(storage(), transfer(), fragment | temporary) ->
file:name().
get_subdir(Storage, Transfer, What) ->
mk_filedir(Storage, Transfer, get_subdirs_for(What)).
get_subdirs_for(fragment) ->
[?FRAGDIR];
get_subdirs_for(temporary) ->
[?TEMPDIR].
-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]).
encode_filemeta(Meta) ->
emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))).
decode_filemeta(Binary) when is_binary(Binary) ->
?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]),
case emqx_ft:decode_filemeta(Map) of
{ok, Meta} ->
Meta;
{error, Reason} ->
error(Reason)
end.
mk_segment_filename({Offset, Content}) ->
lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]).
break_segment_filename(Filename) ->
Regex = "^" ?SEGMENT "[.]([0-9]+)[.]([0-9]+)$",
Result = re:run(Filename, Regex, [{capture, all_but_first, list}]),
case Result of
{match, [Offset, Size]} ->
{ok, #{offset => list_to_integer(Offset), size => list_to_integer(Size)}};
nomatch ->
{error, invalid}
end.
mk_filedir(Storage, {ClientId, FileId}, SubDirs) ->
filename:join([
get_root(Storage),
emqx_ft_fs_util:escape_filename(ClientId),
emqx_ft_fs_util:escape_filename(FileId)
| SubDirs
]).
dirnames_to_transfer(ClientId, FileId) ->
{emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}.
mk_filepath(Storage, Transfer, SubDirs, Filename) ->
filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename).
try_list_dir(Dirname) ->
case file:list_dir(Dirname) of
{ok, List} -> List;
{error, _} -> []
end.
-include_lib("kernel/include/file.hrl").
read_file(Filepath, DecodeFun) ->
emqx_ft_fs_util:read_decode_file(Filepath, DecodeFun).
write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) ->
TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)),
Result = emqx_utils:pipeline(
[
fun filelib:ensure_dir/1,
fun write_contents/2,
fun(_) -> mv_temp_file(TempFilepath, Filepath) end
],
TempFilepath,
Content
),
case Result of
{ok, _, _} ->
_ = file:delete(TempFilepath),
ok;
{error, Reason, _} ->
{error, Reason}
end.
mk_temp_filepath(Storage, Transfer, Filename) ->
Unique = erlang:unique_integer([positive]),
filename:join(get_subdir(Storage, Transfer, temporary), mk_filename([Unique, ".", Filename])).
mk_filename(Comps) ->
lists:append(lists:map(fun mk_filename_component/1, Comps)).
mk_filename_component(I) when is_integer(I) -> integer_to_list(I);
mk_filename_component(A) when is_atom(A) -> atom_to_list(A);
mk_filename_component(B) when is_binary(B) -> unicode:characters_to_list(B);
mk_filename_component(S) when is_list(S) -> S.
write_contents(Filepath, Content) ->
file:write_file(Filepath, Content).
mv_temp_file(TempFilepath, Filepath) ->
_ = filelib:ensure_dir(Filepath),
file:rename(TempFilepath, Filepath).
touch_file(Filepath) ->
Now = erlang:localtime(),
file:change_time(Filepath, _Mtime = Now, _Atime = Now).
filtermap_files(Fun, Dirname, Filenames) ->
lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames).
mk_filefrag(Dirname, Filename = ?MANIFEST) ->
mk_filefrag(Dirname, Filename, filemeta, fun read_frag_filemeta/2);
mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) ->
mk_filefrag(Dirname, Filename, segment, fun read_frag_segmentinfo/2);
mk_filefrag(_Dirname, _Filename) ->
?tp(warning, "rogue_file_found", #{
directory => _Dirname,
filename => _Filename
}),
false.
mk_filefrag(Dirname, Filename, Tag, Fun) ->
Filepath = filename:join(Dirname, Filename),
% TODO error handling?
{ok, Fileinfo} = file:read_file_info(Filepath),
case Fun(Filename, Filepath) of
{ok, Frag} ->
{true, #{
path => Filepath,
timestamp => Fileinfo#file_info.mtime,
size => Fileinfo#file_info.size,
fragment => {Tag, Frag}
}};
{error, _Reason} ->
?tp(warning, "mk_filefrag_failed", #{
directory => Dirname,
filename => Filename,
type => Tag,
reason => _Reason
}),
false
end.
read_frag_filemeta(_Filename, Filepath) ->
read_file(Filepath, fun decode_filemeta/1).
read_frag_segmentinfo(Filename, _Filepath) ->
break_segment_filename(Filename).

393
apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl Normal file
View File

@ -0,0 +1,393 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Filesystem storage GC
%%
%% This is conceptually a part of the Filesystem storage backend, even
%% though it's tied to the backend module with somewhat narrow interface.
-module(emqx_ft_storage_fs_gc).
-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
-include_lib("kernel/include/file.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-export([start_link/1]).
-export([collect/0]).
-export([collect/3]).
-export([reset/0]).
-export([reset/1]).
-behaviour(gen_server).
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-record(st, {
next_gc_timer :: maybe(reference()),
last_gc :: maybe(gcstats())
}).
-type gcstats() :: #gcstats{}.
-define(IS_ENABLED(INTERVAL), (is_integer(INTERVAL) andalso INTERVAL > 0)).
%%
start_link(Storage) ->
gen_server:start_link(mk_server_ref(global), ?MODULE, Storage, []).
-spec collect() -> gcstats().
collect() ->
gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity).
-spec reset() -> ok | {error, _}.
reset() ->
emqx_ft_storage:with_storage_type(local, fun reset/1).
-spec reset(emqx_ft_storage_fs:storage()) -> ok.
reset(Storage) ->
gen_server:cast(mk_server_ref(global), {reset, gc_interval(Storage)}).
collect(Storage, Transfer, Nodes) ->
gc_enabled(Storage) andalso cast_collect(mk_server_ref(global), Storage, Transfer, Nodes).
mk_server_ref(Name) ->
% TODO
{via, gproc, {n, l, {?MODULE, Name}}}.
%%
init(Storage) ->
St = #st{},
{ok, start_timer(gc_interval(Storage), St)}.
handle_call({collect, CalledAt}, _From, St) ->
StNext = maybe_collect_garbage(CalledAt, St),
{reply, StNext#st.last_gc, StNext};
handle_call(Call, From, St) ->
?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}),
{noreply, St}.
handle_cast({collect, Storage, Transfer, [Node | Rest]}, St) ->
ok = do_collect_transfer(Storage, Transfer, Node, St),
case Rest of
[_ | _] ->
cast_collect(self(), Storage, Transfer, Rest);
[] ->
ok
end,
{noreply, St};
handle_cast({reset, Interval}, St) ->
{noreply, start_timer(Interval, cancel_timer(St))};
handle_cast(Cast, St) ->
?SLOG(error, #{msg => "unexpected_cast", cast => Cast}),
{noreply, St}.
handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) ->
StNext = do_collect_garbage(St),
{noreply, start_timer(StNext#st{next_gc_timer = undefined})}.
do_collect_transfer(Storage, Transfer, Node, St = #st{}) when Node == node() ->
Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()),
ok = maybe_report(Stats, St),
ok;
do_collect_transfer(_Storage, _Transfer, _Node, _St = #st{}) ->
% TODO
ok.
cast_collect(Ref, Storage, Transfer, Nodes) ->
gen_server:cast(Ref, {collect, Storage, Transfer, Nodes}).
maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) ->
do_collect_garbage(St);
maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) ->
case FinishedAt > CalledAt of
true ->
St;
false ->
start_timer(do_collect_garbage(cancel_timer(St)))
end.
do_collect_garbage(St = #st{}) ->
emqx_ft_storage:with_storage_type(local, fun(Storage) ->
Stats = collect_garbage(Storage),
ok = maybe_report(Stats, Storage),
St#st{last_gc = Stats}
end).
maybe_report(#gcstats{errors = Errors}, Storage) when map_size(Errors) > 0 ->
?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage});
maybe_report(#gcstats{} = _Stats, _Storage) ->
?tp(garbage_collection, #{stats => _Stats, storage => _Storage}).
start_timer(St) ->
Interval = emqx_ft_storage:with_storage_type(local, fun gc_interval/1),
start_timer(Interval, St).
start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) ->
St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)};
start_timer(Interval, St) ->
?SLOG(warning, #{msg => "periodic_gc_disabled", interval => Interval}),
St.
cancel_timer(St = #st{next_gc_timer = undefined}) ->
St;
cancel_timer(St = #st{next_gc_timer = TRef}) ->
ok = emqx_utils:cancel_timer(TRef),
St#st{next_gc_timer = undefined}.
gc_enabled(Storage) ->
?IS_ENABLED(gc_interval(Storage)).
gc_interval(Storage) ->
emqx_ft_conf:gc_interval(Storage).
%%
collect_garbage(Storage) ->
Stats = init_gcstats(),
{ok, Transfers} = emqx_ft_storage_fs:transfers(Storage),
collect_garbage(Storage, Transfers, Stats).
collect_garbage(Storage, Transfers, Stats) ->
finish_gcstats(
maps:fold(
fun(Transfer, TransferInfo, StatsAcc) ->
% TODO: throttling?
try_collect_transfer(Storage, Transfer, TransferInfo, StatsAcc)
end,
Stats,
Transfers
)
).
try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) ->
% File transfer might still be incomplete.
% Any outdated fragments and temporary files should be collectable. As a kind of
% heuristic we only delete transfer directory itself only if it is also outdated
% _and was empty at the start of GC_, as a precaution against races between
% writers and GCs.
Cutoff =
case get_segments_ttl(Storage, TransferInfo) of
TTL when is_integer(TTL) ->
erlang:system_time(second) - TTL;
undefined ->
0
end,
{FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats),
{TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1),
% TODO: collect empty directories separately
case FragCleaned and TempCleaned of
true ->
collect_transfer_directory(Storage, Transfer, Cutoff, Stats2);
false ->
Stats2
end;
try_collect_transfer(Storage, Transfer, complete, Stats) ->
% File transfer is complete.
% We should be good to delete fragments and temporary files with their respective
% directories altogether.
{_, Stats1} = collect_fragments(Storage, Transfer, Stats),
{_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1),
Stats2.
collect_fragments(Storage, Transfer, Stats) ->
Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment),
maybe_collect_directory(Dirname, true, Stats).
collect_tempfiles(Storage, Transfer, Stats) ->
Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary),
maybe_collect_directory(Dirname, true, Stats).
collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) ->
Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment),
maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats).
collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) ->
Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary),
maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats).
collect_transfer_directory(Storage, Transfer, Cutoff, Stats) ->
Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer),
Filter =
case Stats of
#gcstats{directories = 0} ->
% Nothing were collected, this is a leftover from a past complete transfer GC.
filter_older_than(Cutoff);
#gcstats{} ->
% Usual incomplete transfer GC, collect directories unconditionally.
true
end,
case collect_empty_directory(Dirname, Filter, Stats) of
{true, StatsNext} ->
collect_parents(Dirname, get_segments_root(Storage), StatsNext);
{false, StatsNext} ->
StatsNext
end.
filter_older_than(Cutoff) ->
fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt =< Cutoff end.
collect_parents(Dirname, Until, Stats) ->
Parent = filename:dirname(Dirname),
case is_same_filepath(Parent, Until) orelse file:del_dir(Parent) of
true ->
Stats;
ok ->
?tp(garbage_collected_directory, #{path => Dirname}),
collect_parents(Parent, Until, account_gcstat_directory(Stats));
{error, eexist} ->
Stats;
{error, Reason} ->
register_gcstat_error({directory, Parent}, Reason, Stats)
end.
maybe_collect_directory(Dirpath, Filter, Stats) ->
case filelib:is_dir(Dirpath) of
true ->
collect_filepath(Dirpath, Filter, Stats);
false ->
{true, Stats}
end.
-spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when
Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()).
collect_filepath(Filepath, Filter, Stats) ->
case file:read_link_info(Filepath, [{time, posix}, raw]) of
{ok, Fileinfo} ->
collect_filepath(Filepath, Fileinfo, Filter, Stats);
{error, Reason} ->
{Reason == enoent, register_gcstat_error({path, Filepath}, Reason, Stats)}
end.
collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) ->
collect_directory(Filepath, Fileinfo, Filter, Stats);
collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) ->
case filter_filepath(Filter, Filepath, Fileinfo) andalso file:delete(Filepath, [raw]) of
false ->
{false, Stats};
ok ->
?tp(garbage_collected_file, #{path => Filepath}),
{true, account_gcstat(Fileinfo, Stats)};
{error, Reason} ->
{Reason == enoent, register_gcstat_error({file, Filepath}, Reason, Stats)}
end;
collect_filepath(Filepath, Fileinfo, _Filter, Stats) ->
{false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}.
collect_directory(Dirpath, Fileinfo, Filter, Stats) ->
case file:list_dir(Dirpath) of
{ok, Filenames} ->
{Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats),
case Clean of
true ->
collect_empty_directory(Dirpath, Fileinfo, Filter, StatsNext);
false ->
{false, StatsNext}
end;
{error, Reason} ->
{false, register_gcstat_error({directory, Dirpath}, Reason, Stats)}
end.
collect_files(Dirname, Filenames, Filter, Stats) ->
lists:foldl(
fun(Filename, {Complete, StatsAcc}) ->
Filepath = filename:join(Dirname, Filename),
{Collected, StatsNext} = collect_filepath(Filepath, Filter, StatsAcc),
{Collected andalso Complete, StatsNext}
end,
{true, Stats},
Filenames
).
collect_empty_directory(Dirpath, Filter, Stats) ->
case file:read_link_info(Dirpath, [{time, posix}, raw]) of
{ok, Dirinfo} ->
collect_empty_directory(Dirpath, Dirinfo, Filter, Stats);
{error, Reason} ->
{Reason == enoent, register_gcstat_error({directory, Dirpath}, Reason, Stats)}
end.
collect_empty_directory(Dirpath, Dirinfo, Filter, Stats) ->
case filter_filepath(Filter, Dirpath, Dirinfo) andalso file:del_dir(Dirpath) of
false ->
{false, Stats};
ok ->
?tp(garbage_collected_directory, #{path => Dirpath}),
{true, account_gcstat_directory(Stats)};
{error, Reason} ->
{false, register_gcstat_error({directory, Dirpath}, Reason, Stats)}
end.
filter_filepath(Filter, _, _) when is_boolean(Filter) ->
Filter;
filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) ->
Filter(Filepath, Fileinfo).
is_same_filepath(P1, P2) when is_binary(P1) andalso is_binary(P2) ->
filename:absname(P1) == filename:absname(P2);
is_same_filepath(P1, P2) when is_list(P1) andalso is_list(P2) ->
filename:absname(P1) == filename:absname(P2);
is_same_filepath(P1, P2) when is_binary(P1) ->
is_same_filepath(P1, filepath_to_binary(P2)).
filepath_to_binary(S) ->
unicode:characters_to_binary(S, unicode, file:native_name_encoding()).
get_segments_ttl(Storage, TransferInfo) ->
clamp(emqx_ft_conf:segments_ttl(Storage), try_get_filemeta_ttl(TransferInfo)).
try_get_filemeta_ttl(#{filemeta := Filemeta}) ->
maps:get(segments_ttl, Filemeta, undefined);
try_get_filemeta_ttl(#{}) ->
undefined.
clamp({Min, Max}, V) ->
min(Max, max(Min, V));
clamp(undefined, V) ->
V.
%%
init_gcstats() ->
#gcstats{started_at = erlang:system_time()}.
finish_gcstats(Stats) ->
Stats#gcstats{finished_at = erlang:system_time()}.
account_gcstat(Fileinfo, Stats = #gcstats{files = Files, space = Space}) ->
Stats#gcstats{
files = Files + 1,
space = Space + Fileinfo#file_info.size
}.
account_gcstat_directory(Stats = #gcstats{directories = Directories}) ->
Stats#gcstats{
directories = Directories + 1
}.
register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) ->
Stats#gcstats{errors = Errors#{Subject => Error}}.
%%
get_segments_root(Storage) ->
emqx_ft_storage_fs:get_root(Storage).

36
apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl Normal file
View File

@ -0,0 +1,36 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% This methods are called via rpc by `emqx_ft_storage_fs`
%% They populate the call with actual storage which may be configured differently
%% on a concrete node.
-module(emqx_ft_storage_fs_proxy).
-export([
list_local/2,
pread_local/4,
lookup_local_assembler/1
]).
list_local(Transfer, What) ->
emqx_ft_storage:with_storage_type(local, list, [Transfer, What]).
pread_local(Transfer, Frag, Offset, Size) ->
emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]).
lookup_local_assembler(Transfer) ->
emqx_ft_storage:with_storage_type(local, lookup_local_assembler, [Transfer]).

139
apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl Normal file
View File

@ -0,0 +1,139 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_reader).
-behaviour(gen_server).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
%% API
-export([
start_link/2,
start_supervised/2,
table/1,
table/2,
read/2
]).
%% gen_server callbacks
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-define(DEFAULT_CHUNK_SIZE, 1024).
-define(IS_FILENAME(Filename), (is_list(Filename) or is_binary(Filename))).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
-spec table(pid()) -> qlc:query_handle().
table(ReaderPid) when is_pid(ReaderPid) ->
table(ReaderPid, ?DEFAULT_CHUNK_SIZE).
-spec table(pid(), pos_integer()) -> qlc:query_handle().
table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso Bytes > 0 ->
NextFun = fun NextFun(Pid) ->
case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid, Bytes) of
eof ->
[];
{ok, Data} ->
[Data] ++ fun() -> NextFun(Pid) end;
{ErrorKind, Reason} when ErrorKind =:= badrpc; ErrorKind =:= error ->
?SLOG(warning, #{msg => "file_read_error", kind => ErrorKind, reason => Reason}),
[]
end
end,
qlc:table(fun() -> NextFun(ReaderPid) end, []).
-spec start_link(pid(), filename:filename()) -> startlink_ret().
start_link(CallerPid, Filename) when
is_pid(CallerPid) andalso
?IS_FILENAME(Filename)
->
gen_server:start_link(?MODULE, [CallerPid, Filename], []).
-spec start_supervised(pid(), filename:filename()) -> startlink_ret().
start_supervised(CallerPid, Filename) when
is_pid(CallerPid) andalso
?IS_FILENAME(Filename)
->
emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename).
-spec read(pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()}.
read(Pid, Bytes) when
is_pid(Pid) andalso
is_integer(Bytes) andalso
Bytes > 0
->
gen_server:call(Pid, {read, Bytes}).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([CallerPid, Filename]) ->
MRef = erlang:monitor(process, CallerPid),
case file:open(Filename, [read, raw, binary]) of
{ok, File} ->
{ok, #{
filename => Filename,
file => File,
caller_pid => CallerPid,
mref => MRef
}};
{error, Reason} ->
{stop, Reason}
end.
handle_call({read, Bytes}, _From, #{file := File} = State) ->
case file:read(File, Bytes) of
{ok, Data} ->
?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}),
{reply, {ok, Data}, State};
eof ->
?SLOG(debug, #{msg => "read", eof => true}),
{stop, normal, eof, State};
{error, Reason} = Error ->
{stop, Reason, Error, State}
end;
handle_call(Msg, _From, State) ->
{reply, {error, {bad_call, Msg}}, State}.
handle_info(
{'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State
) ->
{stop, {caller_down, CallerPid}, State};
handle_info(Msg, State) ->
?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}),
{noreply, State}.
handle_cast(Msg, State) ->
?SLOG(warning, #{msg => "unexpected_message", case_msg => Msg}),
{noreply, State}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.

49
apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl Normal file
View File

@ -0,0 +1,49 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_reader_sup).
-behaviour(supervisor).
-export([
init/1,
start_link/0,
start_child/2
]).
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
start_child(CallerPid, Filename) ->
Childspec = #{
id => {CallerPid, Filename},
start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]},
restart => temporary
},
case supervisor:start_child(?MODULE, Childspec) of
{ok, Pid} ->
{ok, Pid};
{error, {Reason, _Child}} ->
{error, Reason}
end.
init(_) ->
SupFlags = #{
strategy => one_for_one,
intensity => 10,
period => 1000
},
{ok, {SupFlags, []}}.

65
apps/emqx_ft/src/emqx_ft_sup.erl Normal file
View File

@ -0,0 +1,65 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_sup).
-behaviour(supervisor).
-export([start_link/0]).
-export([init/1]).
-define(SERVER, ?MODULE).
start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) ->
SupFlags = #{
strategy => one_for_one,
intensity => 100,
period => 10
},
AssemblerSup = #{
id => emqx_ft_assembler_sup,
start => {emqx_ft_assembler_sup, start_link, []},
restart => permanent,
shutdown => infinity,
type => supervisor,
modules => [emqx_ft_assembler_sup]
},
FileReaderSup = #{
id => emqx_ft_storage_fs_reader_sup,
start => {emqx_ft_storage_fs_reader_sup, start_link, []},
restart => permanent,
shutdown => infinity,
type => supervisor,
modules => [emqx_ft_storage_fs_reader_sup]
},
Responder = #{
id => emqx_ft_responder_sup,
start => {emqx_ft_responder_sup, start_link, []},
restart => permanent,
shutdown => infinity,
type => worker,
modules => [emqx_ft_responder_sup]
},
ChildSpecs = [Responder, AssemblerSup, FileReaderSup],
{ok, {SupFlags, ChildSpecs}}.

54
apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl Normal file
View File

@ -0,0 +1,54 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_exporter_fs_proto_v1).
-behaviour(emqx_bpapi).
-export([introduced_in/0]).
-export([list_exports/2]).
-export([read_export_file/3]).
-include_lib("emqx/include/bpapi.hrl").
introduced_in() ->
"5.0.17".
-spec list_exports([node()], emqx_ft_storage:query(_LocalCursor)) ->
emqx_rpc:erpc_multicall(
{ok, [emqx_ft_storage:file_info()]}
| {error, file:posix() | disabled | {invalid_storage_type, _}}
).
list_exports(Nodes, Query) ->
erpc:multicall(
Nodes,
emqx_ft_storage_exporter_fs_proxy,
list_exports_local,
[Query]
).
-spec read_export_file(node(), file:name(), pid()) ->
{ok, emqx_ft_storage:reader()}
| {error, term()}
| no_return().
read_export_file(Node, Filepath, CallerPid) ->
erpc:call(
Node,
emqx_ft_storage_exporter_fs_proxy,
read_export_file_local,
[Filepath, CallerPid]
).

49
apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl Normal file
View File

@ -0,0 +1,49 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_proto_v1).
-behaviour(emqx_bpapi).
-export([introduced_in/0]).
-export([multilist/3]).
-export([pread/5]).
-export([list_assemblers/2]).
-type offset() :: emqx_ft:offset().
-type transfer() :: emqx_ft:transfer().
-type filefrag() :: emqx_ft_storage_fs:filefrag().
-include_lib("emqx/include/bpapi.hrl").
introduced_in() ->
"5.0.17".
-spec multilist([node()], transfer(), fragment | result) ->
emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}).
multilist(Nodes, Transfer, What) ->
erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]).
-spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) ->
{ok, [filefrag()]} | {error, term()} | no_return().
pread(Node, Transfer, Frag, Offset, Size) ->
erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]).
-spec list_assemblers([node()], transfer()) ->
emqx_rpc:erpc_multicall([pid()]).
list_assemblers(Nodes, Transfer) ->
erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, lookup_local_assembler, [Transfer]).

35
apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl Normal file
View File

@ -0,0 +1,35 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_reader_proto_v1).
-behaviour(emqx_bpapi).
-export([introduced_in/0]).
-export([read/3]).
-include_lib("emqx/include/bpapi.hrl").
introduced_in() ->
"5.0.17".
-spec read(node(), pid(), pos_integer()) ->
{ok, binary()} | eof | {error, term()} | no_return().
read(Node, Pid, Bytes) when
is_atom(Node) andalso is_pid(Pid) andalso is_integer(Bytes) andalso Bytes > 0
->
emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid, Bytes]).

782
apps/emqx_ft/test/emqx_ft_SUITE.erl Normal file
View File

@ -0,0 +1,782 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-define(assertRCName(RCName, PublishRes),
?assertMatch(
{ok, #{reason_code_name := RCName}},
PublishRes
)
).
all() ->
[
{group, single_node},
{group, cluster}
].
groups() ->
[
{single_node, [parallel], [
t_assemble_crash,
t_corrupted_segment_retry,
t_invalid_checksum,
t_invalid_fileid,
t_invalid_filename,
t_invalid_meta,
t_invalid_topic_format,
t_meta_conflict,
t_nasty_clientids_fileids,
t_no_meta,
t_no_segment,
t_simple_transfer
]},
{cluster, [], [
t_switch_node,
t_unreliable_migrating_client,
{g_concurrent_fins, [{repeat_until_any_fail, 8}], [
t_concurrent_fins
]}
]}
].
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft]),
ok.
set_special_configs(Config) ->
fun
(emqx_ft) ->
% NOTE
% Inhibit local fs GC to simulate it isn't fast enough to collect
% complete transfers.
Storage = emqx_utils_maps:deep_merge(
emqx_ft_test_helpers:local_storage(Config),
#{<<"local">> => #{<<"segments">> => #{<<"gc">> => #{<<"interval">> => 0}}}}
),
emqx_ft_test_helpers:load_config(#{
<<"enable">> => true,
<<"storage">> => Storage
});
(_) ->
ok
end.
init_per_testcase(Case, Config) ->
ClientId = atom_to_binary(Case),
case ?config(group, Config) of
cluster ->
[{clientid, ClientId} | Config];
_ ->
{ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config]
end.
end_per_testcase(_Case, Config) ->
_ = [ok = emqtt:stop(C) || {client, C} <- Config],
ok.
init_per_group(Group = cluster, Config) ->
Cluster = mk_cluster_specs(Config),
ct:pal("Starting ~p", [Cluster]),
Nodes = [
emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()})
|| {Name, Opts} <- Cluster
],
[{group, Group}, {cluster_nodes, Nodes} | Config];
init_per_group(Group, Config) ->
[{group, Group} | Config].
end_per_group(cluster, Config) ->
ok = lists:foreach(
fun emqx_ft_test_helpers:stop_additional_node/1,
?config(cluster_nodes, Config)
);
end_per_group(_Group, _Config) ->
ok.
mk_cluster_specs(Config) ->
Specs = [
{core, emqx_ft_SUITE1, #{listener_ports => [{tcp, 2883}]}},
{core, emqx_ft_SUITE2, #{listener_ports => [{tcp, 3883}]}}
],
CommOpts = [
{env, [{emqx, boot_modules, [broker, listeners]}]},
{apps, [emqx_ft]},
{conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]},
{env_handler, set_special_configs(Config)}
],
emqx_common_test_helpers:emqx_cluster(
Specs,
CommOpts
).
%%--------------------------------------------------------------------
%% Tests
%%--------------------------------------------------------------------
t_invalid_topic_format(Config) ->
C = ?config(client, Config),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/fileid">>, <<>>, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/fileid/">>, <<>>, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/fileid/offset">>, <<>>, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/fileid/fin/offset">>, <<>>, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/">>, <<>>, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1)
),
%% should not be handled by `emqx_ft`
?assertRCName(
no_matching_subscribers,
emqtt:publish(C, <<"$file">>, <<>>, 1)
).
t_invalid_fileid(Config) ->
C = ?config(client, Config),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file//init">>, <<>>, 1)
).
t_invalid_filename(Config) ->
C = ?config(client, Config),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(<<"f1">>), encode_meta(meta(".", <<>>)), 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("..", <<>>)), 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("../nice", <<>>)), 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(
C,
mk_init_topic(<<"f4">>),
encode_meta(meta(lists:duplicate(1000, $A), <<>>)),
1
)
),
?assertRCName(
success,
emqtt:publish(C, mk_init_topic(<<"f5">>), encode_meta(meta("146%", <<>>)), 1)
).
t_simple_transfer(Config) ->
C = ?config(client, Config),
Filename = "topsecret.pdf",
FileId = <<"f1">>,
Data = [<<"first">>, <<"second">>, <<"third">>],
Meta = #{size := Filesize} = meta(Filename, Data),
?assertRCName(
success,
emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)
),
lists:foreach(
fun({Chunk, Offset}) ->
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1)
)
end,
with_offsets(Data)
),
?assertRCName(
success,
emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1)
),
[Export] = list_files(?config(clientid, Config)),
?assertEqual(
{ok, iolist_to_binary(Data)},
read_export(Export)
).
t_nasty_clientids_fileids(_Config) ->
Transfers = [
{<<".">>, <<".">>},
{<<"🌚"/utf8>>, <<"🌝"/utf8>>},
{<<"../..">>, <<"😤"/utf8>>},
{<<"/etc/passwd">>, <<"whitehat">>},
{<<"; rm -rf / ;">>, <<"whitehat">>}
],
ok = lists:foreach(
fun({ClientId, FileId}) ->
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId),
[Export] = list_files(ClientId),
?assertEqual({ok, ClientId}, read_export(Export))
end,
Transfers
).
t_meta_conflict(Config) ->
C = ?config(client, Config),
Filename = "topsecret.pdf",
FileId = <<"f1">>,
Meta = meta(Filename, [<<"x">>]),
?assertRCName(
success,
emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)
),
ConflictMeta = Meta#{name => "conflict.pdf"},
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(FileId), encode_meta(ConflictMeta), 1)
).
t_no_meta(Config) ->
C = ?config(client, Config),
FileId = <<"f1">>,
Data = <<"first">>,
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, 0), Data, 1)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_fin_topic(FileId, 42), <<>>, 1)
).
t_no_segment(Config) ->
C = ?config(client, Config),
Filename = "topsecret.pdf",
FileId = <<"f1">>,
Data = [<<"first">>, <<"second">>, <<"third">>],
Meta = #{size := Filesize} = meta(Filename, Data),
?assertRCName(
success,
emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)
),
lists:foreach(
fun({Chunk, Offset}) ->
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1)
)
end,
%% Skip the first segment
tl(with_offsets(Data))
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1)
).
t_invalid_meta(Config) ->
C = ?config(client, Config),
FileId = <<"f1">>,
%% Invalid schema
Meta = #{foo => <<"bar">>},
MetaPayload = emqx_utils_json:encode(Meta),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1)
),
%% Invalid JSON
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_init_topic(FileId), <<"{oops;">>, 1)
).
t_invalid_checksum(Config) ->
C = ?config(client, Config),
Filename = "topsecret.pdf",
FileId = <<"f1">>,
Data = [<<"first">>, <<"second">>, <<"third">>],
Meta = #{size := Filesize} = meta(Filename, Data),
MetaPayload = encode_meta(Meta#{checksum => {sha256, sha256(<<"invalid">>)}}),
?assertRCName(
success,
emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1)
),
lists:foreach(
fun({Chunk, Offset}) ->
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1)
)
end,
with_offsets(Data)
),
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1)
).
t_corrupted_segment_retry(Config) ->
C = ?config(client, Config),
Filename = "corruption.pdf",
FileId = <<"4242-4242">>,
Data = [<<"first">>, <<"second">>, <<"third">>],
[
{Seg1, Offset1},
{Seg2, Offset2},
{Seg3, Offset3}
] = with_offsets(Data),
[
Checksum1,
Checksum2,
Checksum3
] = [binary:encode_hex(sha256(S)) || S <- Data],
Meta = #{size := Filesize} = meta(Filename, Data),
?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)),
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset1, Checksum1), Seg1, 1)
),
% segment is corrupted
?assertRCName(
unspecified_error,
emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), <<Seg2/binary, 42>>, 1)
),
% retry
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), Seg2, 1)
),
?assertRCName(
success,
emqtt:publish(C, mk_segment_topic(FileId, Offset3, Checksum3), Seg3, 1)
),
?assertRCName(
success,
emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1)
).
t_switch_node(Config) ->
[Node | _] = ?config(cluster_nodes, Config),
AdditionalNodePort = emqx_ft_test_helpers:tcp_port(Node),
ClientId = <<"t_switch_node-migrating_client">>,
{ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]),
{ok, _} = emqtt:connect(C1),
Filename = "multinode_upload.txt",
FileId = <<"f1">>,
Data = [<<"first">>, <<"second">>, <<"third">>],
[{Data0, Offset0}, {Data1, Offset1}, {Data2, Offset2}] = with_offsets(Data),
%% First, publist metadata and the first segment to the additional node
Meta = #{size := Filesize} = meta(Filename, Data),
?assertRCName(
success,
emqtt:publish(C1, mk_init_topic(FileId), encode_meta(Meta), 1)
),
?assertRCName(
success,
emqtt:publish(C1, mk_segment_topic(FileId, Offset0), Data0, 1)
),
%% Then, switch the client to the main node
%% and publish the rest of the segments
ok = emqtt:stop(C1),
{ok, C2} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(C2),
?assertRCName(
success,
emqtt:publish(C2, mk_segment_topic(FileId, Offset1), Data1, 1)
),
?assertRCName(
success,
emqtt:publish(C2, mk_segment_topic(FileId, Offset2), Data2, 1)
),
?assertRCName(
success,
emqtt:publish(C2, mk_fin_topic(FileId, Filesize), <<>>, 1)
),
ok = emqtt:stop(C2),
%% Now check consistency of the file
[Export] = list_files(ClientId),
?assertEqual(
{ok, iolist_to_binary(Data)},
read_export(Export)
).
t_assemble_crash(Config) ->
C = ?config(client, Config),
meck:new(emqx_ft_storage_fs),
meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end),
?assertRCName(
unspecified_error,
emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1)
).
t_unreliable_migrating_client(Config) ->
NodeSelf = node(),
[Node1, Node2] = ?config(cluster_nodes, Config),
ClientId = ?config(clientid, Config),
FileId = emqx_guid:to_hexstr(emqx_guid:gen()),
Filename = "migratory-birds-in-southern-hemisphere-2013.pdf",
Filesize = 1000,
Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16),
Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)),
Meta = meta(Filename, Payload),
Context = #{
clientid => ClientId,
fileid => FileId,
filesize => Filesize,
payload => Payload
},
Commands = [
% Connect to the broker on the current node
{fun connect_mqtt_client/2, [NodeSelf]},
% Send filemeta and 3 initial segments
% (assuming client chose 100 bytes as a desired segment size)
{fun send_filemeta/2, [Meta]},
{fun send_segment/3, [0, 100]},
{fun send_segment/3, [100, 100]},
{fun send_segment/3, [200, 100]},
% Disconnect the client cleanly
{fun stop_mqtt_client/1, []},
% Connect to the broker on `Node1`
{fun connect_mqtt_client/2, [Node1]},
% Connect to the broker on `Node2` without first disconnecting from `Node1`
% Client forgot the state for some reason and started the transfer again.
% (assuming this is usual for a client on a device that was rebooted)
{fun connect_mqtt_client/2, [Node2]},
{fun send_filemeta/2, [Meta]},
% This time it chose 200 bytes as a segment size
{fun send_segment/3, [0, 200]},
{fun send_segment/3, [200, 200]},
% But now it downscaled back to 100 bytes segments
{fun send_segment/3, [400, 100]},
% Client lost connectivity and reconnected
% (also had last few segments unacked and decided to resend them)
{fun connect_mqtt_client/2, [Node2]},
{fun send_segment/3, [200, 200]},
{fun send_segment/3, [400, 200]},
% Client lost connectivity and reconnected, this time to another node
% (also had last segment unacked and decided to resend it)
{fun connect_mqtt_client/2, [Node1]},
{fun send_segment/3, [400, 200]},
{fun send_segment/3, [600, eof]},
{fun send_finish/1, []},
% Client lost connectivity and reconnected, this time to the current node
% (client had `fin` unacked and decided to resend it)
{fun connect_mqtt_client/2, [NodeSelf]},
{fun send_finish/1, []}
],
_Context = run_commands(Commands, Context),
Exports = list_files(?config(clientid, Config)),
Node1Str = atom_to_list(Node1),
% TODO: this testcase is specific to local fs storage backend
?assertMatch(
[#{"node" := Node1Str}],
fs_exported_file_attributes(Exports)
),
[
?assertEqual({ok, Payload}, read_export(Export))
|| Export <- Exports
].
t_concurrent_fins(Config) ->
ct:timetrap({seconds, 10}),
NodeSelf = node(),
[Node1, Node2] = ?config(cluster_nodes, Config),
ClientId = iolist_to_binary([
?config(clientid, Config),
integer_to_list(erlang:unique_integer())
]),
FileId = emqx_guid:to_hexstr(emqx_guid:gen()),
Filename = "migratory-birds-in-southern-hemisphere-2013.pdf",
Filesize = 100,
Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16),
Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)),
Meta = meta(Filename, Payload),
%% Send filemeta and segments to Node1
Context0 = #{
clientid => ClientId,
fileid => FileId,
filesize => Filesize,
payload => Payload
},
Context1 = run_commands(
[
{fun connect_mqtt_client/2, [Node1]},
{fun send_filemeta/2, [Meta]},
{fun send_segment/3, [0, 100]},
{fun stop_mqtt_client/1, []}
],
Context0
),
%% Now send fins concurrently to the 3 nodes
Nodes = [Node1, Node2, NodeSelf],
SendFin = fun(Node) ->
run_commands(
[
{fun connect_mqtt_client/2, [Node]},
{fun send_finish/1, []}
],
Context1
)
end,
PidMons = lists:map(
fun(Node) ->
erlang:spawn_monitor(fun F() ->
_ = erlang:process_flag(trap_exit, true),
try
SendFin(Node)
catch
C:E ->
% NOTE: random delay to avoid livelock conditions
ct:pal("Node ~p did not send finish successfully: ~p:~p", [Node, C, E]),
ok = timer:sleep(rand:uniform(10)),
F()
end
end)
end,
Nodes
),
ok = lists:foreach(
fun({Pid, MRef}) ->
receive
{'DOWN', MRef, process, Pid, normal} -> ok
end
end,
PidMons
),
%% Only one node should have the file
Exports = list_files(ClientId),
case fs_exported_file_attributes(Exports) of
[#{"node" := _Node}] ->
ok;
[#{"node" := _Node} | _] = Files ->
% ...But we can't really guarantee that
ct:comment({multiple_files_on_different_nodes, Files})
end.
%%------------------------------------------------------------------------------
%% Command helpers
%%------------------------------------------------------------------------------
%% Command runners
run_commands(Commands, Context) ->
lists:foldl(fun run_command/2, Context, Commands).
run_command({Command, Args}, Context) ->
ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]),
erlang:apply(Command, Args ++ [Context]).
%% Commands
connect_mqtt_client(Node, ContextIn) ->
Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn),
NodePort = emqx_ft_test_helpers:tcp_port(Node),
{ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, NodePort}]),
{ok, _} = emqtt:connect(Client),
Context#{client => Client}.
stop_mqtt_client(Context = #{client := Client}) ->
_ = emqtt:stop(Client),
maps:remove(client, Context).
disown_mqtt_client(Context = #{client := Client}) ->
_ = erlang:unlink(Client),
maps:remove(client, Context);
disown_mqtt_client(Context = #{}) ->
Context.
send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) ->
?assertRCName(
success,
emqtt:publish(Client, mk_init_topic(FileId), encode_meta(Meta), 1)
),
Context.
send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) ->
Data =
case Size of
eof ->
binary:part(Payload, Offset, byte_size(Payload) - Offset);
N ->
binary:part(Payload, Offset, N)
end,
?assertRCName(
success,
emqtt:publish(Client, mk_segment_topic(FileId, Offset), Data, 1)
),
Context.
send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) ->
?assertRCName(
success,
emqtt:publish(Client, mk_fin_topic(FileId, Filesize), <<>>, 1)
),
Context.
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
fs_exported_file_attributes(FSExports) ->
lists:map(
fun(#{uri := URIString}) ->
#{query := QS} = uri_string:parse(URIString),
maps:from_list(uri_string:dissect_query(QS))
end,
lists:sort(FSExports)
).
mk_init_topic(FileId) ->
<<"$file/", FileId/binary, "/init">>.
mk_segment_topic(FileId, Offset) when is_integer(Offset) ->
mk_segment_topic(FileId, integer_to_binary(Offset));
mk_segment_topic(FileId, Offset) when is_binary(Offset) ->
<<"$file/", FileId/binary, "/", Offset/binary>>.
mk_segment_topic(FileId, Offset, Checksum) when is_integer(Offset) ->
mk_segment_topic(FileId, integer_to_binary(Offset), Checksum);
mk_segment_topic(FileId, Offset, Checksum) when is_binary(Offset) ->
<<"$file/", FileId/binary, "/", Offset/binary, "/", Checksum/binary>>.
mk_fin_topic(FileId, Size) when is_integer(Size) ->
mk_fin_topic(FileId, integer_to_binary(Size));
mk_fin_topic(FileId, Size) when is_binary(Size) ->
<<"$file/", FileId/binary, "/fin/", Size/binary>>.
with_offsets(Items) ->
{List, _} = lists:mapfoldl(
fun(Item, Offset) ->
{{Item, integer_to_binary(Offset)}, Offset + byte_size(Item)}
end,
0,
Items
),
List.
sha256(Data) ->
crypto:hash(sha256, Data).
meta(FileName, Data) ->
FullData = iolist_to_binary(Data),
#{
name => FileName,
checksum => {sha256, sha256(FullData)},
expire_at => erlang:system_time(_Unit = second) + 3600,
size => byte_size(FullData)
}.
encode_meta(Meta) ->
emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)).
list_files(ClientId) ->
{ok, #{items := Files}} = emqx_ft_storage:files(),
[File || File = #{transfer := {CId, _}} <- Files, CId == ClientId].
read_export(#{path := AbsFilepath}) ->
% TODO: only works for the local filesystem exporter right now
file:read_file(AbsFilepath).

304
apps/emqx_ft/test/emqx_ft_api_SUITE.erl Normal file
View File

@ -0,0 +1,304 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]).
all() ->
[
{group, single},
{group, cluster}
].
groups() ->
[
{single, [], emqx_common_test_helpers:all(?MODULE)},
{cluster, [], emqx_common_test_helpers:all(?MODULE)}
].
init_per_suite(Config) ->
ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config)
),
{ok, _} = emqx:update_config([rpc, port_discovery], manual),
Config.
end_per_suite(_Config) ->
ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]),
ok.
init_per_group(Group = cluster, Config) ->
Cluster = mk_cluster_specs(Config),
ct:pal("Starting ~p", [Cluster]),
Nodes = [
emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()})
|| {Name, Opts} <- Cluster
],
[{group, Group}, {cluster_nodes, Nodes} | Config];
init_per_group(Group, Config) ->
[{group, Group} | Config].
end_per_group(cluster, Config) ->
ok = lists:foreach(
fun emqx_ft_test_helpers:stop_additional_node/1,
?config(cluster_nodes, Config)
);
end_per_group(_Group, _Config) ->
ok.
mk_cluster_specs(Config) ->
Specs = [
{core, emqx_ft_api_SUITE1, #{listener_ports => [{tcp, 2883}]}},
{core, emqx_ft_api_SUITE2, #{listener_ports => [{tcp, 3883}]}}
],
CommOpts = [
{env, [{emqx, boot_modules, [broker, listeners]}]},
{apps, [emqx_ft]},
{conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]},
{env_handler, emqx_ft_test_helpers:env_handler(Config)}
],
emqx_common_test_helpers:emqx_cluster(
Specs,
CommOpts
).
init_per_testcase(Case, Config) ->
[{tc, Case} | Config].
end_per_testcase(t_ft_disabled, _Config) ->
emqx_config:put([file_transfer, enable], true);
end_per_testcase(_Case, _Config) ->
ok.
%%--------------------------------------------------------------------
%% Tests
%%--------------------------------------------------------------------
t_list_files(Config) ->
ClientId = client_id(Config),
FileId = <<"f1">>,
Node = lists:last(cluster(Config)),
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node),
{ok, 200, #{<<"files">> := Files}} =
request_json(get, uri(["file_transfer", "files"])),
?assertMatch(
[#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}],
[File || File = #{<<"clientid">> := CId} <- Files, CId == ClientId]
),
{ok, 200, #{<<"files">> := FilesTransfer}} =
request_json(get, uri(["file_transfer", "files", ClientId, FileId])),
?assertMatch(
[#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}],
FilesTransfer
),
?assertMatch(
{ok, 404, #{<<"code">> := <<"FILES_NOT_FOUND">>}},
request_json(get, uri(["file_transfer", "files", ClientId, <<"no-such-file">>]))
).
t_download_transfer(Config) ->
ClientId = client_id(Config),
FileId = <<"f1">>,
Node = lists:last(cluster(Config)),
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node),
?assertMatch(
{ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}},
request_json(
get,
uri(["file_transfer", "file"]) ++ query(#{fileref => FileId})
)
),
?assertMatch(
{ok, 503, _},
request(
get,
uri(["file_transfer", "file"]) ++
query(#{
fileref => FileId,
node => <<"nonode@nohost">>
})
)
),
?assertMatch(
{ok, 404, _},
request(
get,
uri(["file_transfer", "file"]) ++
query(#{
fileref => <<"unknown_file">>,
node => node()
})
)
),
{ok, 200, #{<<"files">> := [File]}} =
request_json(get, uri(["file_transfer", "files", ClientId, FileId])),
{ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)),
?assertEqual(
<<"data">>,
Response
).
t_list_files_paging(Config) ->
ClientId = client_id(Config),
NFiles = 20,
Nodes = cluster(Config),
Uploads = [
{mk_file_id("file:", N), mk_file_name(N), pick(N, Nodes)}
|| N <- lists:seq(1, NFiles)
],
ok = lists:foreach(
fun({FileId, Name, Node}) ->
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>, Node)
end,
Uploads
),
?assertMatch(
{ok, 200, #{<<"files">> := [_, _, _], <<"cursor">> := _}},
request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 3}))
),
{ok, 200, #{<<"files">> := Files}} =
request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})),
?assert(length(Files) >= NFiles),
?assertNotMatch(
{ok, 200, #{<<"cursor">> := _}},
request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100}))
),
?assertMatch(
{ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}},
request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 0}))
),
?assertMatch(
{ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}},
request_json(
get,
uri(["file_transfer", "files"]) ++ query(#{following => <<"whatsthat!?">>})
)
),
PageThrough = fun PageThrough(Query, Acc) ->
case request_json(get, uri(["file_transfer", "files"]) ++ query(Query)) of
{ok, 200, #{<<"files">> := FilesPage, <<"cursor">> := Cursor}} ->
PageThrough(Query#{following => Cursor}, Acc ++ FilesPage);
{ok, 200, #{<<"files">> := FilesPage}} ->
Acc ++ FilesPage
end
end,
?assertEqual(Files, PageThrough(#{limit => 1}, [])),
?assertEqual(Files, PageThrough(#{limit => 8}, [])),
?assertEqual(Files, PageThrough(#{limit => NFiles}, [])).
t_ft_disabled(_Config) ->
?assertMatch(
{ok, 200, _},
request_json(get, uri(["file_transfer", "files"]))
),
?assertMatch(
{ok, 400, _},
request_json(
get,
uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>})
)
),
ok = emqx_config:put([file_transfer, enable], false),
?assertMatch(
{ok, 503, _},
request_json(get, uri(["file_transfer", "files"]))
),
?assertMatch(
{ok, 503, _},
request_json(
get,
uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>, node => node()})
)
).
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
cluster(Config) ->
[node() | proplists:get_value(cluster_nodes, Config, [])].
client_id(Config) ->
iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])).
mk_file_id(Prefix, N) ->
iolist_to_binary([Prefix, integer_to_list(N)]).
mk_file_name(N) ->
"file." ++ integer_to_list(N).
request(Method, Url) ->
emqx_mgmt_api_test_util:request(Method, Url, []).
request_json(Method, Url) ->
case emqx_mgmt_api_test_util:request(Method, Url, []) of
{ok, Code, Body} ->
{ok, Code, json(Body)};
Otherwise ->
Otherwise
end.
json(Body) when is_binary(Body) ->
emqx_utils_json:decode(Body, [return_maps]).
query(Params) ->
KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)),
"?" ++ string:join(KVs, "&").
uri_encode(T) ->
emqx_http_lib:uri_encode(to_list(T)).
to_list(A) when is_atom(A) ->
atom_to_list(A);
to_list(A) when is_integer(A) ->
integer_to_list(A);
to_list(B) when is_binary(B) ->
binary_to_list(B);
to_list(L) when is_list(L) ->
L.
pick(N, List) ->
lists:nth(1 + (N rem length(List)), List).

265
apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl Normal file
View File

@ -0,0 +1,265 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_assembler_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("kernel/include/file.hrl").
all() ->
[
t_assemble_empty_transfer,
t_assemble_complete_local_transfer,
t_assemble_incomplete_transfer,
t_assemble_no_meta,
% NOTE
% It depends on the side effects of all previous testcases.
t_list_transfers
].
init_per_suite(Config) ->
Apps = application:ensure_all_started(gproc),
[{suite_apps, Apps} | Config].
end_per_suite(_Config) ->
ok.
init_per_testcase(TC, Config) ->
ok = snabbkaffe:start_trace(),
{ok, Pid} = emqx_ft_assembler_sup:start_link(),
[
{storage_root, <<"file_transfer_root">>},
{exports_root, <<"file_transfer_exports">>},
{file_id, atom_to_binary(TC)},
{assembler_sup, Pid}
| Config
].
end_per_testcase(_TC, Config) ->
ok = inspect_storage_root(Config),
ok = gen:stop(?config(assembler_sup, Config)),
ok = snabbkaffe:stop(),
ok.
%%
-define(CLIENTID1, <<"thatsme">>).
-define(CLIENTID2, <<"thatsnotme">>).
t_assemble_empty_transfer(Config) ->
Storage = storage(Config),
Transfer = {?CLIENTID1, ?config(file_id, Config)},
Filename = "important.pdf",
Meta = #{
name => Filename,
size => 0,
expire_at => 42
},
ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta),
?assertMatch(
{ok, [
#{
path := _,
timestamp := {{_, _, _}, {_, _, _}},
fragment := {filemeta, Meta}
}
]},
emqx_ft_storage_fs:list(Storage, Transfer, fragment)
),
Status = complete_assemble(Storage, Transfer, 0),
?assertEqual({shutdown, ok}, Status),
{ok, [_Result = #{size := _Size = 0}]} = list_exports(Config, Transfer),
% ?assertEqual(
% {error, eof},
% emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size)
% ),
ok.
t_assemble_complete_local_transfer(Config) ->
Storage = storage(Config),
Transfer = {?CLIENTID2, ?config(file_id, Config)},
Filename = "topsecret.pdf",
TransferSize = 10000 + rand:uniform(50000),
SegmentSize = 4096,
Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize),
Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)),
Meta = #{
name => Filename,
checksum => {sha256, Hash},
expire_at => 42
},
ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta),
_ = emqx_ft_content_gen:consume(
Gen,
fun({Content, SegmentNum, _Meta}) ->
Offset = (SegmentNum - 1) * SegmentSize,
?assertEqual(
ok,
emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content})
)
end
),
{ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment),
?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)),
?assertEqual(
[Meta],
[FM || #{fragment := {filemeta, FM}} <- Fragments],
Fragments
),
Status = complete_assemble(Storage, Transfer, TransferSize),
?assertEqual({shutdown, ok}, Status),
?assertMatch(
{ok, [
#{
size := TransferSize,
meta := #{}
}
]},
list_exports(Config, Transfer)
),
{ok, [#{path := AssemblyFilename}]} = list_exports(Config, Transfer),
?assertMatch(
{ok, #file_info{type = regular, size = TransferSize}},
file:read_file_info(AssemblyFilename)
),
ok = emqx_ft_content_gen:check_file_consistency(
{Transfer, TransferSize},
100,
AssemblyFilename
).
t_assemble_incomplete_transfer(Config) ->
Storage = storage(Config),
Transfer = {?CLIENTID2, ?config(file_id, Config)},
Filename = "incomplete.pdf",
TransferSize = 10000 + rand:uniform(50000),
SegmentSize = 4096,
Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize),
Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)),
Meta = #{
name => Filename,
checksum => {sha256, Hash},
size => TransferSize,
expire_at => 42
},
ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta),
Status = complete_assemble(Storage, Transfer, TransferSize),
?assertMatch({shutdown, {error, _}}, Status).
t_assemble_no_meta(Config) ->
Storage = storage(Config),
Transfer = {?CLIENTID2, ?config(file_id, Config)},
Status = complete_assemble(Storage, Transfer, 42),
?assertMatch({shutdown, {error, {incomplete, _}}}, Status).
complete_assemble(Storage, Transfer, Size) ->
complete_assemble(Storage, Transfer, Size, 1000).
complete_assemble(Storage, Transfer, Size, Timeout) ->
{async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size),
MRef = erlang:monitor(process, Pid),
Pid ! kickoff,
receive
{'DOWN', MRef, process, Pid, Result} ->
Result
after Timeout ->
ct:fail("Assembler did not finish in time")
end.
%%
t_list_transfers(Config) ->
{ok, Exports} = list_exports(Config),
?assertMatch(
[
#{
transfer := {?CLIENTID2, <<"t_assemble_complete_local_transfer">>},
path := _,
size := Size,
meta := #{name := "topsecret.pdf"}
},
#{
transfer := {?CLIENTID1, <<"t_assemble_empty_transfer">>},
path := _,
size := 0,
meta := #{name := "important.pdf"}
}
] when Size > 0,
lists:sort(Exports)
).
%%
-include_lib("kernel/include/file.hrl").
inspect_storage_root(Config) ->
inspect_dir(?config(storage_root, Config)).
inspect_dir(Dir) ->
FileInfos = filelib:fold_files(
Dir,
".*",
true,
fun(Filename, Acc) -> orddict:store(Filename, inspect_file(Filename), Acc) end,
orddict:new()
),
ct:pal("inspect '~s': ~p", [Dir, FileInfos]).
inspect_file(Filename) ->
{ok, Info} = file:read_file_info(Filename),
{Info#file_info.type, Info#file_info.size, Info#file_info.mtime}.
mk_fileid() ->
integer_to_binary(erlang:system_time(millisecond)).
list_exports(Config) ->
{emqx_ft_storage_exporter_fs, Options} = exporter(Config),
emqx_ft_storage_exporter_fs:list_local(Options).
list_exports(Config, Transfer) ->
{emqx_ft_storage_exporter_fs, Options} = exporter(Config),
emqx_ft_storage_exporter_fs:list_local_transfer(Options, Transfer).
exporter(Config) ->
emqx_ft_storage_exporter:exporter(storage(Config)).
storage(Config) ->
emqx_utils_maps:deep_get(
[storage, local],
emqx_ft_schema:translate(#{
<<"storage">> => #{
<<"local">> => #{
<<"segments">> => #{
<<"root">> => ?config(storage_root, Config)
},
<<"exporter">> => #{
<<"local">> => #{
<<"root">> => ?config(exports_root, Config)
}
}
}
}
})
).

249
apps/emqx_ft/test/emqx_ft_conf_SUITE.erl Normal file
View File

@ -0,0 +1,249 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_conf_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("snabbkaffe/include/test_macros.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
ok.
init_per_testcase(_Case, Config) ->
_ = emqx_config:save_schema_mod_and_names(emqx_ft_schema),
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_ft], fun
(emqx_ft) ->
emqx_ft_test_helpers:load_config(#{});
(_) ->
ok
end
),
{ok, _} = emqx:update_config([rpc, port_discovery], manual),
Config.
end_per_testcase(_Case, _Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]),
ok = emqx_config:erase(file_transfer).
%%--------------------------------------------------------------------
%% Tests
%%--------------------------------------------------------------------
t_update_config(_Config) ->
?assertMatch(
{error, #{kind := validation_error}},
emqx_conf:update(
[file_transfer],
#{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}},
#{}
)
),
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer],
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"segments">> => #{
<<"root">> => <<"/tmp/path">>,
<<"gc">> => #{
<<"interval">> => <<"5m">>
}
},
<<"exporter">> => #{
<<"local">> => #{
<<"root">> => <<"/tmp/exports">>
}
}
}
}
},
#{}
)
),
?assertEqual(
<<"/tmp/path">>,
emqx_config:get([file_transfer, storage, local, segments, root])
),
?assertEqual(
5 * 60 * 1000,
emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1)
),
?assertEqual(
{5 * 60, 24 * 60 * 60},
emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:segments_ttl/1)
).
t_disable_restore_config(Config) ->
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer],
#{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}},
#{}
)
),
?assertEqual(
60 * 60 * 1000,
emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1)
),
% Verify that transfers work
ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <<?MODULE_STRING>>),
% Verify that clearing storage settings reverts config to defaults
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer],
#{<<"enable">> => false, <<"storage">> => undefined},
#{}
)
),
?assertEqual(
false,
emqx_ft_conf:enabled()
),
?assertMatch(
#{local := #{exporter := #{local := _}}},
emqx_ft_conf:storage()
),
ClientId = gen_clientid(),
Client = emqx_ft_test_helpers:start_client(ClientId),
% Verify that transfers fail cleanly when storage is disabled
?check_trace(
?assertMatch(
{ok, #{reason_code_name := no_matching_subscribers}},
emqtt:publish(
Client,
<<"$file/f2/init">>,
emqx_utils_json:encode(emqx_ft:encode_filemeta(#{name => "f2", size => 42})),
1
)
),
fun(Trace) ->
?assertMatch([], ?of_kind("file_transfer_init", Trace))
end
),
ok = emqtt:stop(Client),
% Restore local storage backend
Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])),
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer],
#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"segments">> => #{
<<"root">> => Root,
<<"gc">> => #{<<"interval">> => <<"1s">>}
}
}
}
},
#{}
)
),
% Verify that GC is getting triggered eventually
?check_trace(
?block_until(#{?snk_kind := garbage_collection}, 5000, 0),
fun(Trace) ->
?assertMatch(
[
#{
?snk_kind := garbage_collection,
storage := #{segments := #{root := Root}}
}
],
?of_kind(garbage_collection, Trace)
)
end
),
% Verify that transfers work again
ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <<?MODULE_STRING>>).
t_switch_exporter(_Config) ->
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer],
#{<<"enable">> => true},
#{}
)
),
?assertMatch(
#{local := #{exporter := #{local := _}}},
emqx_ft_conf:storage()
),
% Verify that switching to a different exporter works
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer, storage, local, exporter],
#{
<<"s3">> => #{
<<"bucket">> => <<"emqx">>,
<<"host">> => <<"https://localhost">>,
<<"port">> => 9000,
<<"transport_options">> => #{
<<"ipv6_probe">> => false
}
}
},
#{}
)
),
?assertMatch(
#{local := #{exporter := #{s3 := _}}},
emqx_ft_conf:storage()
),
% Verify that switching back to local exporter works
?assertMatch(
{ok, _},
emqx_conf:remove(
[file_transfer, storage, local, exporter],
#{}
)
),
?assertMatch(
{ok, _},
emqx_conf:update(
[file_transfer, storage, local, exporter],
#{<<"local">> => #{}},
#{}
)
),
?assertMatch(
#{local := #{exporter := #{local := #{}}}},
emqx_ft_conf:storage()
),
% Verify that transfers work
ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <<?MODULE_STRING>>).
gen_clientid() ->
emqx_base62:encode(emqx_guid:gen()).

232
apps/emqx_ft/test/emqx_ft_content_gen.erl Normal file
View File

@ -0,0 +1,232 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Inspired by
%% https://github.com/kafka4beam/kflow/blob/master/src/testbed/payload_gen.erl
-module(emqx_ft_content_gen).
-include_lib("eunit/include/eunit.hrl").
-dialyzer(no_improper_lists).
-export([new/2]).
-export([generate/3]).
-export([next/1]).
-export([consume/1]).
-export([consume/2]).
-export([fold/3]).
-export([hash/2]).
-export([check_file_consistency/3]).
-export_type([cont/1]).
-export_type([stream/1]).
-export_type([binary_payload/0]).
-define(hash_size, 16).
-type payload() :: {Seed :: term(), Size :: integer()}.
-type binary_payload() :: {
binary(), _ChunkNum :: non_neg_integer(), _Meta :: #{}
}.
-type cont(Data) ::
fun(() -> stream(Data))
| stream(Data).
-type stream(Data) ::
maybe_improper_list(Data, cont(Data))
| eos.
-record(chunk_state, {
seed :: term(),
payload_size :: non_neg_integer(),
offset :: non_neg_integer(),
chunk_size :: non_neg_integer()
}).
-type chunk_state() :: #chunk_state{}.
%% -----------------------------------------------------------------------------
%% Generic streams
%% -----------------------------------------------------------------------------
%% @doc Consume one element from the stream.
-spec next(cont(A)) -> stream(A).
next(Fun) when is_function(Fun, 0) ->
Fun();
next(L) ->
L.
%% @doc Consume all elements of the stream and feed them into a
%% callback (e.g. brod:produce)
-spec consume(cont(A), fun((A) -> Ret)) -> [Ret].
consume([Data | Cont], Callback) ->
[Callback(Data) | consume(next(Cont), Callback)];
consume(Cont, Callback) when is_function(Cont, 0) ->
consume(next(Cont), Callback);
consume(eos, _Callback) ->
[].
%% @equiv consume(Stream, fun(A) -> A end)
-spec consume(cont(A)) -> [A].
consume(Stream) ->
consume(Stream, fun(A) -> A end).
-spec fold(fun((A, Acc) -> Acc), Acc, cont(A)) -> Acc.
fold(Fun, Acc, [Data | Cont]) ->
fold(Fun, Fun(Data, Acc), next(Cont));
fold(Fun, Acc, Cont) when is_function(Cont, 0) ->
fold(Fun, Acc, next(Cont));
fold(_Fun, Acc, eos) ->
Acc.
%% -----------------------------------------------------------------------------
%% Binary streams
%% -----------------------------------------------------------------------------
%% @doc Stream of binary chunks.
%% Limitation: `ChunkSize' should be dividable by `?hash_size'
-spec new(payload(), integer()) -> cont(binary_payload()).
new({Seed, Size}, ChunkSize) when ChunkSize rem ?hash_size =:= 0 ->
fun() ->
generate_next_chunk(#chunk_state{
seed = Seed,
payload_size = Size,
chunk_size = ChunkSize,
offset = 0
})
end.
%% @doc Generate chunks of data and feed them into
%% `Callback'
-spec generate(payload(), integer(), fun((binary_payload()) -> A)) -> [A].
generate(Payload, ChunkSize, Callback) ->
consume(new(Payload, ChunkSize), Callback).
-spec hash(cont(binary_payload()), crypto:hash_state()) -> binary().
hash(Stream, HashCtxIn) ->
crypto:hash_final(
fold(
fun({Chunk, _, _}, HashCtx) ->
crypto:hash_update(HashCtx, Chunk)
end,
HashCtxIn,
Stream
)
).
-spec check_consistency(
payload(),
integer(),
fun((integer()) -> {ok, binary()} | undefined)
) -> ok.
check_consistency({Seed, Size}, SampleSize, Callback) ->
SeedHash = seed_hash(Seed),
Random = [rand:uniform(Size) - 1 || _ <- lists:seq(1, SampleSize)],
%% Always check first and last bytes, and one that should not exist:
Samples = [0, Size - 1, Size | Random],
lists:foreach(
fun
(N) when N < Size ->
Expected = do_get_byte(N, SeedHash),
?assertEqual(
{N, {ok, Expected}},
{N, Callback(N)}
);
(N) ->
?assertMatch(undefined, Callback(N))
end,
Samples
).
-spec check_file_consistency(
payload(),
integer(),
file:filename()
) -> ok.
check_file_consistency(Payload, SampleSize, FileName) ->
{ok, FD} = file:open(FileName, [read, raw]),
try
Fun = fun(N) ->
case file:pread(FD, [{N, 1}]) of
{ok, [[X]]} -> {ok, X};
{ok, [eof]} -> undefined
end
end,
check_consistency(Payload, SampleSize, Fun)
after
file:close(FD)
end.
%% =============================================================================
%% Internal functions
%% =============================================================================
%% @doc Continue generating chunks
-spec generate_next_chunk(chunk_state()) -> stream(binary()).
generate_next_chunk(#chunk_state{offset = Offset, payload_size = Size}) when Offset >= Size ->
eos;
generate_next_chunk(State0 = #chunk_state{offset = Offset, chunk_size = ChunkSize}) ->
State = State0#chunk_state{offset = Offset + ChunkSize},
Payload = generate_chunk(
State#chunk_state.seed,
Offset,
ChunkSize,
State#chunk_state.payload_size
),
[Payload | fun() -> generate_next_chunk(State) end].
generate_chunk(Seed, Offset, ChunkSize, Size) ->
SeedHash = seed_hash(Seed),
To = min(Offset + ChunkSize, Size) - 1,
Payload = iolist_to_binary([
generator_fun(I, SeedHash)
|| I <- lists:seq(Offset div 16, To div 16)
]),
ChunkNum = Offset div ChunkSize + 1,
Meta = #{
chunk_size => ChunkSize,
chunk_count => ceil(Size / ChunkSize)
},
Chunk =
case Offset + ChunkSize of
NextOffset when NextOffset > Size ->
binary:part(Payload, 0, Size rem ChunkSize);
_ ->
Payload
end,
{Chunk, ChunkNum, Meta}.
%% @doc First argument is a chunk number, the second one is a seed.
%% This implementation is hardly efficient, but it was chosen for
%% clarity reasons
-spec generator_fun(integer(), binary()) -> binary().
generator_fun(N, Seed) ->
crypto:hash(md5, <<N:32, Seed/binary>>).
%% @doc Hash any term
-spec seed_hash(term()) -> binary().
seed_hash(Seed) ->
crypto:hash(md5, term_to_binary(Seed)).
%% @private Get byte at offset `N'
-spec do_get_byte(integer(), binary()) -> byte().
do_get_byte(N, Seed) ->
Chunk = generator_fun(N div ?hash_size, Seed),
binary:at(Chunk, N rem ?hash_size).

250
apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl Normal file
View File

@ -0,0 +1,250 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_fs_util_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("kernel/include/file.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
t_fold_single_level(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{"a", #file_info{type = directory}, ["a"]},
{"c", #file_info{type = directory}, ["c"]},
{"d", #file_info{type = directory}, ["d"]}
],
sort(fold(fun cons/4, [], Root, ['*']))
).
t_fold_multi_level(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]},
{"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]},
{"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
],
sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*']))
),
?assertMatch(
[
{"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]},
{"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]},
{"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]}
],
sort(fold(fun cons/4, [], Root, ['*', '*', '*']))
).
t_fold_no_glob(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[{"", #file_info{type = directory}, []}],
sort(fold(fun cons/4, [], Root, []))
).
t_fold_glob_too_deep(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[],
sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*']))
).
t_fold_invalid_root(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[],
sort(fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*']))
),
?assertMatch(
[],
sort(fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*']))
).
t_fold_filter_unicode(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]},
{"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
],
sort(fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1]))
),
?assertMatch(
[
{"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}
],
sort(fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)]))
).
t_fold_filter_levels(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]},
{"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]}
],
sort(fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*']))
).
t_fold_errors(Config) ->
Root = ?config(data_dir, Config),
ok = meck:new(emqx_ft_fs_util, [passthrough]),
ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) ->
ct:pal("read_info(~p)", [AbsFilepath]),
Filename = filename:basename(AbsFilepath),
case Filename of
"b" -> {error, eacces};
"link" -> {error, enotsup};
"bar" -> {error, enotdir};
"needle" -> {error, ebusy};
_ -> meck:passthrough([AbsFilepath])
end
end),
?assertMatch(
[
{"a/b", {error, eacces}, ["b", "a"]},
{"a/link", {error, enotsup}, ["link", "a"]},
{"c/link", {error, enotsup}, ["link", "c"]},
{"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]}
],
sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*']))
).
t_seek_fold(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{leaf, "a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]},
{leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]},
{leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
| _Nodes
],
sort(
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:seek(["a", "a"], Root, ['*', '*', '*', '*'])
)
)
),
?assertMatch(
[
{leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]},
{leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
| _Nodes
],
sort(
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:seek(["a", "b", "foo", "42"], Root, ['*', '*', '*', '*'])
)
)
),
?assertMatch(
[
{leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
| _Nodes
],
sort(
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:seek(["c", "d", "e", "f"], Root, ['*', '*', '*', '*'])
)
)
).
t_seek_empty(Config) ->
Root = ?config(data_dir, Config),
?assertEqual(
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:new(Root, ['*', '*', '*', '*'])
),
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:seek([], Root, ['*', '*', '*', '*'])
)
).
t_seek_past_end(Config) ->
Root = ?config(data_dir, Config),
?assertEqual(
none,
emqx_ft_fs_iterator:next(
emqx_ft_fs_iterator:seek(["g", "h"], Root, ['*', '*', '*', '*'])
)
).
t_seek_with_filter(Config) ->
Root = ?config(data_dir, Config),
?assertMatch(
[
{leaf, "d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]}
| _Nodes
],
sort(
emqx_ft_fs_iterator:fold(
fun cons/2,
[],
emqx_ft_fs_iterator:seek(["a", "link"], Root, ['*', fun is_letter/1, '*'])
)
)
).
%%
fold(FoldFun, Acc, Root, Glob) ->
emqx_ft_fs_util:fold(FoldFun, Acc, Root, Glob).
is_not(F) ->
fun(X) -> not F(X) end.
is_latin1(Filename) ->
case unicode:characters_to_binary(Filename, unicode, latin1) of
{error, _, _} ->
false;
_ ->
true
end.
is_letter(Filename) ->
case Filename of
[_] ->
true;
_ ->
false
end.
cons(Path, Info, Stack, Acc) ->
[{Path, Info, Stack} | Acc].
cons(Entry, Acc) ->
[Entry | Acc].
sort(L) when is_list(L) ->
lists:sort(L).

0
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 Normal file
View File

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я Normal file
View File

@ -0,0 +1 @@
Ты

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link Symbolic link
View File

@ -0,0 +1 @@
../c

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 Normal file
View File

@ -0,0 +1 @@
Zhōngwén

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link Symbolic link
View File

@ -0,0 +1 @@
../a

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle Normal file
View File

@ -0,0 +1 @@
haystack

1
apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack Normal file
View File

@ -0,0 +1 @@
needle

65
apps/emqx_ft/test/emqx_ft_fs_util_tests.erl Normal file
View File

@ -0,0 +1,65 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_fs_util_tests).
-include_lib("eunit/include/eunit.hrl").
filename_safe_test_() ->
[
?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe("im.safe")),
?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"im.safe">>)),
?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<".safe.100%">>)),
?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"safe.as.🦺"/utf8>>))
].
filename_unsafe_test_() ->
[
?_assertEqual({error, empty}, emqx_ft_fs_util:is_filename_safe("")),
?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(".")),
?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe("..")),
?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(<<"..">>)),
?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe(<<".././..">>)),
?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("/etc/passwd")),
?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("../cookie")),
?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("C:$cookie")),
?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([1, 2, 3])),
?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe(<<4, 5, 6>>)),
?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([$a, 16#7F, $z]))
].
-define(NAMES, [
{"just.file", <<"just.file">>},
{".hidden", <<".hidden">>},
{".~what", <<".~what">>},
{"100%25.file", <<"100%.file">>},
{"%2E%2E", <<"..">>},
{"...", <<"...">>},
{"%2Fetc%2Fpasswd", <<"/etc/passwd">>},
{"%01%02%0A ", <<1, 2, 10, 32>>}
]).
escape_filename_test_() ->
[
?_assertEqual(Filename, emqx_ft_fs_util:escape_filename(Input))
|| {Filename, Input} <- ?NAMES
].
unescape_filename_test_() ->
[
?_assertEqual(Input, emqx_ft_fs_util:unescape_filename(Filename))
|| {Filename, Input} <- ?NAMES
].

84
apps/emqx_ft/test/emqx_ft_responder_SUITE.erl Normal file
View File

@ -0,0 +1,84 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_responder_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("stdlib/include/assert.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft]),
ok.
init_per_testcase(_Case, Config) ->
Config.
end_per_testcase(_Case, _Config) ->
ok.
t_start_ack(_Config) ->
Key = <<"test">>,
DefaultAction = fun({ack, Ref}) -> Ref end,
?assertMatch(
{ok, _Pid},
emqx_ft_responder:start(Key, DefaultAction, 1000)
),
?assertMatch(
{error, {already_started, _Pid}},
emqx_ft_responder:start(Key, DefaultAction, 1000)
),
Ref = make_ref(),
?assertEqual(
Ref,
emqx_ft_responder:ack(Key, Ref)
),
?assertExit(
{noproc, _},
emqx_ft_responder:ack(Key, Ref)
).
t_timeout(_Config) ->
Key = <<"test">>,
Self = self(),
DefaultAction = fun(timeout) -> Self ! {timeout, Key} end,
{ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20),
receive
{timeout, Key} ->
ok
after 100 ->
ct:fail("emqx_ft_responder not called")
end,
?assertExit(
{noproc, _},
emqx_ft_responder:ack(Key, oops)
).
t_unknown_msgs(_Config) ->
{ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100),
Pid ! {unknown_msg, <<"test">>},
ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}),
?assertEqual(
{error, unknown_call},
gen_server:call(Pid, {unknown_call, <<"test">>})
).

199
apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl Normal file
View File

@ -0,0 +1,199 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_exporter_s3_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-define(assertS3Data(Data, Url),
case httpc:request(Url) of
{ok, {{_StatusLine, 200, "OK"}, _Headers, Body}} ->
?assertEqual(Data, list_to_binary(Body), "S3 data mismatch");
OtherResponse ->
ct:fail("Unexpected response: ~p", [OtherResponse])
end
).
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
ok.
set_special_configs(Config) ->
fun
(emqx_ft) ->
Storage = emqx_ft_test_helpers:local_storage(Config, #{
exporter => s3, bucket_name => ?config(bucket_name, Config)
}),
emqx_ft_test_helpers:load_config(#{<<"enable">> => true, <<"storage">> => Storage});
(_) ->
ok
end.
init_per_testcase(Case, Config0) ->
ClientId = atom_to_binary(Case),
BucketName = create_bucket(),
Config1 = [{bucket_name, BucketName}, {clientid, ClientId} | Config0],
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config1)),
Config1.
end_per_testcase(_Case, _Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]),
ok.
%%--------------------------------------------------------------------
%% Test Cases
%%-------------------------------------------------------------------
t_happy_path(Config) ->
ClientId = ?config(clientid, Config),
FileId = <<"🌚"/utf8>>,
Name = "cool_name",
Data = <<"data"/utf8>>,
?assertEqual(
ok,
emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data)
),
{ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(),
?assertS3Data(Data, Uri),
Key = binary_to_list(ClientId) ++ "/" ++ binary_to_list(FileId) ++ "/" ++ Name,
Meta = erlcloud_s3:get_object_metadata(
?config(bucket_name, Config), Key, emqx_ft_test_helpers:aws_config()
),
?assertEqual(
ClientId,
metadata_field("clientid", Meta)
),
?assertEqual(
FileId,
metadata_field("fileid", Meta)
),
NameBin = list_to_binary(Name),
?assertMatch(
#{
<<"name">> := NameBin,
<<"size">> := 4
},
emqx_utils_json:decode(metadata_field("filemeta", Meta), [return_maps])
).
t_upload_error(Config) ->
ClientId = ?config(clientid, Config),
FileId = <<"🌚"/utf8>>,
Name = "cool_name",
Data = <<"data"/utf8>>,
{ok, _} = emqx_conf:update(
[file_transfer, storage, local, exporter, s3, bucket], <<"invalid-bucket">>, #{}
),
?assertEqual(
{error, unspecified_error},
emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data)
).
t_paging(Config) ->
ClientId = ?config(clientid, Config),
N = 1050,
FileId = fun integer_to_binary/1,
Name = "cool_name",
Data = fun integer_to_binary/1,
ok = lists:foreach(
fun(I) ->
ok = emqx_ft_test_helpers:upload_file(ClientId, FileId(I), Name, Data(I))
end,
lists:seq(1, N)
),
{ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(#{transfer => {ClientId, FileId(123)}}),
?assertS3Data(Data(123), Uri),
lists:foreach(
fun(PageSize) ->
Pages = file_pages(#{limit => PageSize}),
?assertEqual(
expected_page_count(PageSize, N),
length(Pages)
),
FileIds = [
FId
|| #{transfer := {_, FId}} <- lists:concat(Pages)
],
?assertEqual(
lists:sort([FileId(I) || I <- lists:seq(1, N)]),
lists:sort(FileIds)
)
end,
%% less than S3 limit, greater than S3 limit
[20, 550]
).
t_invalid_cursor(_Config) ->
InvalidUtf8 = <<16#80>>,
?assertError(
{badarg, cursor},
emqx_ft_storage:files(#{following => InvalidUtf8})
).
%%--------------------------------------------------------------------
%% Helper Functions
%%--------------------------------------------------------------------
expected_page_count(PageSize, Total) ->
case Total rem PageSize of
0 -> Total div PageSize;
_ -> Total div PageSize + 1
end.
file_pages(Query) ->
case emqx_ft_storage:files(Query) of
{ok, #{items := Items, cursor := NewCursor}} ->
[Items] ++ file_pages(Query#{following => NewCursor});
{ok, #{items := Items}} ->
[Items];
{error, Error} ->
ct:fail("Failed to download files: ~p", [Error])
end.
metadata_field(Field, Meta) ->
Key = "x-amz-meta-" ++ Field,
case lists:keyfind(Key, 1, Meta) of
{Key, Value} -> list_to_binary(Value);
false -> false
end.
create_bucket() ->
BucketName = emqx_s3_test_helpers:unique_bucket(),
_ = application:ensure_all_started(lhttpc),
ok = erlcloud_s3:create_bucket(BucketName, emqx_ft_test_helpers:aws_config()),
BucketName.

93
apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl Normal file
View File

@ -0,0 +1,93 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
all() ->
[
{group, cluster}
].
-define(CLUSTER_CASES, [t_multinode_exports]).
groups() ->
[
{cluster, [sequence], ?CLUSTER_CASES}
].
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft]),
ok.
init_per_testcase(Case, Config) ->
[{tc, Case} | Config].
end_per_testcase(_Case, _Config) ->
ok.
init_per_group(cluster, Config) ->
Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1),
[{additional_node, Node} | Config];
init_per_group(_Group, Config) ->
Config.
end_per_group(cluster, Config) ->
ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config));
end_per_group(_Group, _Config) ->
ok.
%%--------------------------------------------------------------------
%% Tests
%%--------------------------------------------------------------------
t_multinode_exports(Config) ->
Node1 = ?config(additional_node, Config),
ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1),
Node2 = node(),
ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2),
?assertMatch(
[
#{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"},
#{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"}
],
lists:sort(list_files(Config))
).
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
client_id(Config) ->
atom_to_binary(?config(tc, Config), utf8).
storage(Config) ->
RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)},
#{storage := #{local := Storage}} = emqx_ft_schema:translate(RawConfig),
Storage.
list_files(Config) ->
{ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}),
Files.

363
apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl Normal file
View File

@ -0,0 +1,363 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_gc_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("snabbkaffe/include/test_macros.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
_ = application:load(emqx_ft),
ok = emqx_common_test_helpers:start_apps([]),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([]),
ok.
init_per_testcase(TC, Config) ->
SegmentsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, segments]),
ExportsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, exports]),
ok = emqx_common_test_helpers:start_app(
emqx_ft,
fun(emqx_ft) ->
emqx_ft_test_helpers:load_config(#{
<<"enable">> => true,
<<"storage">> => #{
<<"local">> => #{
<<"segments">> => #{<<"root">> => SegmentsRoot},
<<"exporter">> => #{
<<"local">> => #{<<"root">> => ExportsRoot}
}
}
}
})
end
),
ok = snabbkaffe:start_trace(),
Config.
end_per_testcase(_TC, _Config) ->
ok = snabbkaffe:stop(),
ok = application:stop(emqx_ft),
ok.
%%
-define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)).
t_gc_triggers_periodically(_Config) ->
Interval = 500,
ok = set_gc_config(interval, Interval),
ok = emqx_ft_storage_fs_gc:reset(),
?check_trace(
timer:sleep(Interval * 3),
fun(Trace) ->
[Event, _ | _] = ?of_kind(garbage_collection, Trace),
?assertMatch(
#{
stats := #gcstats{
files = 0,
directories = 0,
space = 0,
errors = #{} = Errors
}
} when map_size(Errors) == 0,
Event
)
end
).
t_gc_triggers_manually(_Config) ->
?check_trace(
?assertMatch(
#gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when
map_size(Errors) == 0,
emqx_ft_storage_fs_gc:collect()
),
fun(Trace) ->
[Event] = ?of_kind(garbage_collection, Trace),
?assertMatch(
#{stats := #gcstats{}},
Event
)
end
).
t_gc_complete_transfers(_Config) ->
{local, Storage} = emqx_ft_storage:backend(),
ok = set_gc_config(minimum_segments_ttl, 0),
ok = set_gc_config(maximum_segments_ttl, 3),
ok = set_gc_config(interval, 500),
ok = emqx_ft_storage_fs_gc:reset(),
Transfers = [
{
T1 = {<<"client1">>, mk_file_id()},
#{name => "cat.cur", segments_ttl => 10},
emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16)
},
{
T2 = {<<"client2">>, mk_file_id()},
#{name => "cat.ico", segments_ttl => 10},
emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64)
},
{
T3 = {<<"client42">>, mk_file_id()},
#{name => "cat.jpg", segments_ttl => 10},
emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024)
}
],
% 1. Start all transfers
TransferSizes = emqx_utils:pmap(
fun(Transfer) -> start_transfer(Storage, Transfer) end,
Transfers
),
?assertEqual([S1, S2, S3], TransferSizes),
?assertMatch(
#gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0,
emqx_ft_storage_fs_gc:collect()
),
% 2. Complete just the first transfer
{ok, {ok, Event}} = ?wait_async_action(
?assertEqual(ok, complete_transfer(Storage, T1, S1)),
#{?snk_kind := garbage_collection},
1000
),
?assertMatch(
#{
stats := #gcstats{
files = Files,
directories = 2,
space = Space,
errors = #{} = Es
}
} when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0,
Event
),
% 3. Complete rest of transfers
{ok, Sub} = snabbkaffe_collector:subscribe(
?match_event(#{?snk_kind := garbage_collection}),
2,
1000,
0
),
?assertEqual(
[ok, ok],
emqx_utils:pmap(
fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end,
[{T2, S2}, {T3, S3}]
)
),
{ok, Events} = snabbkaffe_collector:receive_events(Sub),
CFiles = lists:sum([Stats#gcstats.files || #{stats := Stats} <- Events]),
CDirectories = lists:sum([Stats#gcstats.directories || #{stats := Stats} <- Events]),
CSpace = lists:sum([Stats#gcstats.space || #{stats := Stats} <- Events]),
CErrors = lists:foldl(
fun maps:merge/2,
#{},
[Stats#gcstats.errors || #{stats := Stats} <- Events]
),
?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles),
?assertEqual(2 + 2, CDirectories),
?assertMatch(Space when Space > S2 + S3, CSpace),
?assertMatch(Errors when map_size(Errors) == 0, CErrors),
% 4. Ensure that empty transfer directories will be eventually collected
{ok, _} = ?block_until(
#{
?snk_kind := garbage_collection,
stats := #gcstats{
files = 0,
directories = 6,
space = 0
}
},
5000,
0
).
t_gc_incomplete_transfers(_Config) ->
ok = set_gc_config(minimum_segments_ttl, 0),
ok = set_gc_config(maximum_segments_ttl, 4),
{local, Storage} = emqx_ft_storage:backend(),
Transfers = [
{
{<<"client43"/utf8>>, <<"file-🦕"/utf8>>},
#{name => "dog.cur", segments_ttl => 1},
emqx_ft_content_gen:new({?LINE, S1 = 123}, SS1 = 32)
},
{
{<<"client44">>, <<"file-🦖"/utf8>>},
#{name => "dog.ico", segments_ttl => 2},
emqx_ft_content_gen:new({?LINE, S2 = 456}, SS2 = 64)
},
{
{<<"client1337">>, <<"file-🦀"/utf8>>},
#{name => "dog.jpg", segments_ttl => 3000},
emqx_ft_content_gen:new({?LINE, S3 = 7890}, SS3 = 128)
},
{
{<<"client31337">>, <<"file-⏳"/utf8>>},
#{name => "dog.jpg"},
emqx_ft_content_gen:new({?LINE, S4 = 1230}, SS4 = 256)
}
],
% 1. Start transfers, send all the segments but don't trigger completion.
_ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers),
% 2. Enable periodic GC every 0.5 seconds.
ok = set_gc_config(interval, 500),
ok = emqx_ft_storage_fs_gc:reset(),
% 3. First we need the first transfer to be collected.
{ok, _} = ?block_until(
#{
?snk_kind := garbage_collection,
stats := #gcstats{
files = Files,
directories = 4,
space = Space
}
} when Files == (?NSEGS(S1, SS1)) andalso Space > S1,
5000,
0
),
% 4. Then the second one.
{ok, _} = ?block_until(
#{
?snk_kind := garbage_collection,
stats := #gcstats{
files = Files,
directories = 4,
space = Space
}
} when Files == (?NSEGS(S2, SS2)) andalso Space > S2,
5000,
0
),
% 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL.
{ok, _} = ?block_until(
#{
?snk_kind := garbage_collection,
stats := #gcstats{
files = Files,
directories = 4 * 2,
space = Space
}
} when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4,
5000,
0
).
t_gc_handling_errors(_Config) ->
ok = set_gc_config(minimum_segments_ttl, 0),
ok = set_gc_config(maximum_segments_ttl, 0),
{local, Storage} = emqx_ft_storage:backend(),
Transfer1 = {<<"client1">>, mk_file_id()},
Transfer2 = {<<"client2">>, mk_file_id()},
Filemeta = #{name => "oops.pdf"},
Size = 420,
SegSize = 16,
_ = start_transfer(
Storage,
{Transfer1, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)}
),
_ = start_transfer(
Storage,
{Transfer2, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)}
),
% 1. Throw some chaos in the transfer directory.
DirFragment1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, fragment),
DirTemporary1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, temporary),
PathShadyLink = filename:join(DirTemporary1, "linked-here"),
ok = file:make_symlink(DirFragment1, PathShadyLink),
DirTransfer2 = emqx_ft_storage_fs:get_subdir(Storage, Transfer2),
PathTripUp = filename:join(DirTransfer2, "trip-up-here"),
ok = file:write_file(PathTripUp, <<"HAHA">>),
ok = timer:sleep(timer:seconds(1)),
% 2. Observe the errors are reported consistently.
?check_trace(
?assertMatch(
#gcstats{
files = Files,
directories = 3,
space = Space,
errors = #{
% NOTE: dangling symlink looks like `enoent` for some reason
{file, PathShadyLink} := {unexpected, _},
{directory, DirTransfer2} := eexist
}
} when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2,
emqx_ft_storage_fs_gc:collect()
),
fun(Trace) ->
?assertMatch(
[
#{
errors := #{
{file, PathShadyLink} := {unexpected, _},
{directory, DirTransfer2} := eexist
}
}
],
?of_kind("garbage_collection_errors", Trace)
)
end
).
%%
set_gc_config(Name, Value) ->
emqx_config:put([file_transfer, storage, local, segments, gc, Name], Value).
start_transfer(Storage, {Transfer, Meta, Gen}) ->
?assertEqual(
ok,
emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta)
),
emqx_ft_content_gen:fold(
fun({Content, SegmentNum, #{chunk_size := SegmentSize}}, _Transferred) ->
Offset = (SegmentNum - 1) * SegmentSize,
?assertEqual(
ok,
emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content})
),
Offset + byte_size(Content)
end,
0,
Gen
).
complete_transfer(Storage, Transfer, Size) ->
complete_transfer(Storage, Transfer, Size, 100).
complete_transfer(Storage, Transfer, Size, Timeout) ->
{async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size),
MRef = erlang:monitor(process, Pid),
Pid ! kickoff,
receive
{'DOWN', MRef, process, Pid, {shutdown, Result}} ->
Result
after Timeout ->
ct:fail("Assembler did not finish in time")
end.
mk_file_id() ->
emqx_guid:to_hexstr(emqx_guid:gen()).

153
apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl Normal file
View File

@ -0,0 +1,153 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_storage_fs_reader_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_ft]),
ok.
init_per_testcase(_Case, Config) ->
file:make_dir(?config(data_dir, Config)),
Data = <<"hello world">>,
Path = expand_path(Config, "test_file"),
ok = mk_test_file(Path, Data),
[{path, Path} | Config].
end_per_testcase(_Case, _Config) ->
ok.
t_successful_read(Config) ->
Path = ?config(path, Config),
{ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path),
?assertEqual(
{ok, <<"hello ">>},
emqx_ft_storage_fs_reader:read(ReaderPid, 6)
),
?assertEqual(
{ok, <<"world">>},
emqx_ft_storage_fs_reader:read(ReaderPid, 6)
),
?assertEqual(
eof,
emqx_ft_storage_fs_reader:read(ReaderPid, 6)
),
?assertNot(is_process_alive(ReaderPid)).
t_caller_dead(Config) ->
erlang:process_flag(trap_exit, true),
Path = ?config(path, Config),
CallerPid = spawn_link(
fun() ->
receive
stop -> ok
end
end
),
{ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(CallerPid, Path),
_ = erlang:monitor(process, ReaderPid),
?assertEqual(
{ok, <<"hello ">>},
emqx_ft_storage_fs_reader:read(ReaderPid, 6)
),
CallerPid ! stop,
receive
{'DOWN', _, process, ReaderPid, _} -> ok
after 1000 ->
ct:fail("Reader process did not die")
end.
t_tables(Config) ->
Path = ?config(path, Config),
{ok, ReaderPid0} = emqx_ft_storage_fs_reader:start_link(self(), Path),
ReaderQH0 = emqx_ft_storage_fs_reader:table(ReaderPid0, 6),
?assertEqual(
[<<"hello ">>, <<"world">>],
qlc:eval(ReaderQH0)
),
{ok, ReaderPid1} = emqx_ft_storage_fs_reader:start_link(self(), Path),
ReaderQH1 = emqx_ft_storage_fs_reader:table(ReaderPid1),
?assertEqual(
[<<"hello world">>],
qlc:eval(ReaderQH1)
).
t_bad_messages(Config) ->
Path = ?config(path, Config),
{ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path),
ReaderPid ! {bad, message},
gen_server:cast(ReaderPid, {bad, message}),
?assertEqual(
{error, {bad_call, {bad, message}}},
gen_server:call(ReaderPid, {bad, message})
).
t_nonexistent_file(_Config) ->
?assertEqual(
{error, enoent},
emqx_ft_storage_fs_reader:start_link(self(), "/a/b/c/bar")
).
t_start_supervised(Config) ->
Path = ?config(path, Config),
{ok, ReaderPid} = emqx_ft_storage_fs_reader:start_supervised(self(), Path),
?assertEqual(
{ok, <<"hello ">>},
emqx_ft_storage_fs_reader:read(ReaderPid, 6)
).
t_rpc_error(_Config) ->
ReaderQH = emqx_ft_storage_fs_reader:table(fake_remote_pid('dummy@127.0.0.1'), 6),
?assertEqual(
[],
qlc:eval(ReaderQH)
).
mk_test_file(Path, Data) ->
ok = file:write_file(Path, Data).
expand_path(Config, Filename) ->
filename:join([?config(data_dir, Config), Filename]).
%% This is a hack to create a pid that is not registered on the local node.
%% https://www.erlang.org/doc/apps/erts/erl_ext_dist.html#new_pid_ext
fake_remote_pid(Node) ->
<<131, NodeAtom/binary>> = term_to_binary(Node),
PidBin = <<131, 88, NodeAtom/binary, 1:32/big, 1:32/big, 1:32/big>>,
binary_to_term(PidBin).

128
apps/emqx_ft/test/emqx_ft_test_helpers.erl Normal file
View File

@ -0,0 +1,128 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_ft_test_helpers).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("common_test/include/ct.hrl").
-define(S3_HOST, <<"minio">>).
-define(S3_PORT, 9000).
start_additional_node(Config, Name) ->
emqx_common_test_helpers:start_slave(
Name,
[
{apps, [emqx_ft]},
{join_to, node()},
{configure_gen_rpc, true},
{env_handler, env_handler(Config)}
]
).
stop_additional_node(Node) ->
ok = rpc:call(Node, ekka, leave, []),
ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]),
ok = emqx_common_test_helpers:stop_slave(Node),
ok.
env_handler(Config) ->
fun
(emqx_ft) ->
load_config(#{<<"enable">> => true, <<"storage">> => local_storage(Config)});
(_) ->
ok
end.
local_storage(Config) ->
local_storage(Config, #{exporter => local}).
local_storage(Config, Opts) ->
#{
<<"local">> => #{
<<"segments">> => #{<<"root">> => root(Config, node(), [segments])},
<<"exporter">> => exporter(Config, Opts)
}
}.
exporter(Config, #{exporter := local}) ->
#{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}};
exporter(_Config, #{exporter := s3, bucket_name := BucketName}) ->
BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp),
#{
<<"s3">> => BaseConfig#{
<<"bucket">> => list_to_binary(BucketName),
<<"host">> => ?S3_HOST,
<<"port">> => ?S3_PORT
}
}.
load_config(Config) ->
emqx_common_test_helpers:load_config(emqx_ft_schema, #{<<"file_transfer">> => Config}).
tcp_port(Node) ->
{_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]),
Port.
root(Config, Node, Tail) ->
iolist_to_binary(filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail])).
start_client(ClientId) ->
start_client(ClientId, node()).
start_client(ClientId, Node) ->
Port = tcp_port(Node),
{ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]),
{ok, _} = emqtt:connect(Client),
Client.
upload_file(ClientId, FileId, Name, Data) ->
upload_file(ClientId, FileId, Name, Data, node()).
upload_file(ClientId, FileId, Name, Data, Node) ->
C1 = start_client(ClientId, Node),
Size = byte_size(Data),
Meta = #{
name => Name,
expire_at => erlang:system_time(_Unit = second) + 3600,
size => Size
},
MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)),
ct:pal("MetaPayload = ~ts", [MetaPayload]),
MetaTopic = <<"$file/", FileId/binary, "/init">>,
{ok, #{reason_code_name := success}} = emqtt:publish(C1, MetaTopic, MetaPayload, 1),
{ok, #{reason_code_name := success}} = emqtt:publish(
C1, <<"$file/", FileId/binary, "/0">>, Data, 1
),
FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>,
FinResult =
case emqtt:publish(C1, FinTopic, <<>>, 1) of
{ok, #{reason_code_name := success}} ->
ok;
{ok, #{reason_code_name := Error}} ->
{error, Error}
end,
ok = emqtt:stop(C1),
FinResult.
aws_config() ->
emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT).

Some files were not shown because too many files have changed in this diff Show More